Update frontend

- Add HudCanvas component to Waveform
- Allow waveform to be selectable
- Fix selection rendering on viewport change
- Add spacebar handler

frontend/src/App.tsx

@@ -25,7 +25,7 @@ const initialViewportCanvasPixels = 100;
 
 const apiURL = process.env.REACT_APP_API_URL || 'http://localhost:8888';
 
-// Frames represents a selection of audio frames.
+// Frames represents a range of audio frames.
 export interface Frames {
   start: number;
   end: number;
@@ -41,7 +41,8 @@ const audio = document.createElement('audio');
 
 function App(): JSX.Element {
   const [mediaSet, setMediaSet] = useState<MediaSet | null>(null);
-  const [viewport, setViewport] = useState({ start: 0, end: 0 });
+  const [viewport, setViewport] = useState<Frames>({ start: 0, end: 0 });
+  const [selection, setSelection] = useState<Frames>({ start: 0, end: 0 });
   const [overviewPeaks, setOverviewPeaks] = useState<Observable<number[]>>(
     from([])
   );
@@ -81,17 +82,35 @@ function App(): JSX.Element {
     }
 
     const intervalID = setInterval(() => {
-      if (video.currentTime == positionRef.current.currentTime) {
+      const currTime = audio.currentTime;
+      if (currTime == positionRef.current.currentTime) {
         return;
       }
       const duration = mediaSet.audioFrames / mediaSet.audioSampleRate;
-      const percent = (video.currentTime / duration) * 100;
+      const percent = (currTime / duration) * 100;
 
-      setPosition({ currentTime: video.currentTime, percent: percent });
+      // check if the end of selection has been passed, and pause if so:
+      if (
+        currentTimeToFrame(position.currentTime) < selection.end &&
+        currentTimeToFrame(currTime) >= selection.end
+      ) {
+        handlePause();
+      }
+
+      // update the current position
+      setPosition({ currentTime: audio.currentTime, percent: percent });
     }, updatePlayerPositionIntevalMillis);
 
     return () => clearInterval(intervalID);
-  }, [mediaSet]);
+  }, [mediaSet, selection]);
+
+  // bind to keypress handler.
+  // selection is a dependency of the handleKeyPress handler, and must be
+  // included here.
+  useEffect(() => {
+    document.addEventListener('keypress', handleKeyPress);
+    return () => document.removeEventListener('keypress', handleKeyPress);
+  }, [selection]);
 
   // load audio when MediaSet is loaded:
   useEffect(() => {
@@ -161,23 +180,57 @@ function App(): JSX.Element {
 
   // handlers
 
-  const handleOverviewSelectionChange = (newViewport: Frames) => {
+  const handleKeyPress = useCallback(
-    if (mediaSet == null) {
+    (evt: KeyboardEvent) => {
-      return;
+      if (evt.code != 'Space') {
-    }
+        return;
-    console.log('set new viewport', newViewport);
+      }
-    setViewport({ ...newViewport });
 
-    if (!audio.paused) {
+      if (audio.paused) {
-      return;
+        handlePlay();
-    }
+      } else {
+        handlePause();
+      }
+    },
+    [selection]
+  );
 
-    const ratio = newViewport.start / mediaSet.audioFrames;
+  // handler called when the selection in the overview (zoom setting) is changed.
-    const currentTime =
+  const handleOverviewSelectionChange = useCallback(
-      (mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio;
+    (newViewport: Frames) => {
-    audio.currentTime = currentTime;
+      if (mediaSet == null) {
-    video.currentTime = currentTime;
+        return;
-  };
+      }
+      console.log('set new viewport', newViewport);
+      setViewport({ ...newViewport });
+
+      if (!audio.paused) {
+        return;
+      }
+
+      setPositionFromFrame(newViewport.start);
+    },
+    [mediaSet, audio, video]
+  );
+
+  // handler called when the selection in the main waveform view is changed.
+  const handleWaveformSelectionChange = useCallback(
+    (selection: Frames) => {
+      setSelection(selection);
+
+      if (mediaSet == null) {
+        return;
+      }
+
+      // move playback position to start of selection
+      const ratio = selection.start / mediaSet.audioFrames;
+      const currentTime =
+        (mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio;
+      audio.currentTime = currentTime;
+      video.currentTime = currentTime;
+    },
+    [mediaSet, audio, video]
+  );
 
   const handlePlay = useCallback(() => {
     audio.play();
@@ -187,7 +240,39 @@ function App(): JSX.Element {
   const handlePause = useCallback(() => {
     video.pause();
     audio.pause();
-  }, [audio, video]);
+
+    if (selection.start != selection.end) {
+      setPositionFromFrame(selection.start);
+    }
+  }, [audio, video, selection]);
+
+  const setPositionFromFrame = useCallback(
+    (frame: number) => {
+      if (mediaSet == null) {
+        return;
+      }
+      const ratio = frame / mediaSet.audioFrames;
+      const currentTime =
+        (mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio;
+      audio.currentTime = currentTime;
+      video.currentTime = currentTime;
+    },
+    [mediaSet, audio, video]
+  );
+
+  // helpers
+
+  const currentTimeToFrame = useCallback(
+    (currentTime: number): number => {
+      if (mediaSet == null) {
+        return 0;
+      }
+      const dur = mediaSet.audioFrames / mediaSet.audioSampleRate;
+      const ratio = currentTime / dur;
+      return Math.round(mediaSet.audioFrames * ratio);
+    },
+    [mediaSet]
+  );
 
   // render component
 
@@ -229,6 +314,7 @@ function App(): JSX.Element {
             position={position}
             viewport={viewport}
             offsetPixels={offsetPixels}
+            onSelectionChange={handleWaveformSelectionChange}
           />
 
           <SeekBar

frontend/src/Helpers.tsx

@@ -1,23 +1,10 @@
 import { MouseEvent } from 'react';
-import { Frames } from './App';
-
-// TODO: pass CanvasLogicalWidth as an argument instead.
-import { CanvasLogicalWidth } from './Waveform';
 
 interface Point {
   x: number;
   y: number;
 }
 
-// TODO: add tests
-export const mouseEventToCanvasX = (
-  evt: MouseEvent<HTMLCanvasElement>
-): number => {
-  const rect = evt.currentTarget.getBoundingClientRect();
-  const elementX = evt.clientX - rect.left;
-  return (elementX * CanvasLogicalWidth) / rect.width;
-};
-
 // TODO: add tests
 export const mouseEventToCanvasPoint = (
   evt: MouseEvent<HTMLCanvasElement>
@@ -31,26 +18,3 @@ export const mouseEventToCanvasPoint = (
     y: (elementY * evt.currentTarget.height) / rect.height,
   };
 };
-
-// TODO: add tests
-export const canvasXToFrame = (x: number, numFrames: number): number => {
-  return Math.floor((x / CanvasLogicalWidth) * numFrames);
-};
-
-// TODO: add tests
-// secsToCanvasX returns the logical x coordinate for a given position
-// marker. It is null if the marker is outside of the current viewport.
-export const secsToCanvasX = (
-  secs: number,
-  sampleRate: number,
-  viewport: Frames
-): number | null => {
-  const frame = Math.round(secs * sampleRate);
-  if (frame < viewport.start || frame > viewport.end) {
-    return null;
-  }
-
-  const logicalPixelsPerFrame =
-    CanvasLogicalWidth / (viewport.end - viewport.start);
-  return (frame - viewport.start) * logicalPixelsPerFrame;
-};

frontend/src/HudCanvas.tsx

@@ -1,11 +1,18 @@
 import { useState, useEffect, useRef, MouseEvent } from 'react';
-import { VideoPosition } from './App';
+
+interface Styles {
+  borderLineWidth: number;
+  borderStrokeStyle: string;
+  positionLineWidth: number;
+  positionStrokeStyle: string;
+}
 
 interface Props {
   width: number;
   height: number;
   zIndex: number;
-  position: VideoPosition;
+  styles: Styles;
+  position: number | null;
   selection: Selection;
   onSelectionChange: (selection: Selection) => void;
 }
@@ -36,6 +43,12 @@ export const HudCanvas: React.FC<Props> = ({
   width,
   height,
   zIndex,
+  styles: {
+    borderLineWidth,
+    borderStrokeStyle,
+    positionLineWidth,
+    positionStrokeStyle,
+  },
   position,
   selection,
   onSelectionChange,
@@ -91,26 +104,31 @@ export const HudCanvas: React.FC<Props> = ({
       }
 
       ctx.beginPath();
-      ctx.strokeStyle = 'red';
+      ctx.strokeStyle = borderStrokeStyle;
-      ctx.lineWidth = 4;
+      ctx.lineWidth = borderLineWidth;
       const alpha = hoverState == HoverState.OverSelection ? '0.15' : '0.13';
       ctx.fillStyle = `rgba(255, 255, 255, ${alpha})`;
       ctx.rect(
         currentSelection.start,
-        2,
+        borderLineWidth,
         currentSelection.end - currentSelection.start,
-        canvas.height - 10
+        canvas.height - borderLineWidth * 2
       );
       ctx.fill();
       ctx.stroke();
 
       // draw position marker
 
-      const markerX = canvas.width * (position.percent / 100);
+      if (position == null) {
+        return;
+      }
+
       ctx.beginPath();
-      ctx.moveTo(markerX, 0);
+      ctx.strokeStyle = positionStrokeStyle;
+      ctx.lineWidth = positionLineWidth;
+      ctx.moveTo(position, 0);
       ctx.lineWidth = 4;
-      ctx.lineTo(markerX, canvas.height - 4);
+      ctx.lineTo(position, canvas.height - 4);
       ctx.stroke();
     });
   }, [selection, newSelection, position]);
@@ -202,19 +220,19 @@ export const HudCanvas: React.FC<Props> = ({
           break;
         }
 
-        setNewSelection({ ...newSelection, start: start });
+        setNewSelection({ ...selection, start: start });
         break;
       }
       case Mode.ResizingEnd: {
         const diff = x - moveOffsetX.current;
-        const start = constrainXToCanvas(selection.end + diff);
+        const end = constrainXToCanvas(selection.end + diff);
 
-        if (start < selection.start) {
+        if (end < selection.start) {
-          setNewSelection({ start: Math.max(0, start), end: selection.start });
+          setNewSelection({ start: Math.max(0, end), end: selection.start });
           break;
         }
 
-        setNewSelection({ ...newSelection, end: start });
+        setNewSelection({ ...selection, end: end });
         break;
       }
       case Mode.Dragging: {

frontend/src/Overview.tsx

@@ -33,6 +33,7 @@ export const Overview: React.FC<Props> = ({
   onSelectionChange,
 }: Props) => {
   const [selectedPixels, setSelectedPixels] = useState({ start: 0, end: 0 });
+  const [positionPixels, setPositionPixels] = useState(0);
 
   // side effects
 
@@ -48,6 +49,14 @@ export const Overview: React.FC<Props> = ({
     });
   }, [viewport, mediaSet]);
 
+  // convert position from frames to canvas pixels:
+  useEffect(() => {
+    const ratio =
+      position.currentTime / (mediaSet.audioFrames / mediaSet.audioSampleRate);
+    setPositionPixels(Math.round(ratio * CanvasLogicalWidth));
+    frames;
+  }, [mediaSet, position]);
+
   // handlers
 
   // convert selection change from canvas pixels to frames, and trigger callback.
@@ -70,6 +79,13 @@ export const Overview: React.FC<Props> = ({
     height: `${height}px`,
   } as React.CSSProperties;
 
+  const hudStyles = {
+    borderLineWidth: 4,
+    borderStrokeStyle: 'red',
+    positionLineWidth: 4,
+    positionStrokeStyle: 'red',
+  };
+
   return (
     <>
       <div style={containerStyles}>
@@ -87,7 +103,8 @@ export const Overview: React.FC<Props> = ({
           width={CanvasLogicalWidth}
           height={CanvasLogicalHeight}
           zIndex={1}
-          position={position}
+          styles={hudStyles}
+          position={positionPixels}
           selection={selectedPixels}
           onSelectionChange={handleSelectionChange}
         />

frontend/src/Waveform.tsx

@@ -1,8 +1,8 @@
-import { useEffect, useState, useRef } from 'react';
+import { useEffect, useState, useCallback } from 'react';
 import { Frames, VideoPosition, newRPC } from './App';
 import { MediaSetServiceClientImpl, MediaSet } from './generated/media_set';
 import { WaveformCanvas } from './WaveformCanvas';
-import { secsToCanvasX } from './Helpers';
+import { Selection, HudCanvas } from './HudCanvas';
 import { from, Observable } from 'rxjs';
 import { bufferCount } from 'rxjs/operators';
 
@@ -11,6 +11,7 @@ interface Props {
   position: VideoPosition;
   viewport: Frames;
   offsetPixels: number;
+  onSelectionChange: (selection: Selection) => void;
 }
 
 export const CanvasLogicalWidth = 2000;
@@ -21,9 +22,15 @@ export const Waveform: React.FC<Props> = ({
   position,
   viewport,
   offsetPixels,
+  onSelectionChange,
 }: Props) => {
   const [peaks, setPeaks] = useState<Observable<number[]>>(from([]));
-  const hudCanvasRef = useRef<HTMLCanvasElement>(null);
+  const [selectedFrames, setSelectedFrames] = useState({ start: 0, end: 0 });
+  const [selectedPixels, setSelectedPixels] = useState({
+    start: 0,
+    end: 0,
+  });
+  const [positionPixels, setPositionPixels] = useState<number | null>(0);
 
   // effects
 
@@ -57,41 +64,75 @@ export const Waveform: React.FC<Props> = ({
     })();
   }, [viewport]);
 
-  // render HUD
+  // convert position to canvas pixels
   useEffect(() => {
-    const canvas = hudCanvasRef.current;
+    const frame = Math.round(position.currentTime * mediaSet.audioSampleRate);
-    if (canvas == null) {
+    if (frame < viewport.start || frame > viewport.end) {
+      setPositionPixels(null);
       return;
     }
+    const logicalPixelsPerFrame =
+      CanvasLogicalWidth / (viewport.end - viewport.start);
+    const positionPixels = (frame - viewport.start) * logicalPixelsPerFrame;
+    setPositionPixels(positionPixels);
+  }, [mediaSet, position, viewport]);
 
-    const ctx = canvas.getContext('2d');
+  // update selectedPixels on viewport change
-    if (ctx == null) {
+  useEffect(() => {
-      console.error('no hud 2d context available');
+    const start = frameToCanvasX(selectedFrames.start);
-      return;
+    const end = frameToCanvasX(selectedFrames.end);
+
+    // more verbose than it has to be to make TypeScript happy
+    if (start == null && end == null) {
+      setSelectedPixels({ start: 0, end: 0 });
+    } else if (start == null && end != null) {
+      setSelectedPixels({ start: 0, end: end });
+    } else if (start != null && end == null) {
+      setSelectedPixels({ start: 0, end: CanvasLogicalWidth });
+    } else if (start != null && end != null) {
+      setSelectedPixels({ start, end });
+    } else {
+      console.error('unreachable');
     }
+  }, [viewport]);
 
-    ctx.clearRect(0, 0, canvas.width, canvas.height);
+  // handlers
 
-    if (mediaSet == null) {
+  const handleSelectionChange = useCallback(
-      return;
+    (selection: Selection) => {
-    }
+      setSelectedPixels(selection);
 
-    const x = secsToCanvasX(
+      const framesPerPixel =
-      position.currentTime,
+        (viewport.end - viewport.start) / CanvasLogicalWidth;
-      mediaSet.audioSampleRate,
+      const selectedFrames = {
-      viewport
+        start: Math.round(viewport.start + selection.start * framesPerPixel),
-    );
+        end: Math.round(viewport.start + selection.end * framesPerPixel),
-    if (x == null) {
+      };
-      return;
-    }
 
-    ctx.strokeStyle = 'red';
+      setSelectedFrames(selectedFrames);
-    ctx.beginPath();
+      onSelectionChange(selectedFrames);
-    ctx.moveTo(x, 0);
+    },
-    ctx.lineWidth = 4;
+    [viewport]
-    ctx.lineTo(x, canvas.height);
+  );
-    ctx.stroke();
+
-  }, [mediaSet, position]);
+  // helpers
+
+  const frameToCanvasX = useCallback(
+    (frame: number): number | null => {
+      if (mediaSet == null) {
+        return null;
+      }
+
+      if (frame < viewport.start || frame > viewport.end) {
+        return null;
+      }
+
+      const pixelsPerFrame =
+        CanvasLogicalWidth / (viewport.end - viewport.start);
+      return (frame - viewport.start) * pixelsPerFrame;
+    },
+    [mediaSet, viewport]
+  );
 
   // render component
 
@@ -102,12 +143,12 @@ export const Waveform: React.FC<Props> = ({
     position: 'relative',
   } as React.CSSProperties;
 
-  const canvasStyles = {
+  const hudStyles = {
-    position: 'absolute',
+    borderLineWidth: 0,
-    width: '100%',
+    borderStrokeStyle: 'transparent',
-    height: '100%',
+    positionLineWidth: 6,
-    display: 'block',
+    positionStrokeStyle: 'red',
-  } as React.CSSProperties;
+  };
 
   return (
     <>
@@ -122,12 +163,15 @@ export const Waveform: React.FC<Props> = ({
           zIndex={0}
           alpha={1}
         ></WaveformCanvas>
-        <canvas
+        <HudCanvas
           width={CanvasLogicalWidth}
           height={CanvasLogicalHeight}
-          ref={hudCanvasRef}
+          zIndex={1}
-          style={canvasStyles}
+          styles={hudStyles}
-        ></canvas>
+          position={positionPixels}
+          selection={selectedPixels}
+          onSelectionChange={handleSelectionChange}
+        />
       </div>
     </>
   );