Update frontend

- Add HudCanvas component to Waveform - Allow waveform to be selectable - Fix selection rendering on viewport change - Add spacebar handler
2021-12-11 17:25:43 +01:00 · 2021-12-11 17:25:43 +01:00 · b876fb915a
parent 65cc365717
commit b876fb915a
5 changed files with 241 additions and 112 deletions
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@ -25,7 +25,7 @@ const initialViewportCanvasPixels = 100;

 const apiURL = process.env.REACT_APP_API_URL || 'http://localhost:8888';

-// Frames represents a selection of audio frames.
+// Frames represents a range of audio frames.
 export interface Frames {
  start: number;
  end: number;
@ -41,7 +41,8 @@ const audio = document.createElement('audio');

 function App(): JSX.Element {
  const [mediaSet, setMediaSet] = useState<MediaSet | null>(null);
-  const [viewport, setViewport] = useState({ start: 0, end: 0 });
+  const [viewport, setViewport] = useState<Frames>({ start: 0, end: 0 });
+  const [selection, setSelection] = useState<Frames>({ start: 0, end: 0 });
  const [overviewPeaks, setOverviewPeaks] = useState<Observable<number[]>>(
    from([])
  );
@ -81,17 +82,35 @@ function App(): JSX.Element {
    }

    const intervalID = setInterval(() => {
-      if (video.currentTime == positionRef.current.currentTime) {
+      const currTime = audio.currentTime;
+      if (currTime == positionRef.current.currentTime) {
        return;
      }
      const duration = mediaSet.audioFrames / mediaSet.audioSampleRate;
-      const percent = (video.currentTime / duration) * 100;
+      const percent = (currTime / duration) * 100;

-      setPosition({ currentTime: video.currentTime, percent: percent });
+      // check if the end of selection has been passed, and pause if so:
+      if (
+        currentTimeToFrame(position.currentTime) < selection.end &&
+        currentTimeToFrame(currTime) >= selection.end
+      ) {
+        handlePause();
+      }
+
+      // update the current position
+      setPosition({ currentTime: audio.currentTime, percent: percent });
    }, updatePlayerPositionIntevalMillis);

    return () => clearInterval(intervalID);
-  }, [mediaSet]);
+  }, [mediaSet, selection]);
+
+  // bind to keypress handler.
+  // selection is a dependency of the handleKeyPress handler, and must be
+  // included here.
+  useEffect(() => {
+    document.addEventListener('keypress', handleKeyPress);
+    return () => document.removeEventListener('keypress', handleKeyPress);
+  }, [selection]);

  // load audio when MediaSet is loaded:
  useEffect(() => {
@ -161,23 +180,57 @@ function App(): JSX.Element {

  // handlers

-  const handleOverviewSelectionChange = (newViewport: Frames) => {
-    if (mediaSet == null) {
-      return;
-    }
-    console.log('set new viewport', newViewport);
-    setViewport({ ...newViewport });
+  const handleKeyPress = useCallback(
+    (evt: KeyboardEvent) => {
+      if (evt.code != 'Space') {
+        return;
+      }

-    if (!audio.paused) {
-      return;
-    }
+      if (audio.paused) {
+        handlePlay();
+      } else {
+        handlePause();
+      }
+    },
+    [selection]
+  );

-    const ratio = newViewport.start / mediaSet.audioFrames;
-    const currentTime =
-      (mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio;
-    audio.currentTime = currentTime;
-    video.currentTime = currentTime;
-  };
+  // handler called when the selection in the overview (zoom setting) is changed.
+  const handleOverviewSelectionChange = useCallback(
+    (newViewport: Frames) => {
+      if (mediaSet == null) {
+        return;
+      }
+      console.log('set new viewport', newViewport);
+      setViewport({ ...newViewport });
+
+      if (!audio.paused) {
+        return;
+      }
+
+      setPositionFromFrame(newViewport.start);
+    },
+    [mediaSet, audio, video]
+  );
+
+  // handler called when the selection in the main waveform view is changed.
+  const handleWaveformSelectionChange = useCallback(
+    (selection: Frames) => {
+      setSelection(selection);
+
+      if (mediaSet == null) {
+        return;
+      }
+
+      // move playback position to start of selection
+      const ratio = selection.start / mediaSet.audioFrames;
+      const currentTime =
+        (mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio;
+      audio.currentTime = currentTime;
+      video.currentTime = currentTime;
+    },
+    [mediaSet, audio, video]
+  );

  const handlePlay = useCallback(() => {
    audio.play();
@ -187,7 +240,39 @@ function App(): JSX.Element {
  const handlePause = useCallback(() => {
    video.pause();
    audio.pause();
-  }, [audio, video]);
+
+    if (selection.start != selection.end) {
+      setPositionFromFrame(selection.start);
+    }
+  }, [audio, video, selection]);
+
+  const setPositionFromFrame = useCallback(
+    (frame: number) => {
+      if (mediaSet == null) {
+        return;
+      }
+      const ratio = frame / mediaSet.audioFrames;
+      const currentTime =
+        (mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio;
+      audio.currentTime = currentTime;
+      video.currentTime = currentTime;
+    },
+    [mediaSet, audio, video]
+  );
+
+  // helpers
+
+  const currentTimeToFrame = useCallback(
+    (currentTime: number): number => {
+      if (mediaSet == null) {
+        return 0;
+      }
+      const dur = mediaSet.audioFrames / mediaSet.audioSampleRate;
+      const ratio = currentTime / dur;
+      return Math.round(mediaSet.audioFrames * ratio);
+    },
+    [mediaSet]
+  );

  // render component

@ -229,6 +314,7 @@ function App(): JSX.Element {
            position={position}
            viewport={viewport}
            offsetPixels={offsetPixels}
+            onSelectionChange={handleWaveformSelectionChange}
          />

          <SeekBar
--- a/frontend/src/Helpers.tsx
+++ b/frontend/src/Helpers.tsx
@ -1,23 +1,10 @@
 import { MouseEvent } from 'react';
-import { Frames } from './App';
-
-// TODO: pass CanvasLogicalWidth as an argument instead.
-import { CanvasLogicalWidth } from './Waveform';

 interface Point {
  x: number;
  y: number;
 }

-// TODO: add tests
-export const mouseEventToCanvasX = (
-  evt: MouseEvent<HTMLCanvasElement>
-): number => {
-  const rect = evt.currentTarget.getBoundingClientRect();
-  const elementX = evt.clientX - rect.left;
-  return (elementX * CanvasLogicalWidth) / rect.width;
-};
-
 // TODO: add tests
 export const mouseEventToCanvasPoint = (
  evt: MouseEvent<HTMLCanvasElement>
@ -31,26 +18,3 @@ export const mouseEventToCanvasPoint = (
    y: (elementY * evt.currentTarget.height) / rect.height,
  };
 };
-
-// TODO: add tests
-export const canvasXToFrame = (x: number, numFrames: number): number => {
-  return Math.floor((x / CanvasLogicalWidth) * numFrames);
-};
-
-// TODO: add tests
-// secsToCanvasX returns the logical x coordinate for a given position
-// marker. It is null if the marker is outside of the current viewport.
-export const secsToCanvasX = (
-  secs: number,
-  sampleRate: number,
-  viewport: Frames
-): number | null => {
-  const frame = Math.round(secs * sampleRate);
-  if (frame < viewport.start || frame > viewport.end) {
-    return null;
-  }
-
-  const logicalPixelsPerFrame =
-    CanvasLogicalWidth / (viewport.end - viewport.start);
-  return (frame - viewport.start) * logicalPixelsPerFrame;
-};
--- a/frontend/src/HudCanvas.tsx
+++ b/frontend/src/HudCanvas.tsx
@ -1,11 +1,18 @@
 import { useState, useEffect, useRef, MouseEvent } from 'react';
-import { VideoPosition } from './App';
+
+interface Styles {
+  borderLineWidth: number;
+  borderStrokeStyle: string;
+  positionLineWidth: number;
+  positionStrokeStyle: string;
+}

 interface Props {
  width: number;
  height: number;
  zIndex: number;
-  position: VideoPosition;
+  styles: Styles;
+  position: number | null;
  selection: Selection;
  onSelectionChange: (selection: Selection) => void;
 }
@ -36,6 +43,12 @@ export const HudCanvas: React.FC<Props> = ({
  width,
  height,
  zIndex,
+  styles: {
+    borderLineWidth,
+    borderStrokeStyle,
+    positionLineWidth,
+    positionStrokeStyle,
+  },
  position,
  selection,
  onSelectionChange,
@ -91,26 +104,31 @@ export const HudCanvas: React.FC<Props> = ({
      }

      ctx.beginPath();
-      ctx.strokeStyle = 'red';
-      ctx.lineWidth = 4;
+      ctx.strokeStyle = borderStrokeStyle;
+      ctx.lineWidth = borderLineWidth;
      const alpha = hoverState == HoverState.OverSelection ? '0.15' : '0.13';
      ctx.fillStyle = `rgba(255, 255, 255, ${alpha})`;
      ctx.rect(
        currentSelection.start,
-        2,
+        borderLineWidth,
        currentSelection.end - currentSelection.start,
-        canvas.height - 10
+        canvas.height - borderLineWidth * 2
      );
      ctx.fill();
      ctx.stroke();

      // draw position marker

-      const markerX = canvas.width * (position.percent / 100);
+      if (position == null) {
+        return;
+      }
+
      ctx.beginPath();
-      ctx.moveTo(markerX, 0);
+      ctx.strokeStyle = positionStrokeStyle;
+      ctx.lineWidth = positionLineWidth;
+      ctx.moveTo(position, 0);
      ctx.lineWidth = 4;
-      ctx.lineTo(markerX, canvas.height - 4);
+      ctx.lineTo(position, canvas.height - 4);
      ctx.stroke();
    });
  }, [selection, newSelection, position]);
@ -202,19 +220,19 @@ export const HudCanvas: React.FC<Props> = ({
          break;
        }

-        setNewSelection({ ...newSelection, start: start });
+        setNewSelection({ ...selection, start: start });
        break;
      }
      case Mode.ResizingEnd: {
        const diff = x - moveOffsetX.current;
-        const start = constrainXToCanvas(selection.end + diff);
+        const end = constrainXToCanvas(selection.end + diff);

-        if (start < selection.start) {
-          setNewSelection({ start: Math.max(0, start), end: selection.start });
+        if (end < selection.start) {
+          setNewSelection({ start: Math.max(0, end), end: selection.start });
          break;
        }

-        setNewSelection({ ...newSelection, end: start });
+        setNewSelection({ ...selection, end: end });
        break;
      }
      case Mode.Dragging: {
--- a/frontend/src/Overview.tsx
+++ b/frontend/src/Overview.tsx
@ -33,6 +33,7 @@ export const Overview: React.FC<Props> = ({
  onSelectionChange,
 }: Props) => {
  const [selectedPixels, setSelectedPixels] = useState({ start: 0, end: 0 });
+  const [positionPixels, setPositionPixels] = useState(0);

  // side effects

@ -48,6 +49,14 @@ export const Overview: React.FC<Props> = ({
    });
  }, [viewport, mediaSet]);

+  // convert position from frames to canvas pixels:
+  useEffect(() => {
+    const ratio =
+      position.currentTime / (mediaSet.audioFrames / mediaSet.audioSampleRate);
+    setPositionPixels(Math.round(ratio * CanvasLogicalWidth));
+    frames;
+  }, [mediaSet, position]);
+
  // handlers

  // convert selection change from canvas pixels to frames, and trigger callback.
@ -70,6 +79,13 @@ export const Overview: React.FC<Props> = ({
    height: `${height}px`,
  } as React.CSSProperties;

+  const hudStyles = {
+    borderLineWidth: 4,
+    borderStrokeStyle: 'red',
+    positionLineWidth: 4,
+    positionStrokeStyle: 'red',
+  };
+
  return (
    <>
      <div style={containerStyles}>
@ -87,7 +103,8 @@ export const Overview: React.FC<Props> = ({
          width={CanvasLogicalWidth}
          height={CanvasLogicalHeight}
          zIndex={1}
-          position={position}
+          styles={hudStyles}
+          position={positionPixels}
          selection={selectedPixels}
          onSelectionChange={handleSelectionChange}
        />
--- a/frontend/src/Waveform.tsx
+++ b/frontend/src/Waveform.tsx
@ -1,8 +1,8 @@
-import { useEffect, useState, useRef } from 'react';
+import { useEffect, useState, useCallback } from 'react';
 import { Frames, VideoPosition, newRPC } from './App';
 import { MediaSetServiceClientImpl, MediaSet } from './generated/media_set';
 import { WaveformCanvas } from './WaveformCanvas';
-import { secsToCanvasX } from './Helpers';
+import { Selection, HudCanvas } from './HudCanvas';
 import { from, Observable } from 'rxjs';
 import { bufferCount } from 'rxjs/operators';

@ -11,6 +11,7 @@ interface Props {
  position: VideoPosition;
  viewport: Frames;
  offsetPixels: number;
+  onSelectionChange: (selection: Selection) => void;
 }

 export const CanvasLogicalWidth = 2000;
@ -21,9 +22,15 @@ export const Waveform: React.FC<Props> = ({
  position,
  viewport,
  offsetPixels,
+  onSelectionChange,
 }: Props) => {
  const [peaks, setPeaks] = useState<Observable<number[]>>(from([]));
-  const hudCanvasRef = useRef<HTMLCanvasElement>(null);
+  const [selectedFrames, setSelectedFrames] = useState({ start: 0, end: 0 });
+  const [selectedPixels, setSelectedPixels] = useState({
+    start: 0,
+    end: 0,
+  });
+  const [positionPixels, setPositionPixels] = useState<number | null>(0);

  // effects

@ -57,41 +64,75 @@ export const Waveform: React.FC<Props> = ({
    })();
  }, [viewport]);

-  // render HUD
+  // convert position to canvas pixels
  useEffect(() => {
-    const canvas = hudCanvasRef.current;
-    if (canvas == null) {
+    const frame = Math.round(position.currentTime * mediaSet.audioSampleRate);
+    if (frame < viewport.start || frame > viewport.end) {
+      setPositionPixels(null);
      return;
    }
+    const logicalPixelsPerFrame =
+      CanvasLogicalWidth / (viewport.end - viewport.start);
+    const positionPixels = (frame - viewport.start) * logicalPixelsPerFrame;
+    setPositionPixels(positionPixels);
+  }, [mediaSet, position, viewport]);

-    const ctx = canvas.getContext('2d');
-    if (ctx == null) {
-      console.error('no hud 2d context available');
-      return;
+  // update selectedPixels on viewport change
+  useEffect(() => {
+    const start = frameToCanvasX(selectedFrames.start);
+    const end = frameToCanvasX(selectedFrames.end);
+
+    // more verbose than it has to be to make TypeScript happy
+    if (start == null && end == null) {
+      setSelectedPixels({ start: 0, end: 0 });
+    } else if (start == null && end != null) {
+      setSelectedPixels({ start: 0, end: end });
+    } else if (start != null && end == null) {
+      setSelectedPixels({ start: 0, end: CanvasLogicalWidth });
+    } else if (start != null && end != null) {
+      setSelectedPixels({ start, end });
+    } else {
+      console.error('unreachable');
    }
+  }, [viewport]);

-    ctx.clearRect(0, 0, canvas.width, canvas.height);
+  // handlers

-    if (mediaSet == null) {
-      return;
-    }
+  const handleSelectionChange = useCallback(
+    (selection: Selection) => {
+      setSelectedPixels(selection);

-    const x = secsToCanvasX(
-      position.currentTime,
-      mediaSet.audioSampleRate,
-      viewport
-    );
-    if (x == null) {
-      return;
-    }
+      const framesPerPixel =
+        (viewport.end - viewport.start) / CanvasLogicalWidth;
+      const selectedFrames = {
+        start: Math.round(viewport.start + selection.start * framesPerPixel),
+        end: Math.round(viewport.start + selection.end * framesPerPixel),
+      };

-    ctx.strokeStyle = 'red';
-    ctx.beginPath();
-    ctx.moveTo(x, 0);
-    ctx.lineWidth = 4;
-    ctx.lineTo(x, canvas.height);
-    ctx.stroke();
-  }, [mediaSet, position]);
+      setSelectedFrames(selectedFrames);
+      onSelectionChange(selectedFrames);
+    },
+    [viewport]
+  );
+
+  // helpers
+
+  const frameToCanvasX = useCallback(
+    (frame: number): number | null => {
+      if (mediaSet == null) {
+        return null;
+      }
+
+      if (frame < viewport.start || frame > viewport.end) {
+        return null;
+      }
+
+      const pixelsPerFrame =
+        CanvasLogicalWidth / (viewport.end - viewport.start);
+      return (frame - viewport.start) * pixelsPerFrame;
+    },
+    [mediaSet, viewport]
+  );

  // render component

@ -102,12 +143,12 @@ export const Waveform: React.FC<Props> = ({
    position: 'relative',
  } as React.CSSProperties;

-  const canvasStyles = {
-    position: 'absolute',
-    width: '100%',
-    height: '100%',
-    display: 'block',
-  } as React.CSSProperties;
+  const hudStyles = {
+    borderLineWidth: 0,
+    borderStrokeStyle: 'transparent',
+    positionLineWidth: 6,
+    positionStrokeStyle: 'red',
+  };

  return (
    <>
@ -122,12 +163,15 @@ export const Waveform: React.FC<Props> = ({
          zIndex={0}
          alpha={1}
        ></WaveformCanvas>
-        <canvas
+        <HudCanvas
          width={CanvasLogicalWidth}
          height={CanvasLogicalHeight}
-          ref={hudCanvasRef}
-          style={canvasStyles}
-        ></canvas>
+          zIndex={1}
+          styles={hudStyles}
+          position={positionPixels}
+          selection={selectedPixels}
+          onSelectionChange={handleSelectionChange}
+        />
      </div>
    </>
  );