clipper/frontend/src/Waveform.tsx

import { useEffect, useState, useRef, MouseEvent } from 'react';
import { Waveform as WaveformOverview } from './Waveform/Overview';
import { Canvas as WaveformCanvas } from './Waveform/Canvas';
import {
  secsToCanvasX,
  canvasXToFrame,
  mouseEventToCanvasX,
} from './Waveform/Helpers';

type Props = {
  audioContext: AudioContext;
};

export type AudioFile = {
  bytes: number;
  channels: number;
  frames: number;
  sampleRate: number;
};

export type Selection = {
  x1: number;
  x2: number;
};

type ZoomSettings = {
  startFrame: number;
  endFrame: number;
};

const defaultZoomSettings: ZoomSettings = { startFrame: 0, endFrame: 0 };

export const CanvasLogicalWidth = 2000;
export const CanvasLogicalHeight = 500;

export const Waveform: React.FC<Props> = ({ audioContext }: Props) => {
  const [audioFile, setAudioFile] = useState<AudioFile | null>(null);
  const [currentTime, setCurrentTime] = useState(0);
  // TODO: fix linter error
  const [audio, setAudio] = useState(new Audio());
  const [zoomSettings, setZoomSettings] = useState(defaultZoomSettings);
  const [waveformPeaks, setWaveformPeaks] = useState(null);
  const [overviewPeaks, setOverviewPeaks] = useState(null);
  const hudCanvasRef = useRef<HTMLCanvasElement>(null);

  // TODO: error handling
  const videoID = new URLSearchParams(window.location.search).get('video_id');

  // effects

  // setup player on page load:
  useEffect(() => {
    (async function () {
      audio.addEventListener('timeupdate', () => {
        setCurrentTime(audio.currentTime);
      });
    })();
  }, [audio]);

  // fetch audio data on page load:
  useEffect(() => {
    (async function () {
      console.log('fetching audio data...');

      const resp = await fetch(
        `http://localhost:8888/api/download?video_id=${videoID}`
      );
      const respBody = await resp.json();

      if (respBody.error) {
        console.log('error fetching audio data:', respBody.error);
        return;
      }

      // TODO: safer deserialization?
      const audioFile: AudioFile = {
        bytes: respBody.bytes,
        channels: respBody.channels,
        frames: respBody.frames,
        sampleRate: respBody.sample_rate,
      };

      setAudioFile(audioFile);
      setZoomSettings({ startFrame: 0, endFrame: audioFile.frames });
    })();
  }, [audioContext]);

  // render overview waveform to canvas when the audio file is loaded:

  // fetch new waveform peaks when zoom settings are updated:
  useEffect(() => {
    (async function () {
      if (audioFile == null) {
        return;
      }

      let endFrame = zoomSettings.endFrame;
      if (endFrame <= zoomSettings.startFrame) {
        endFrame = audioFile.frames;
      }

      const resp = await fetch(
        `http://localhost:8888/api/peaks?video_id=${videoID}&start=${zoomSettings.startFrame}&end=${endFrame}&bins=${CanvasLogicalWidth}`
      );
      const peaks = await resp.json();
      console.log('respBody from peaks =', peaks);

      setWaveformPeaks(peaks);

      if (overviewPeaks == null) {
        setOverviewPeaks(peaks);
      }
    })();
  }, [zoomSettings]);

  // redraw HUD
  useEffect(() => {
    (async function () {
      const canvas = hudCanvasRef.current;
      if (canvas == null) {
        console.error('no hud canvas ref available');
        return;
      }

      const ctx = canvas.getContext('2d');
      if (ctx == null) {
        console.error('no hud 2d context available');
        return;
      }

      ctx.clearRect(0, 0, canvas.width, canvas.height);

      if (audioFile == null) {
        return;
      }

      const x = secsToCanvasX(
        currentTime,
        audioFile.sampleRate,
        audioFile.frames
      );

      ctx.strokeStyle = 'red';
      ctx.beginPath();
      ctx.moveTo(x, 0);
      ctx.lineTo(x, canvas.height);
      ctx.stroke();
    })();
  }, [currentTime]);

  // callbacks

  const handleMouseMove = (evt: MouseEvent<HTMLCanvasElement>) => {
    if (audioFile == null) {
      return;
    }
    const canvasX = mouseEventToCanvasX(evt);
    console.log(
      'mousemove, x =',
      canvasX,
      'frame =',
      canvasXToFrame(canvasX, numFrames)
    );
  };

  const handleMouseDown = () => {
    return null;
  };

  const handleMouseUp = () => {
    return null;
  };

  const handlePlay = async () => {
    const url = `http://localhost:8888/api/audio?video_id=${videoID}`;
    audio.src = url;
    await audio.play();
    console.log('playing audio from', url);
  };

  const handlePause = () => {
    audio.pause();
    console.log('paused audio');
  };

  const handleZoomIn = () => {
    if (audioFile == null) {
      return;
    }
    console.log('zoom in');
    const diff = zoomSettings.endFrame - zoomSettings.startFrame;
    const endFrame = zoomSettings.startFrame + Math.floor(diff / 2);
    const settings = { ...zoomSettings, endFrame: endFrame };
    setZoomSettings(settings);
  };

  const handleZoomOut = () => {
    if (audioFile == null) {
      return;
    }
    console.log('zoom out');
    const diff = zoomSettings.endFrame - zoomSettings.startFrame;
    const newDiff = diff * 2;
    const endFrame = Math.min(
      zoomSettings.endFrame + newDiff,
      audioFile.frames
    );
    const settings = { ...zoomSettings, endFrame: endFrame };
    setZoomSettings(settings);
  };

  const handleSelectionChange = (selection: Selection) => {
    if (audioFile == null) {
      return;
    }
    const settings: ZoomSettings = {
      startFrame: canvasXToFrame(selection.x1, audioFile.frames),
      endFrame: canvasXToFrame(selection.x2, audioFile.frames),
    };
    setZoomSettings(settings);
  };

  // render component:

  const wrapperProps = {
    width: '90%',
    height: '350px',
    position: 'relative',
    margin: '0 auto',
  } as React.CSSProperties;

  const waveformCanvasProps = {
    width: '100%',
    height: '100%',
    position: 'absolute',
    top: 0,
    left: 0,
    right: 0,
    bottom: 0,
    zIndex: 0,
  } as React.CSSProperties;

  const hudCanvasProps = {
    width: '100%',
    height: '100%',
    position: 'absolute',
    top: 0,
    left: 0,
    right: 0,
    bottom: 0,
    zIndex: 1,
  } as React.CSSProperties;

  const overviewStyles = { ...wrapperProps, height: '90px' };

  // TODO: why is the margin needed?
  const controlPanelStyles = { margin: '1em' } as React.CSSProperties;
  const clockTextAreaProps = { color: '#999', width: '400px' };

  let numFrames = 0;
  if (audioFile != null) {
    numFrames = audioFile.frames;
  }

  return (
    <>
      <h1>clipper</h1>
      <div style={wrapperProps}>
        <WaveformCanvas
          peaks={waveformPeaks}
          fillStyle="black"
          strokeStyle="green"
          style={waveformCanvasProps}
        ></WaveformCanvas>
        <canvas
          ref={hudCanvasRef}
          onMouseMove={handleMouseMove}
          onMouseDown={handleMouseDown}
          onMouseUp={handleMouseUp}
          style={hudCanvasProps}
          width={CanvasLogicalWidth}
          height={CanvasLogicalHeight}
        ></canvas>
      </div>
      <WaveformOverview
        peaks={overviewPeaks}
        numFrames={numFrames}
        style={overviewStyles}
        onSelectionChange={handleSelectionChange}
      ></WaveformOverview>
      <div style={controlPanelStyles}>
        <button onClick={handlePlay}>Play</button>
        <button onClick={handlePause}>Pause</button>
        <button onClick={handleZoomIn}>+</button>
        <button onClick={handleZoomOut}>-</button>
        <input type="readonly" style={clockTextAreaProps} />
      </div>
    </>
  );
};