clipper/frontend/src/Waveform.tsx

import { useEffect, useState, useRef, MouseEvent } from 'react';

type WaveformProps = {
  audioContext: AudioContext;
};

type AudioFile = {
  bytes: number;
  channels: number;
  frames: number;
  sampleRate: number;
};

type ZoomSettings = {
  startFrame: number;
  endFrame: number;
};

const defaultZoomSettings: ZoomSettings = { startFrame: 0, endFrame: 0 };

export const Waveform: React.FC<WaveformProps> = ({
  audioContext,
}: WaveformProps) => {
  const [audioFile, setAudioFile] = useState<AudioFile | null>(null);
  const [currentTime, setCurrentTime] = useState(0);
  const [audio, setAudio] = useState(new Audio());
  const [zoomSettings, setZoomSettings] = useState(defaultZoomSettings);

  const waveformCanvasRef = useRef<HTMLCanvasElement>(null);
  const hudCanvasRef = useRef<HTMLCanvasElement>(null);
  const canvasLogicalWidth = 2000;
  const canvasLogicalHeight = 500;
  const videoID = new URLSearchParams(window.location.search).get('video_id');

  // helpers

  const mouseEventToCanvasX = (evt: MouseEvent<HTMLCanvasElement>): number => {
    // TODO: use offsetX/offsetY?
    const rect = evt.currentTarget.getBoundingClientRect();
    const elementX = evt.clientX - rect.left;
    const canvas = evt.target as HTMLCanvasElement;
    return (elementX * canvas.width) / rect.width;
  };

  const canvasXToFrame = (x: number): number => {
    if (audioFile == null) {
      return 0;
    }
    return Math.floor((x / canvasLogicalWidth) * audioFile.frames);
  };

  // const canvasXToSecs = (x: number): number => {
  //   if (audioFile == null) {
  //     return 0;
  //   }
  //   const duration = audioFile.frames / audioFile.sampleRate;
  //   return (canvasXToFrame(x) / audioFile.frames) * duration;
  // };

  const secsToCanvasX = (canvasWidth: number, secs: number): number => {
    if (audioFile == null) {
      return 0;
    }
    const duration = audioFile.frames / audioFile.sampleRate;
    return Math.floor(canvasWidth * (secs / duration));
  };

  // effects

  // setup player on page load:
  useEffect(() => {
    (async function () {
      audio.addEventListener('timeupdate', () => {
        setCurrentTime(audio.currentTime);
      });
    })();
  }, [audio]);

  // load audio data on page load:
  useEffect(() => {
    (async function () {
      console.log('fetching audio data...');

      const resp = await fetch(
        `http://localhost:8888/api/download?video_id=${videoID}`
      );
      const respBody = await resp.json();

      if (respBody.error) {
        console.log('error fetching audio data:', respBody.error);
        return;
      }

      // TODO: safer deserialization?
      const audioFile: AudioFile = {
        bytes: respBody.bytes,
        channels: respBody.channels,
        frames: respBody.frames,
        sampleRate: respBody.sample_rate,
      };

      setAudioFile(audioFile);
      setZoomSettings({ startFrame: 0, endFrame: audioFile.frames });
    })();
  }, [audioContext]);

  // render waveform to canvas when zoom settings are updated:
  useEffect(() => {
    (async function () {
      if (audioFile == null) {
        return;
      }

      console.log('audiofile is', audioFile);

      const canvas = waveformCanvasRef.current;
      if (canvas == null) {
        console.error('no canvas ref available');
        return;
      }

      const ctx = canvas.getContext('2d');
      if (ctx == null) {
        console.error('no 2d context available');
        return;
      }

      let endFrame = zoomSettings.endFrame;
      if (endFrame <= zoomSettings.startFrame) {
        endFrame = audioFile.frames;
      }

      const resp = await fetch(
        `http://localhost:8888/api/peaks?video_id=${videoID}&start=${zoomSettings.startFrame}&end=${endFrame}&bins=${canvas.width}`
      );
      const peaks = await resp.json();
      console.log('respBody from peaks =', peaks);

      ctx.strokeStyle = '#00aa00';
      ctx.fillStyle = 'black';
      ctx.fillRect(0, 0, canvas.width, canvas.height);

      const numChannels = peaks.length;
      const chanHeight = canvas.height / numChannels;
      for (let c = 0; c < numChannels; c++) {
        const yOffset = chanHeight * c;
        for (let i = 0; i < peaks[c].length; i++) {
          const val = peaks[c][i];
          const height = Math.floor((val / 32768) * chanHeight);
          const y1 = (chanHeight - height) / 2 + yOffset;
          const y2 = y1 + height;
          ctx.beginPath();
          ctx.moveTo(i, y1);
          ctx.lineTo(i, y2);
          ctx.stroke();
        }
      }
    })();
  }, [zoomSettings]);

  // redraw HUD
  useEffect(() => {
    (async function () {
      const canvas = hudCanvasRef.current;
      if (canvas == null) {
        console.error('no hud canvas ref available');
        return;
      }

      const ctx = canvas.getContext('2d');
      if (ctx == null) {
        console.error('no hud 2d context available');
        return;
      }

      ctx.clearRect(0, 0, canvas.width, canvas.height);

      if (audioFile == null) {
        return;
      }

      const x = secsToCanvasX(canvas.width, currentTime);

      ctx.strokeStyle = 'red';
      ctx.beginPath();
      ctx.moveTo(x, 0);
      ctx.lineTo(x, canvas.height);
      ctx.stroke();
    })();
  }, [currentTime]);

  // callbacks

  const handleMouseMove = (evt: MouseEvent<HTMLCanvasElement>) => {
    const canvasX = mouseEventToCanvasX(evt);
    console.log('mousemove, x =', canvasX, 'frame =', canvasXToFrame(canvasX));
  };

  const handleMouseDown = () => {
    return null;
  };

  const handleMouseUp = () => {
    return null;
  };

  const handlePlay = async () => {
    const url = `http://localhost:8888/api/audio?video_id=${videoID}`;
    audio.src = url;
    await audio.play();
    console.log('playing audio from', url);
  };

  const handlePause = () => {
    audio.pause();
    console.log('paused audio');
  };

  const handleZoomIn = () => {
    if (audioFile == null) {
      return;
    }
    console.log('zoom in');
    const diff = zoomSettings.endFrame - zoomSettings.startFrame;
    const endFrame = zoomSettings.startFrame + Math.floor(diff / 2);
    const settings = { ...zoomSettings, endFrame: endFrame };
    setZoomSettings(settings);
  };

  const handleZoomOut = () => {
    if (audioFile == null) {
      return;
    }
    console.log('zoom out');
    const diff = zoomSettings.endFrame - zoomSettings.startFrame;
    const newDiff = diff * 2;
    const endFrame = Math.min(
      zoomSettings.endFrame + newDiff,
      audioFile.frames
    );
    const settings = { ...zoomSettings, endFrame: endFrame };
    setZoomSettings(settings);
  };

  // render component:

  const wrapperProps = {
    width: '90%',
    height: '500px',
    position: 'relative',
    margin: '0 auto',
  } as React.CSSProperties;

  const waveformCanvasProps = {
    width: '100%',
    position: 'absolute',
    top: 0,
    left: 0,
    right: 0,
    bottom: 0,
    zIndex: 0,
  } as React.CSSProperties;

  const hudCanvasProps = {
    width: '100%',
    position: 'absolute',
    top: 0,
    left: 0,
    right: 0,
    bottom: 0,
    zIndex: 1,
  } as React.CSSProperties;

  const clockTextAreaProps = { color: '#999', width: '400px' };

  return (
    <>
      <h1>clipper</h1>
      <div style={wrapperProps}>
        <canvas
          ref={waveformCanvasRef}
          width={canvasLogicalWidth}
          height={canvasLogicalHeight}
          style={waveformCanvasProps}
        ></canvas>
        <canvas
          ref={hudCanvasRef}
          width={canvasLogicalWidth}
          height={canvasLogicalHeight}
          onMouseMove={handleMouseMove}
          onMouseDown={handleMouseDown}
          onMouseUp={handleMouseUp}
          style={hudCanvasProps}
        ></canvas>
      </div>
      <button onClick={handlePlay}>Play</button>
      <button onClick={handlePause}>Pause</button>
      <button onClick={handleZoomIn}>+</button>
      <button onClick={handleZoomOut}>-</button>
      <input type="readonly" style={clockTextAreaProps} />
    </>
  );
};