clipper/frontend/src/App.tsx

import {
  MediaSet,
  GrpcWebImpl,
  MediaSetServiceClientImpl,
  GetVideoProgress,
  GetPeaksProgress,
} from './generated/media_set';

import { useState, useEffect, useRef, useCallback } from 'react';
import { AudioFormat } from './generated/media_set';
import { VideoPreview } from './VideoPreview';
import { Overview, CanvasLogicalWidth } from './Overview';
import { Waveform } from './Waveform';
import { ControlBar } from './ControlBar';
import { SeekBar } from './SeekBar';
import './App.css';
import { firstValueFrom, from, Observable } from 'rxjs';
import { first, map } from 'rxjs/operators';
import millisFromDuration from './helpers/millisFromDuration';

// ported from backend, where should they live?
const thumbnailWidth = 177;
const thumbnailHeight = 100;

const initialViewportCanvasPixels = 100;

const apiURL = process.env.REACT_APP_API_URL || 'http://localhost:8888';

// Frames represents a range of audio frames.
export interface Frames {
  start: number;
  end: number;
}

export interface VideoPosition {
  currentTime: number;
  percent: number;
}

const video = document.createElement('video');
const audio = document.createElement('audio');

function App(): JSX.Element {
  const [mediaSet, setMediaSet] = useState<MediaSet | null>(null);
  const [viewport, setViewport] = useState<Frames>({ start: 0, end: 0 });
  const [selection, setSelection] = useState<Frames>({ start: 0, end: 0 });
  const [overviewPeaks, setOverviewPeaks] = useState<Observable<number[]>>(
    from([])
  );

  // position stores the current playback position. positionRef makes it
  // available inside a setInterval callback.
  const [position, setPosition] = useState({ currentTime: 0, percent: 0 });
  const positionRef = useRef(position);
  positionRef.current = position;

  // effects

  // TODO: error handling
  const videoID = new URLSearchParams(window.location.search).get('video_id');
  if (videoID == null) {
    return <></>;
  }

  // fetch mediaset on page load:
  useEffect(() => {
    (async function () {
      const rpc = newRPC();
      const service = new MediaSetServiceClientImpl(rpc);
      const mediaSet = await service.Get({ youtubeId: videoID });

      console.log('got media set:', mediaSet);
      setMediaSet(mediaSet);
    })();
  }, []);

  const updatePlayerPositionIntevalMillis = 30;

  // setup player on first page load only:
  useEffect(() => {
    if (mediaSet == null) {
      return;
    }

    const intervalID = setInterval(() => {
      const currTime = audio.currentTime;
      if (currTime == positionRef.current.currentTime) {
        return;
      }
      const duration = mediaSet.audioFrames / mediaSet.audioSampleRate;
      const percent = (currTime / duration) * 100;

      // check if the end of selection has been passed, and pause if so:
      if (
        currentTimeToFrame(position.currentTime) < selection.end &&
        currentTimeToFrame(currTime) >= selection.end
      ) {
        handlePause();
      }

      // update the current position
      setPosition({ currentTime: audio.currentTime, percent: percent });
    }, updatePlayerPositionIntevalMillis);

    return () => clearInterval(intervalID);
  }, [mediaSet, selection]);

  // bind to keypress handler.
  // selection is a dependency of the handleKeyPress handler, and must be
  // included here.
  useEffect(() => {
    document.addEventListener('keypress', handleKeyPress);
    return () => document.removeEventListener('keypress', handleKeyPress);
  }, [selection]);

  // load audio when MediaSet is loaded:
  useEffect(() => {
    (async function () {
      if (mediaSet == null) {
        return;
      }
      console.log('fetching audio...');
      const service = new MediaSetServiceClientImpl(newRPC());
      const audioProgressStream = service.GetPeaks({
        id: mediaSet.id,
        numBins: CanvasLogicalWidth,
      });
      const peaks = audioProgressStream.pipe(map((progress) => progress.peaks));
      setOverviewPeaks(peaks);

      const pipe = audioProgressStream.pipe(
        first((progress: GetPeaksProgress) => progress.url != '')
      );
      const progressWithURL = await firstValueFrom(pipe);

      audio.src = progressWithURL.url;
      audio.muted = false;
      audio.volume = 1;
      console.log('set audio src', progressWithURL.url);
    })();
  }, [mediaSet]);

  // load video when MediaSet is loaded:
  useEffect(() => {
    (async function () {
      if (mediaSet == null) {
        return;
      }

      console.log('fetching video...');
      const service = new MediaSetServiceClientImpl(newRPC());
      const videoProgressStream = service.GetVideo({ id: mediaSet.id });
      const pipe = videoProgressStream.pipe(
        first((progress: GetVideoProgress) => progress.url != '')
      );
      const progressWithURL = await firstValueFrom(pipe);

      video.src = progressWithURL.url;
      console.log('set video src', progressWithURL.url);
    })();
  }, [mediaSet]);

  // set viewport when MediaSet is loaded:
  useEffect(() => {
    if (mediaSet == null) {
      return;
    }

    const numFrames = Math.min(
      Math.round(mediaSet.audioFrames / CanvasLogicalWidth) *
        initialViewportCanvasPixels,
      mediaSet.audioFrames
    );

    setViewport({ start: 0, end: numFrames });
  }, [mediaSet]);

  useEffect(() => {
    console.debug('viewport updated', viewport);
  }, [viewport]);

  // handlers

  const handleKeyPress = useCallback(
    (evt: KeyboardEvent) => {
      if (evt.code != 'Space') {
        return;
      }

      if (audio.paused) {
        handlePlay();
      } else {
        handlePause();
      }
    },
    [selection]
  );

  // handler called when the selection in the overview (zoom setting) is changed.
  const handleOverviewSelectionChange = useCallback(
    (newViewport: Frames) => {
      if (mediaSet == null) {
        return;
      }
      console.log('set new viewport', newViewport);
      setViewport({ ...newViewport });

      if (!audio.paused) {
        return;
      }

      setPositionFromFrame(newViewport.start);
    },
    [mediaSet, audio, video, selection]
  );

  // handler called when the selection in the main waveform view is changed.
  const handleWaveformSelectionChange = useCallback(
    (newSelection: Frames) => {
      setSelection(newSelection);

      if (mediaSet == null) {
        return;
      }

      // move playback position to start of selection
      const ratio = newSelection.start / mediaSet.audioFrames;
      const currentTime =
        (mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio;
      audio.currentTime = currentTime;
      video.currentTime = currentTime;
    },
    [mediaSet, audio, video, selection]
  );

  const handlePlay = useCallback(() => {
    audio.play();
    video.play();
  }, [audio, video]);

  const handlePause = useCallback(() => {
    video.pause();
    audio.pause();

    if (selection.start != selection.end) {
      setPositionFromFrame(selection.start);
    }
  }, [audio, video, selection]);

  const handleClip = useCallback(() => {
    (async function () {
      console.debug('clip', selection);

      if (mediaSet == null) {
        return;
      }

      // TODO: support File System Access API fallback
      const h = await window.showSaveFilePicker({ suggestedName: 'clip.mp3' });
      const fileStream = await h.createWritable();

      const rpc = newRPC();
      const service = new MediaSetServiceClientImpl(rpc);
      const stream = service.GetAudioSegment({
        id: mediaSet.id,
        format: AudioFormat.MP3,
        startFrame: selection.start,
        endFrame: selection.end,
      });

      await stream.forEach((p) => fileStream.write(p.audioData));
      console.debug('finished writing stream');

      await fileStream.close();
      console.debug('closed stream');
    })();
  }, [mediaSet, selection]);

  const handleZoomIn = useCallback(() => {
    if (mediaSet == null) {
      return;
    }
    if (viewport.start == viewport.end) {
      return;
    }
    setViewport({
      ...viewport,
      end: viewport.end - Math.round((viewport.end - viewport.start) / 2),
    });
  }, [mediaSet, viewport]);

  const handleZoomOut = useCallback(() => {
    if (mediaSet == null) {
      return;
    }
    if (viewport.start == viewport.end) {
      return;
    }
    let end = viewport.end + Math.round(viewport.end - viewport.start);
    if (end > mediaSet.audioFrames) {
      end = mediaSet.audioFrames;
    }
    setViewport({
      ...viewport,
      end: end,
    });
  }, [mediaSet, viewport]);

  const setPositionFromFrame = useCallback(
    (frame: number) => {
      if (mediaSet == null) {
        return;
      }
      const ratio = frame / mediaSet.audioFrames;
      const currentTime =
        (mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio;
      audio.currentTime = currentTime;
      video.currentTime = currentTime;
    },
    [mediaSet, audio, video]
  );

  // helpers

  const currentTimeToFrame = useCallback(
    (currentTime: number): number => {
      if (mediaSet == null) {
        return 0;
      }
      const dur = mediaSet.audioFrames / mediaSet.audioSampleRate;
      const ratio = currentTime / dur;
      return Math.round(mediaSet.audioFrames * ratio);
    },
    [mediaSet]
  );

  // render component

  const containerStyles = {
    border: '1px solid black',
    width: '90%',
    margin: '1em auto',
    minHeight: '500px',
    height: '700px',
    display: 'flex',
    flexDirection: 'column',
  } as React.CSSProperties;

  const offsetPixels = Math.floor(thumbnailWidth / 2);

  if (mediaSet == null) {
    // TODO: improve
    return <></>;
  }

  return (
    <>
      <div className="App">
        <div style={containerStyles}>
          <ControlBar
            onPlay={handlePlay}
            onPause={handlePause}
            onClip={handleClip}
            onZoomIn={handleZoomIn}
            onZoomOut={handleZoomOut}
          />

          <Overview
            peaks={overviewPeaks}
            mediaSet={mediaSet}
            offsetPixels={offsetPixels}
            height={80}
            viewport={viewport}
            position={position}
            onSelectionChange={handleOverviewSelectionChange}
          />

          <Waveform
            mediaSet={mediaSet}
            position={position}
            viewport={viewport}
            offsetPixels={offsetPixels}
            onSelectionChange={handleWaveformSelectionChange}
          />

          <SeekBar
            position={video.currentTime}
            duration={mediaSet.audioFrames / mediaSet.audioSampleRate}
            offsetPixels={offsetPixels}
            onPositionChanged={(position: number) => {
              video.currentTime = position;
              audio.currentTime = position;
            }}
          />

          <VideoPreview
            mediaSet={mediaSet}
            video={video}
            position={position}
            duration={millisFromDuration(mediaSet.videoDuration)}
            height={thumbnailHeight}
          />
        </div>
        <ul style={{ listStyleType: 'none' } as React.CSSProperties}>
          <li>Frames: {mediaSet.audioFrames}</li>
          <li>
            Viewport (frames): {viewport.start} to {viewport.end}
          </li>
          <li>
            Selection (frames): {selection.start} to {selection.end}
          </li>
          <li>
            Position (frames):{' '}
            {Math.round(mediaSet.audioFrames * (position.percent / 100))}
          </li>
          <li>Position (seconds): {position.currentTime}</li>
          <li></li>
        </ul>
      </div>
    </>
  );
}

export default App;

export function newRPC(): GrpcWebImpl {
  return new GrpcWebImpl(apiURL, {});
}