clipper/frontend/src/App.tsx

import {
  MediaSet,
  GrpcWebImpl,
  MediaSetServiceClientImpl,
  GetVideoProgress,
  GetPeaksProgress,
} from './generated/media_set';

import { useState, useEffect, useRef, useCallback } from 'react';
import { AudioFormat } from './generated/media_set';
import { VideoPreview } from './VideoPreview';
import { Overview, CanvasLogicalWidth } from './Overview';
import { Waveform } from './Waveform';
import { SelectionChangeEvent } from './HudCanvas';
import { Selection, SelectionMode } from './HudCanvasState';
import { ControlBar } from './ControlBar';
import { SeekBar } from './SeekBar';
import { firstValueFrom, from, Observable } from 'rxjs';
import { first, map } from 'rxjs/operators';
import millisFromDuration from './helpers/millisFromDuration';
import {
  canZoomViewportIn,
  canZoomViewportOut,
  zoomViewportIn,
  zoomViewportOut,
} from './helpers/zoom';
import toHHMMSS from './helpers/toHHMMSS';
import framesToDuration from './helpers/framesToDuration';
import { ClockIcon, ExternalLinkIcon } from '@heroicons/react/solid';

// ported from backend, where should they live?
const thumbnailWidth = 177; // height 100

const initialViewportCanvasPixels = 100;

const zoomFactor = 2;

const apiURL = process.env.REACT_APP_API_URL || 'http://localhost:8888';

// Frames represents a range of audio frames.
export interface Frames {
  start: number;
  end: number;
}

export interface VideoPosition {
  currentTime: number;
  percent: number;
}

export enum PlayState {
  Paused,
  Playing,
}

const video = document.createElement('video');
const audio = document.createElement('audio');

function App(): JSX.Element {
  const [mediaSet, setMediaSet] = useState<MediaSet | null>(null);
  const [viewport, setViewport] = useState<Frames>({ start: 0, end: 0 });
  const [selection, setSelection] = useState<Frames>({ start: 0, end: 0 });
  const [overviewPeaks, setOverviewPeaks] = useState<Observable<number[]>>(
    from([])
  );
  const [playState, setPlayState] = useState(PlayState.Paused);

  // position stores the current playback position. positionRef makes it
  // available inside a setInterval callback.
  const [position, setPosition] = useState({ currentTime: 0, percent: 0 });
  const positionRef = useRef(position);
  positionRef.current = position;

  // effects

  // TODO: error handling
  const videoID = new URLSearchParams(window.location.search).get('video_id');
  if (videoID == null) {
    return <></>;
  }

  // fetch mediaset on page load:
  useEffect(() => {
    (async function () {
      const rpc = newRPC();
      const service = new MediaSetServiceClientImpl(rpc);
      const mediaSet = await service.Get({ youtubeId: videoID });

      console.log('got media set:', mediaSet);
      setMediaSet(mediaSet);

      // fetch audio asynchronously
      console.log('fetching audio...');
      const audioProgressStream = service.GetPeaks({
        id: mediaSet.id,
        numBins: CanvasLogicalWidth,
      });
      const peaks = audioProgressStream.pipe(map((progress) => progress.peaks));
      setOverviewPeaks(peaks);

      const audioPipe = audioProgressStream.pipe(
        first((progress: GetPeaksProgress) => progress.url != '')
      );
      const fetchAudioTask = firstValueFrom(audioPipe);

      // fetch video asynchronously
      console.log('fetching video...');
      const videoProgressStream = service.GetVideo({ id: mediaSet.id });
      const videoPipe = videoProgressStream.pipe(
        first((progress: GetVideoProgress) => progress.url != '')
      );
      const fetchVideoTask = firstValueFrom(videoPipe);

      // wait for both audio, then video.
      const audioProgress = await fetchAudioTask;

      audio.src = audioProgress.url;
      audio.muted = false;
      audio.volume = 1;
      console.log('set audio src', audioProgress.url);
      setMediaSet({ ...mediaSet, audioFrames: audioProgress.audioFrames });

      const videoProgress = await fetchVideoTask;
      video.src = videoProgress.url;
      console.log('set video src', videoProgress.url);
    })();
  }, []);

  const updatePlayerPositionIntevalMillis = 20;

  // setup player on first page load only:
  useEffect(() => {
    if (mediaSet == null) {
      return;
    }

    const intervalID = setInterval(() => {
      const currTime = audio.currentTime;
      if (currTime == positionRef.current.currentTime) {
        return;
      }
      const duration = mediaSet.audioFrames / mediaSet.audioSampleRate;
      const percent = (currTime / duration) * 100;

      // check if the end of selection has been passed, and pause if so:
      if (
        selection.start != selection.end &&
        currentTimeToFrame(positionRef.current.currentTime) < selection.end &&
        currentTimeToFrame(currTime) >= selection.end
      ) {
        pause();
      }

      // update the current position
      setPosition({ currentTime: audio.currentTime, percent: percent });
    }, updatePlayerPositionIntevalMillis);

    return () => clearInterval(intervalID);
  }, [mediaSet, selection]);

  // bind to keypress handler.
  // selection is a dependency of the handleKeyPress handler, and must be
  // included here.
  useEffect(() => {
    document.addEventListener('keypress', handleKeyPress);
    return () => document.removeEventListener('keypress', handleKeyPress);
  }, [selection, playState]);

  // set viewport when MediaSet is loaded:
  useEffect(() => {
    if (mediaSet == null) {
      return;
    }

    const numFrames = Math.min(
      Math.round(mediaSet.audioFrames / CanvasLogicalWidth) *
        initialViewportCanvasPixels,
      mediaSet.audioFrames
    );

    setViewport({ start: 0, end: numFrames });
  }, [mediaSet]);

  useEffect(() => {
    console.debug('viewport updated', viewport);
  }, [viewport]);

  // handlers

  const handleKeyPress = (evt: KeyboardEvent) => {
    if (evt.code != 'Space') {
      return;
    }

    togglePlay();
  };

  // handler called when the selection in the overview (zoom setting) is changed.
  const handleOverviewSelectionChange = ({
    selection: newViewport,
  }: SelectionChangeEvent) => {
    if (mediaSet == null) {
      return;
    }
    console.log('set new viewport', newViewport);
    setViewport({ ...newViewport });

    if (!audio.paused) {
      return;
    }

    setPositionFromFrame(newViewport.start);
  };

  const setPositionAfterSelectionChange = (
    newSelection: Selection,
    mode: SelectionMode,
    prevMode: SelectionMode
  ): boolean => {
    // if creating a new selection from scratch, reset position on mouseup.
    if (prevMode == SelectionMode.Selecting && mode == SelectionMode.Normal) {
      return true;
    }

    // if re-entering normal mode, reset position if the current position is
    // outside the new selection on mouseup.
    if (prevMode != SelectionMode.Normal && mode == SelectionMode.Normal) {
      const currFrame = currentTimeToFrame(positionRef.current.currentTime);
      if (currFrame < newSelection.start || currFrame > newSelection.end) {
        return true;
      }
    }

    return false;
  };

  // handler called when the selection in the main waveform view is changed.
  const handleWaveformSelectionChange = ({
    selection: newSelection,
    mode,
    prevMode,
  }: SelectionChangeEvent) => {
    setSelection(newSelection);

    if (setPositionAfterSelectionChange(newSelection, mode, prevMode)) {
      setPositionFromFrame(newSelection.start);
    }
  };

  const togglePlay = () => {
    if (playState == PlayState.Paused) {
      play();
    } else {
      pause();
    }
  };

  const play = () => {
    audio.play();
    video.play();

    setPlayState(PlayState.Playing);
  };

  const pause = () => {
    video.pause();
    audio.pause();

    setPositionFromFrame(selection.start);

    setPlayState(PlayState.Paused);
  };

  const handleClip = () => {
    if (!window.showSaveFilePicker) {
      downloadClipHTTP();
      return;
    }

    downloadClipFileSystemAccessAPI();
  };

  const downloadClipHTTP = () => {
    (async function () {
      if (mediaSet == null) {
        return;
      }

      console.debug('clip http', selection);

      const form = document.createElement('form');
      form.method = 'POST';
      form.action = `${apiURL}/api/media_sets/${mediaSet.id}/clip`;
      const startFrameInput = document.createElement('input');
      startFrameInput.type = 'hidden';
      startFrameInput.name = 'start_frame';
      startFrameInput.value = String(selection.start);
      form.appendChild(startFrameInput);
      const endFrameInput = document.createElement('input');
      endFrameInput.type = 'hidden';
      endFrameInput.name = 'end_frame';
      endFrameInput.value = String(selection.end);
      form.appendChild(endFrameInput);
      const formatInput = document.createElement('input');
      formatInput.type = 'hidden';
      formatInput.name = 'format';
      formatInput.value = 'mp3';
      form.appendChild(formatInput);
      document.body.appendChild(form);
      form.submit();
    })();
  };

  const downloadClipFileSystemAccessAPI = () => {
    (async function () {
      if (mediaSet == null) {
        return;
      }

      console.debug('clip grpc', selection);
      const h = await window.showSaveFilePicker({ suggestedName: 'clip.mp3' });
      const fileStream = await h.createWritable();

      const rpc = newRPC();
      const service = new MediaSetServiceClientImpl(rpc);
      const stream = service.GetAudioSegment({
        id: mediaSet.id,
        format: AudioFormat.MP3,
        startFrame: selection.start,
        endFrame: selection.end,
      });

      await stream.forEach((p) => fileStream.write(p.audioData));
      console.debug('finished writing stream');

      await fileStream.close();
      console.debug('closed stream');
    })();
  };

  const handleZoomIn = () => {
    if (mediaSet == null) {
      return;
    }

    const newViewport = zoomViewportIn(
      viewport,
      mediaSet.audioFrames,
      selection,
      currentTimeToFrame(positionRef.current.currentTime),
      zoomFactor
    );

    setViewport(newViewport);
  };

  const handleZoomOut = () => {
    if (mediaSet == null) {
      return;
    }

    const newViewport = zoomViewportOut(
      viewport,
      mediaSet.audioFrames,
      selection,
      currentTimeToFrame(positionRef.current.currentTime),
      zoomFactor
    );

    setViewport(newViewport);
  };

  const setPositionFromFrame = useCallback(
    (frame: number) => {
      if (mediaSet == null) {
        return;
      }
      const ratio = frame / mediaSet.audioFrames;
      const currentTime =
        (mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio;
      audio.currentTime = currentTime;
      video.currentTime = currentTime;
    },
    [mediaSet]
  );

  // helpers

  const currentTimeToFrame = useCallback(
    (currentTime: number): number => {
      if (mediaSet == null) {
        return 0;
      }
      const dur = mediaSet.audioFrames / mediaSet.audioSampleRate;
      const ratio = currentTime / dur;
      return Math.round(mediaSet.audioFrames * ratio);
    },
    [mediaSet]
  );

  const durationString = useCallback((): string => {
    if (!mediaSet || !mediaSet.videoDuration) {
      return '';
    }

    const totalDur = toHHMMSS(mediaSet.videoDuration);
    if (selection.start == selection.end) {
      return totalDur;
    }

    const clipDur = toHHMMSS(
      framesToDuration(
        selection.end - selection.start,
        mediaSet.audioSampleRate
      )
    );

    return `Selected ${clipDur} of ${totalDur}`;
  }, [mediaSet, selection]);

  // render component

  const offsetPixels = Math.floor(thumbnailWidth / 2);
  const marginClass = 'mx-[88px]'; // offsetPixels

  if (mediaSet == null) {
    // TODO: improve
    return <></>;
  }

  return (
    <>
      <div className="App bg-gray-800 h-screen flex flex-col">
        <header className="bg-green-900 h-16 grow-0 flex items-center mb-12 px-[88px]">
          <h1 className="text-3xl font-bold">Clipper</h1>
        </header>
        <div className="flex flex-col grow bg-gray-800 w-full h-full mx-auto">
          <div className={`flex flex-col grow ${marginClass}`}>
            <div className="flex grow-0 h-8 pt-4 pb-2 items-center space-x-2 text-white">
              <span className="text-gray-300">{mediaSet.author}</span>
              <span>/</span>
              <span>{mediaSet.title}</span>
              <a
                href={`https://www.youtube.com/watch?v=${mediaSet.youtubeId}`}
                target="_blank"
                rel="noreferrer"
                title="Open in YouTube"
              >
                <ExternalLinkIcon className="h-6 w-6 text-gray-500 hover:text-gray-200" />
              </a>
              <span className="flex grow justify-end text-gray-500">
                <ClockIcon className="h-5 w-5 mr-1 mt-0.5" />
                {durationString()}
              </span>
            </div>
            <ControlBar
              playState={playState}
              zoomInEnabled={canZoomViewportIn(viewport, selection, zoomFactor)}
              zoomOutEnabled={canZoomViewportOut(
                viewport,
                mediaSet.audioFrames
              )}
              onTogglePlay={togglePlay}
              onClip={handleClip}
              onZoomIn={handleZoomIn}
              onZoomOut={handleZoomOut}
              downloadClipEnabled={selection.start != selection.end}
            />

            <div className="w-full bg-gray-600 h-6"></div>
            <Overview
              peaks={overviewPeaks}
              mediaSet={mediaSet}
              viewport={viewport}
              position={position}
              onSelectionChange={handleOverviewSelectionChange}
            />

            <Waveform
              mediaSet={mediaSet}
              position={position}
              viewport={viewport}
              onSelectionChange={handleWaveformSelectionChange}
            />
          </div>

          <SeekBar
            position={video.currentTime}
            duration={mediaSet.audioFrames / mediaSet.audioSampleRate}
            offsetPixels={offsetPixels}
            onPositionChanged={(position: number) => {
              video.currentTime = position;
              audio.currentTime = position;
            }}
          />

          <VideoPreview
            mediaSet={mediaSet}
            video={video}
            position={position}
            duration={millisFromDuration(mediaSet.videoDuration)}
          />
        </div>
        <ul className="hidden">
          <li>Frames: {mediaSet.audioFrames}</li>
          <li>
            Viewport (frames): {viewport.start} to {viewport.end}
          </li>
          <li>
            Selection (frames): {selection.start} to {selection.end}
          </li>
          <li>
            Position (frames):{' '}
            {Math.round(mediaSet.audioFrames * (position.percent / 100))}
          </li>
          <li>Position (seconds): {position.currentTime}</li>
          <li></li>
        </ul>
      </div>
    </>
  );
}

export default App;

export function newRPC(): GrpcWebImpl {
  return new GrpcWebImpl(apiURL, {});
}