clipper/frontend/src/App.tsx

import { grpc } from '@improbable-eng/grpc-web';
import { MediaSetService } from './generated/media_set_pb_service';
import {
  MediaSet as MediaSetPb,
  GetMediaSetRequest,
} from './generated/media_set_pb';

import { useState, useEffect } from 'react';
import { VideoPreview } from './VideoPreview';
import { Overview } from './Overview';
import { Waveform } from './Waveform';
import { ControlBar } from './ControlBar';
import { SeekBar } from './SeekBar';
import './App.css';

// Audio corresponds to media.Audio.
export interface Audio {
  bytes: number;
  channels: number;
  frames: number;
  sampleRate: number;
}

// Video corresponds to media.Video.
export interface Video {
  bytes: number;
  thumbnailWidth: number;
  thumbnailHeight: number;
  durationMillis: number;
}

// MediaSet corresponds to media.MediaSet.
export interface MediaSet {
  id: string;
  source: string;
  audio: Audio;
  video: Video;
}

// Frames represents a selection of audio frames.
export interface Frames {
  start: number;
  end: number;
}

function App(): JSX.Element {
  const [mediaSet, setMediaSet] = useState<MediaSet | null>(null);
  const [video, _setVideo] = useState(document.createElement('video'));
  const [position, setPosition] = useState(0);
  const [viewport, setViewport] = useState({ start: 0, end: 0 });

  // effects

  // TODO: error handling
  const videoID = new URLSearchParams(window.location.search).get('video_id');
  if (videoID == null) {
    return <></>;
  }

  // fetch mediaset on page load:
  useEffect(() => {
    (async function () {
      const request = new GetMediaSetRequest();
      request.setId(videoID);
      request.setSource('youtube');

      grpc.invoke(MediaSetService.GetMediaSet, {
        request: request,
        host: 'http://localhost:8888',
        onMessage: (mediaSet: MediaSetPb) => {
          console.log('rcvd media set: ', mediaSet.toObject());
        },
        onEnd: (
          code: grpc.Code,
          msg: string | undefined,
          trailers: grpc.Metadata
        ) => {
          console.log(
            'finished, got code',
            code,
            'msg',
            msg,
            'trailers',
            trailers
          );
        },
      });

      // console.log('fetching media...');
      // const resp = await fetch(
      //   `http://localhost:8888/api/media_sets/${videoID}`
      // );
      // const respBody = await resp.json();
      // if (respBody.error) {
      //   console.log('error fetching media set:', respBody.error);
      //   return;
      // }
      // const mediaSet = {
      //   id: respBody.id,
      //   source: respBody.source,
      //   audio: {
      //     sampleRate: respBody.audio.sample_rate,
      //     bytes: respBody.audio.bytes,
      //     frames: respBody.audio.frames,
      //     channels: respBody.audio.channels,
      //   },
      //   video: {
      //     bytes: respBody.video.bytes,
      //     thumbnailWidth: respBody.video.thumbnail_width,
      //     thumbnailHeight: respBody.video.thumbnail_height,
      //     durationMillis: Math.floor(respBody.video.duration / 1000 / 1000),
      //   },
      // };
      // setMediaSet(mediaSet);
    })();
  }, []);

  // setup player on first page load only:
  useEffect(() => {
    setInterval(() => {
      setPosition(video.currentTime);
    }, 100);
  }, []);

  // load video when MediaSet is loaded:
  useEffect(() => {
    if (mediaSet == null) {
      return;
    }

    video.src = `http://localhost:8888/api/media_sets/${videoID}/video`;
    video.muted = false;
    video.volume = 1;
    console.log('set video src', video.src);
  }, [mediaSet]);

  // set viewport when MediaSet is loaded:
  useEffect(() => {
    if (mediaSet == null) {
      return;
    }

    setViewport({ start: 0, end: mediaSet.audio.frames });
  }, [mediaSet]);

  useEffect(() => {
    console.debug('viewport updated', viewport);
  }, [viewport]);

  // handlers

  const handleOverviewSelectionChange = (selection: Frames) => {
    console.log('in handleOverviewSelectionChange', selection);
    if (mediaSet == null) {
      return;
    }

    if (selection.start >= selection.end) {
      setViewport({ start: 0, end: mediaSet.audio.frames });
      return;
    }

    setViewport({ ...selection });
  };

  // render component

  const containerStyles = {
    border: '1px solid black',
    width: '90%',
    margin: '1em auto',
    minHeight: '500px',
    height: '700px',
    display: 'flex',
    flexDirection: 'column',
  } as React.CSSProperties;

  let offsetPixels = 75;
  if (mediaSet != null) {
    offsetPixels = Math.floor(mediaSet.video.thumbnailWidth / 2);
  }

  if (mediaSet == null) {
    // TODO: improve
    return <></>;
  }

  return (
    <>
      <div className="App">
        <div style={containerStyles}>
          <ControlBar
            onPlay={() => {
              video.play();
            }}
            onPause={() => {
              video.pause();
            }}
          />

          <Overview
            mediaSet={mediaSet}
            offsetPixels={offsetPixels}
            height={80}
            position={position}
            onSelectionStart={(x1: number) => {
              console.log('onSelectionStart', x1);
            }}
            onSelectionChange={handleOverviewSelectionChange}
          />

          <Waveform
            mediaSet={mediaSet}
            position={position}
            viewport={viewport}
            offsetPixels={offsetPixels}
          />

          <SeekBar
            position={video.currentTime}
            duration={mediaSet.audio.frames / mediaSet.audio.sampleRate}
            offsetPixels={offsetPixels}
            onPositionChanged={(position: number) => {
              video.currentTime = position;
            }}
          />

          <VideoPreview
            video={video}
            position={position}
            duration={mediaSet.video.durationMillis}
            height={mediaSet.video.thumbnailHeight}
          />
        </div>
      </div>
    </>
  );
}

export default App;