Re-enable audio playback

frontend/src/App.tsx

@@ -3,16 +3,19 @@ import {
   GrpcWebImpl,
   MediaSetServiceClientImpl,
   GetVideoProgress,
+  GetAudioProgress,
 } from './generated/media_set';
 
 import { useState, useEffect } from 'react';
 import { VideoPreview } from './VideoPreview';
-import { Overview } from './Overview';
+import { Overview, CanvasLogicalWidth } from './Overview';
 import { Waveform } from './Waveform';
 import { ControlBar } from './ControlBar';
 import { SeekBar } from './SeekBar';
 import './App.css';
 import { Duration } from './generated/google/protobuf/duration';
+import { from, Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
 
 // ported from backend, where should they live?
 const thumbnailWidth = 177;
@@ -36,8 +39,12 @@ export interface VideoPosition {
 function App(): JSX.Element {
   const [mediaSet, setMediaSet] = useState<MediaSet | null>(null);
   const [video, _setVideo] = useState(document.createElement('video'));
+  const [audio, _setAudio] = useState(document.createElement('audio'));
   const [position, setPosition] = useState({ currentTime: 0, percent: 0 });
   const [viewport, setViewport] = useState({ start: 0, end: 0 });
+  const [overviewPeaks, setOverviewPeaks] = useState<Observable<number[]>>(
+    from([])
+  );
 
   // effects
 
@@ -77,6 +84,37 @@ function App(): JSX.Element {
     }, 100);
   }, [mediaSet]);
 
+  // load audio when MediaSet is loaded:
+  useEffect(() => {
+    (async function () {
+      if (mediaSet == null) {
+        return;
+      }
+      console.log('fetching audio...');
+      // TODO move this call to app.tsx, pass the stream in as a prop.
+      const service = new MediaSetServiceClientImpl(newRPC());
+      const audioProgressStream = service.GetAudio({
+        id: mediaSet.id,
+        numBins: CanvasLogicalWidth,
+      });
+      const peaks = audioProgressStream.pipe(map((progress) => progress.peaks));
+      setOverviewPeaks(peaks);
+
+      let url = '';
+      // TODO: probably a nicer way to do this.
+      await audioProgressStream.forEach((progress: GetAudioProgress) => {
+        if (progress.url != '') {
+          url = progress.url;
+        }
+      });
+
+      audio.src = url;
+      audio.muted = false;
+      audio.volume = 1;
+      console.log('got audio URL', url);
+    })();
+  }, [mediaSet]);
+
   // load video when MediaSet is loaded:
   useEffect(() => {
     (async function () {
@@ -98,8 +136,6 @@ function App(): JSX.Element {
       });
 
       video.src = url;
-      video.muted = false;
-      video.volume = 1;
       console.log('set video src', video.src);
     })();
   }, [mediaSet]);
@@ -161,14 +197,17 @@ function App(): JSX.Element {
         <div style={containerStyles}>
           <ControlBar
             onPlay={() => {
+              audio.play();
               video.play();
             }}
             onPause={() => {
               video.pause();
+              audio.pause();
             }}
           />
 
           <Overview
+            peaks={overviewPeaks}
             mediaSet={mediaSet}
             offsetPixels={offsetPixels}
             height={80}
@@ -189,6 +228,7 @@ function App(): JSX.Element {
             offsetPixels={offsetPixels}
             onPositionChanged={(position: number) => {
               video.currentTime = position;
+              audio.currentTime = position;
             }}
           />
 

frontend/src/Overview.tsx

@@ -1,13 +1,8 @@
 import { useState, useEffect, useRef, MouseEvent } from 'react';
-import {
+import { MediaSet } from './generated/media_set';
-  MediaSetServiceClientImpl,
+import { Frames, VideoPosition } from './App';
-  MediaSet,
-  GetAudioProgress,
-} from './generated/media_set';
-import { Frames, newRPC, VideoPosition } from './App';
 import { WaveformCanvas } from './WaveformCanvas';
-import { from, Observable } from 'rxjs';
+import { Observable } from 'rxjs';
-import { map } from 'rxjs/operators';
 
 export interface Selection {
   start: number;
@@ -15,6 +10,7 @@ export interface Selection {
 }
 
 interface Props {
+  peaks: Observable<number[]>;
   mediaSet: MediaSet;
   height: number;
   offsetPixels: number;
@@ -37,12 +33,13 @@ enum HoverState {
   OverSelection,
 }
 
-const CanvasLogicalWidth = 2_000;
+export const CanvasLogicalWidth = 2_000;
-const CanvasLogicalHeight = 500;
+export const CanvasLogicalHeight = 500;
 
 const emptySelection = { start: 0, end: 0 };
 
 export const Overview: React.FC<Props> = ({
+  peaks,
   mediaSet,
   height,
   offsetPixels,
@@ -50,7 +47,6 @@ export const Overview: React.FC<Props> = ({
   onSelectionChange,
 }: Props) => {
   const hudCanvasRef = useRef<HTMLCanvasElement>(null);
-  const [peaks, setPeaks] = useState<Observable<number[]>>(from([]));
   const [mode, setMode] = useState(Mode.Normal);
   const [hoverState, setHoverState] = useState(HoverState.Normal);
   const [newSelection, setNewSelection] = useState({
@@ -112,23 +108,6 @@ export const Overview: React.FC<Props> = ({
         console.error('no hud 2d context available');
         return;
       }
-      console.log('fetching audio...');
-      const service = new MediaSetServiceClientImpl(newRPC());
-      const audioProgressStream = service.GetAudio({
-        id: mediaSet.id,
-        numBins: CanvasLogicalWidth,
-      });
-      const peaks = audioProgressStream.pipe(map((progress) => progress.peaks));
-      setPeaks(peaks);
-
-      let url = '';
-      // TODO: probably a nicer way to do this.
-      await audioProgressStream.forEach((progress: GetAudioProgress) => {
-        if (progress.url != '') {
-          url = progress.url;
-        }
-      });
-      console.log('got audio URL', url);
     })();
   }, [mediaSet]);