diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index c8a48f1..88825c7 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -3,16 +3,19 @@ import { GrpcWebImpl, MediaSetServiceClientImpl, GetVideoProgress, + GetAudioProgress, } from './generated/media_set'; import { useState, useEffect } from 'react'; import { VideoPreview } from './VideoPreview'; -import { Overview } from './Overview'; +import { Overview, CanvasLogicalWidth } from './Overview'; import { Waveform } from './Waveform'; import { ControlBar } from './ControlBar'; import { SeekBar } from './SeekBar'; import './App.css'; import { Duration } from './generated/google/protobuf/duration'; +import { from, Observable } from 'rxjs'; +import { map } from 'rxjs/operators'; // ported from backend, where should they live? const thumbnailWidth = 177; @@ -36,8 +39,12 @@ export interface VideoPosition { function App(): JSX.Element { const [mediaSet, setMediaSet] = useState(null); const [video, _setVideo] = useState(document.createElement('video')); + const [audio, _setAudio] = useState(document.createElement('audio')); const [position, setPosition] = useState({ currentTime: 0, percent: 0 }); const [viewport, setViewport] = useState({ start: 0, end: 0 }); + const [overviewPeaks, setOverviewPeaks] = useState>( + from([]) + ); // effects @@ -77,6 +84,37 @@ function App(): JSX.Element { }, 100); }, [mediaSet]); + // load audio when MediaSet is loaded: + useEffect(() => { + (async function () { + if (mediaSet == null) { + return; + } + console.log('fetching audio...'); + // TODO move this call to app.tsx, pass the stream in as a prop. + const service = new MediaSetServiceClientImpl(newRPC()); + const audioProgressStream = service.GetAudio({ + id: mediaSet.id, + numBins: CanvasLogicalWidth, + }); + const peaks = audioProgressStream.pipe(map((progress) => progress.peaks)); + setOverviewPeaks(peaks); + + let url = ''; + // TODO: probably a nicer way to do this. + await audioProgressStream.forEach((progress: GetAudioProgress) => { + if (progress.url != '') { + url = progress.url; + } + }); + + audio.src = url; + audio.muted = false; + audio.volume = 1; + console.log('got audio URL', url); + })(); + }, [mediaSet]); + // load video when MediaSet is loaded: useEffect(() => { (async function () { @@ -98,8 +136,6 @@ function App(): JSX.Element { }); video.src = url; - video.muted = false; - video.volume = 1; console.log('set video src', video.src); })(); }, [mediaSet]); @@ -161,14 +197,17 @@ function App(): JSX.Element {
{ + audio.play(); video.play(); }} onPause={() => { video.pause(); + audio.pause(); }} /> { video.currentTime = position; + audio.currentTime = position; }} /> diff --git a/frontend/src/Overview.tsx b/frontend/src/Overview.tsx index 376c791..e246add 100644 --- a/frontend/src/Overview.tsx +++ b/frontend/src/Overview.tsx @@ -1,13 +1,8 @@ import { useState, useEffect, useRef, MouseEvent } from 'react'; -import { - MediaSetServiceClientImpl, - MediaSet, - GetAudioProgress, -} from './generated/media_set'; -import { Frames, newRPC, VideoPosition } from './App'; +import { MediaSet } from './generated/media_set'; +import { Frames, VideoPosition } from './App'; import { WaveformCanvas } from './WaveformCanvas'; -import { from, Observable } from 'rxjs'; -import { map } from 'rxjs/operators'; +import { Observable } from 'rxjs'; export interface Selection { start: number; @@ -15,6 +10,7 @@ export interface Selection { } interface Props { + peaks: Observable; mediaSet: MediaSet; height: number; offsetPixels: number; @@ -37,12 +33,13 @@ enum HoverState { OverSelection, } -const CanvasLogicalWidth = 2_000; -const CanvasLogicalHeight = 500; +export const CanvasLogicalWidth = 2_000; +export const CanvasLogicalHeight = 500; const emptySelection = { start: 0, end: 0 }; export const Overview: React.FC = ({ + peaks, mediaSet, height, offsetPixels, @@ -50,7 +47,6 @@ export const Overview: React.FC = ({ onSelectionChange, }: Props) => { const hudCanvasRef = useRef(null); - const [peaks, setPeaks] = useState>(from([])); const [mode, setMode] = useState(Mode.Normal); const [hoverState, setHoverState] = useState(HoverState.Normal); const [newSelection, setNewSelection] = useState({ @@ -112,23 +108,6 @@ export const Overview: React.FC = ({ console.error('no hud 2d context available'); return; } - console.log('fetching audio...'); - const service = new MediaSetServiceClientImpl(newRPC()); - const audioProgressStream = service.GetAudio({ - id: mediaSet.id, - numBins: CanvasLogicalWidth, - }); - const peaks = audioProgressStream.pipe(map((progress) => progress.peaks)); - setPeaks(peaks); - - let url = ''; - // TODO: probably a nicer way to do this. - await audioProgressStream.forEach((progress: GetAudioProgress) => { - if (progress.url != '') { - url = progress.url; - } - }); - console.log('got audio URL', url); })(); }, [mediaSet]);