clipper/frontend/src/App.tsx

370 lines
10 KiB
TypeScript

import {
MediaSet,
GrpcWebImpl,
MediaSetServiceClientImpl,
GetVideoProgress,
GetPeaksProgress,
} from './generated/media_set';
import { useState, useEffect, useRef, useCallback } from 'react';
import { VideoPreview } from './VideoPreview';
import { Overview, CanvasLogicalWidth } from './Overview';
import { Waveform } from './Waveform';
import { ControlBar } from './ControlBar';
import { SeekBar } from './SeekBar';
import './App.css';
import { Duration } from './generated/google/protobuf/duration';
import { firstValueFrom, from, Observable } from 'rxjs';
import { first, map } from 'rxjs/operators';
// ported from backend, where should they live?
const thumbnailWidth = 177;
const thumbnailHeight = 100;
const initialViewportCanvasPixels = 100;
const apiURL = process.env.REACT_APP_API_URL || 'http://localhost:8888';
// Frames represents a range of audio frames.
export interface Frames {
start: number;
end: number;
}
export interface VideoPosition {
currentTime: number;
percent: number;
}
const video = document.createElement('video');
const audio = document.createElement('audio');
function App(): JSX.Element {
const [mediaSet, setMediaSet] = useState<MediaSet | null>(null);
const [viewport, setViewport] = useState<Frames>({ start: 0, end: 0 });
const [selection, setSelection] = useState<Frames>({ start: 0, end: 0 });
const [overviewPeaks, setOverviewPeaks] = useState<Observable<number[]>>(
from([])
);
// position stores the current playback position. positionRef makes it
// available inside a setInterval callback.
const [position, setPosition] = useState({ currentTime: 0, percent: 0 });
const positionRef = useRef(position);
positionRef.current = position;
// effects
// TODO: error handling
const videoID = new URLSearchParams(window.location.search).get('video_id');
if (videoID == null) {
return <></>;
}
// fetch mediaset on page load:
useEffect(() => {
(async function () {
const rpc = newRPC();
const service = new MediaSetServiceClientImpl(rpc);
const mediaSet = await service.Get({ youtubeId: videoID });
console.log('got media set:', mediaSet);
setMediaSet(mediaSet);
})();
}, []);
const updatePlayerPositionIntevalMillis = 30;
// setup player on first page load only:
useEffect(() => {
if (mediaSet == null) {
return;
}
const intervalID = setInterval(() => {
const currTime = audio.currentTime;
if (currTime == positionRef.current.currentTime) {
return;
}
const duration = mediaSet.audioFrames / mediaSet.audioSampleRate;
const percent = (currTime / duration) * 100;
// check if the end of selection has been passed, and pause if so:
if (
currentTimeToFrame(position.currentTime) < selection.end &&
currentTimeToFrame(currTime) >= selection.end
) {
handlePause();
}
// update the current position
setPosition({ currentTime: audio.currentTime, percent: percent });
}, updatePlayerPositionIntevalMillis);
return () => clearInterval(intervalID);
}, [mediaSet, selection]);
// bind to keypress handler.
// selection is a dependency of the handleKeyPress handler, and must be
// included here.
useEffect(() => {
document.addEventListener('keypress', handleKeyPress);
return () => document.removeEventListener('keypress', handleKeyPress);
}, [selection]);
// load audio when MediaSet is loaded:
useEffect(() => {
(async function () {
if (mediaSet == null) {
return;
}
console.log('fetching audio...');
const service = new MediaSetServiceClientImpl(newRPC());
const audioProgressStream = service.GetPeaks({
id: mediaSet.id,
numBins: CanvasLogicalWidth,
});
const peaks = audioProgressStream.pipe(map((progress) => progress.peaks));
setOverviewPeaks(peaks);
const pipe = audioProgressStream.pipe(
first((progress: GetPeaksProgress) => progress.url != '')
);
const progressWithURL = await firstValueFrom(pipe);
audio.src = progressWithURL.url;
audio.muted = false;
audio.volume = 1;
console.log('set audio src', progressWithURL.url);
})();
}, [mediaSet]);
// load video when MediaSet is loaded:
useEffect(() => {
(async function () {
if (mediaSet == null) {
return;
}
console.log('fetching video...');
const service = new MediaSetServiceClientImpl(newRPC());
const videoProgressStream = service.GetVideo({ id: mediaSet.id });
const pipe = videoProgressStream.pipe(
first((progress: GetVideoProgress) => progress.url != '')
);
const progressWithURL = await firstValueFrom(pipe);
video.src = progressWithURL.url;
console.log('set video src', progressWithURL.url);
})();
}, [mediaSet]);
// set viewport when MediaSet is loaded:
useEffect(() => {
if (mediaSet == null) {
return;
}
const numFrames = Math.min(
Math.round(mediaSet.audioFrames / CanvasLogicalWidth) *
initialViewportCanvasPixels,
mediaSet.audioFrames
);
setViewport({ start: 0, end: numFrames });
}, [mediaSet]);
useEffect(() => {
console.debug('viewport updated', viewport);
}, [viewport]);
// handlers
const handleKeyPress = useCallback(
(evt: KeyboardEvent) => {
if (evt.code != 'Space') {
return;
}
if (audio.paused) {
handlePlay();
} else {
handlePause();
}
},
[selection]
);
// handler called when the selection in the overview (zoom setting) is changed.
const handleOverviewSelectionChange = useCallback(
(newViewport: Frames) => {
if (mediaSet == null) {
return;
}
console.log('set new viewport', newViewport);
setViewport({ ...newViewport });
if (!audio.paused) {
return;
}
setPositionFromFrame(newViewport.start);
},
[mediaSet, audio, video, selection]
);
// handler called when the selection in the main waveform view is changed.
const handleWaveformSelectionChange = useCallback(
(newSelection: Frames) => {
setSelection(newSelection);
if (mediaSet == null) {
return;
}
// move playback position to start of selection
const ratio = newSelection.start / mediaSet.audioFrames;
const currentTime =
(mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio;
audio.currentTime = currentTime;
video.currentTime = currentTime;
},
[mediaSet, audio, video, selection]
);
const handlePlay = useCallback(() => {
audio.play();
video.play();
}, [audio, video]);
const handlePause = useCallback(() => {
video.pause();
audio.pause();
if (selection.start != selection.end) {
setPositionFromFrame(selection.start);
}
}, [audio, video, selection]);
const setPositionFromFrame = useCallback(
(frame: number) => {
if (mediaSet == null) {
return;
}
const ratio = frame / mediaSet.audioFrames;
const currentTime =
(mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio;
audio.currentTime = currentTime;
video.currentTime = currentTime;
},
[mediaSet, audio, video]
);
// helpers
const currentTimeToFrame = useCallback(
(currentTime: number): number => {
if (mediaSet == null) {
return 0;
}
const dur = mediaSet.audioFrames / mediaSet.audioSampleRate;
const ratio = currentTime / dur;
return Math.round(mediaSet.audioFrames * ratio);
},
[mediaSet]
);
// render component
const containerStyles = {
border: '1px solid black',
width: '90%',
margin: '1em auto',
minHeight: '500px',
height: '700px',
display: 'flex',
flexDirection: 'column',
} as React.CSSProperties;
const offsetPixels = Math.floor(thumbnailWidth / 2);
if (mediaSet == null) {
// TODO: improve
return <></>;
}
return (
<>
<div className="App">
<div style={containerStyles}>
<ControlBar onPlay={handlePlay} onPause={handlePause} />
<Overview
peaks={overviewPeaks}
mediaSet={mediaSet}
offsetPixels={offsetPixels}
height={80}
viewport={viewport}
position={position}
onSelectionChange={handleOverviewSelectionChange}
/>
<Waveform
mediaSet={mediaSet}
position={position}
viewport={viewport}
offsetPixels={offsetPixels}
onSelectionChange={handleWaveformSelectionChange}
/>
<SeekBar
position={video.currentTime}
duration={mediaSet.audioFrames / mediaSet.audioSampleRate}
offsetPixels={offsetPixels}
onPositionChanged={(position: number) => {
video.currentTime = position;
audio.currentTime = position;
}}
/>
<VideoPreview
mediaSet={mediaSet}
video={video}
position={position}
duration={millisFromDuration(mediaSet.videoDuration)}
height={thumbnailHeight}
/>
</div>
<ul style={{ listStyleType: 'none' } as React.CSSProperties}>
<li>Frames: {mediaSet.audioFrames}</li>
<li>
Viewport (frames): {viewport.start} to {viewport.end}
</li>
<li>
Selection (frames): {selection.start} to {selection.end}
</li>
<li>
Position (frames):{' '}
{Math.round(mediaSet.audioFrames * (position.percent / 100))}
</li>
<li>Position (seconds): {position.currentTime}</li>
<li></li>
</ul>
</div>
</>
);
}
export default App;
function millisFromDuration(dur?: Duration): number {
if (dur == undefined) {
return 0;
}
return Math.floor(dur.seconds * 1000.0 + dur.nanos / 1000.0 / 1000.0);
}
export function newRPC(): GrpcWebImpl {
return new GrpcWebImpl(apiURL, {});
}