import { MediaSet, GrpcWebImpl, MediaSetServiceClientImpl, GetVideoProgress, GetPeaksProgress, } from './generated/media_set'; import { useState, useEffect, useRef, useCallback } from 'react'; import { AudioFormat } from './generated/media_set'; import { VideoPreview } from './VideoPreview'; import { Overview, CanvasLogicalWidth } from './Overview'; import { Waveform } from './Waveform'; import { SelectionChangeEvent } from './HudCanvas'; import { Selection, SelectionMode } from './HudCanvasState'; import { ControlBar } from './ControlBar'; import { SeekBar } from './SeekBar'; import { firstValueFrom, from, Observable } from 'rxjs'; import { first, map } from 'rxjs/operators'; import millisFromDuration from './helpers/millisFromDuration'; import { canZoomViewportIn, canZoomViewportOut, zoomViewportIn, zoomViewportOut, } from './helpers/zoom'; import toHHMMSS from './helpers/toHHMMSS'; import framesToDuration from './helpers/framesToDuration'; import { ClockIcon, ExternalLinkIcon } from '@heroicons/react/solid'; // ported from backend, where should they live? const thumbnailWidth = 177; // height 100 const initialViewportCanvasPixels = 100; const zoomFactor = 2; const apiURL = process.env.REACT_APP_API_URL || 'http://localhost:8888'; // Frames represents a range of audio frames. export interface Frames { start: number; end: number; } export interface VideoPosition { currentTime: number; percent: number; } export enum PlayState { Paused, Playing, } const video = document.createElement('video'); const audio = document.createElement('audio'); function App(): JSX.Element { const [mediaSet, setMediaSet] = useState(null); const [viewport, setViewport] = useState({ start: 0, end: 0 }); const [selection, setSelection] = useState({ start: 0, end: 0 }); const [overviewPeaks, setOverviewPeaks] = useState>( from([]) ); const [playState, setPlayState] = useState(PlayState.Paused); // position stores the current playback position. positionRef makes it // available inside a setInterval callback. const [position, setPosition] = useState({ currentTime: 0, percent: 0 }); const positionRef = useRef(position); positionRef.current = position; // effects // TODO: error handling const videoID = new URLSearchParams(window.location.search).get('video_id'); if (videoID == null) { return <>; } // fetch mediaset on page load: useEffect(() => { (async function () { const rpc = newRPC(); const service = new MediaSetServiceClientImpl(rpc); const mediaSet = await service.Get({ youtubeId: videoID }); console.log('got media set:', mediaSet); setMediaSet(mediaSet); // fetch audio asynchronously console.log('fetching audio...'); const audioProgressStream = service.GetPeaks({ id: mediaSet.id, numBins: CanvasLogicalWidth, }); const peaks = audioProgressStream.pipe(map((progress) => progress.peaks)); setOverviewPeaks(peaks); const audioPipe = audioProgressStream.pipe( first((progress: GetPeaksProgress) => progress.url != '') ); const fetchAudioTask = firstValueFrom(audioPipe); // fetch video asynchronously console.log('fetching video...'); const videoProgressStream = service.GetVideo({ id: mediaSet.id }); const videoPipe = videoProgressStream.pipe( first((progress: GetVideoProgress) => progress.url != '') ); const fetchVideoTask = firstValueFrom(videoPipe); // wait for both audio, then video. const audioProgress = await fetchAudioTask; audio.src = audioProgress.url; audio.muted = false; audio.volume = 1; console.log('set audio src', audioProgress.url); setMediaSet({ ...mediaSet, audioFrames: audioProgress.audioFrames }); const videoProgress = await fetchVideoTask; video.src = videoProgress.url; console.log('set video src', videoProgress.url); })(); }, []); const updatePlayerPositionIntevalMillis = 20; // setup player on first page load only: useEffect(() => { if (mediaSet == null) { return; } const intervalID = setInterval(() => { const currTime = audio.currentTime; if (currTime == positionRef.current.currentTime) { return; } const duration = mediaSet.audioFrames / mediaSet.audioSampleRate; const percent = (currTime / duration) * 100; // check if the end of selection has been passed, and pause if so: if ( selection.start != selection.end && currentTimeToFrame(positionRef.current.currentTime) < selection.end && currentTimeToFrame(currTime) >= selection.end ) { pause(); } // update the current position setPosition({ currentTime: audio.currentTime, percent: percent }); }, updatePlayerPositionIntevalMillis); return () => clearInterval(intervalID); }, [mediaSet, selection]); // bind to keypress handler. // selection is a dependency of the handleKeyPress handler, and must be // included here. useEffect(() => { document.addEventListener('keypress', handleKeyPress); return () => document.removeEventListener('keypress', handleKeyPress); }, [selection, playState]); // set viewport when MediaSet is loaded: useEffect(() => { if (mediaSet == null) { return; } const numFrames = Math.min( Math.round(mediaSet.audioFrames / CanvasLogicalWidth) * initialViewportCanvasPixels, mediaSet.audioFrames ); setViewport({ start: 0, end: numFrames }); }, [mediaSet]); useEffect(() => { console.debug('viewport updated', viewport); }, [viewport]); // handlers const handleKeyPress = (evt: KeyboardEvent) => { if (evt.code != 'Space') { return; } togglePlay(); }; // handler called when the selection in the overview (zoom setting) is changed. const handleOverviewSelectionChange = ({ selection: newViewport, }: SelectionChangeEvent) => { if (mediaSet == null) { return; } console.log('set new viewport', newViewport); setViewport({ ...newViewport }); if (!audio.paused) { return; } setPositionFromFrame(newViewport.start); }; const setPositionAfterSelectionChange = ( newSelection: Selection, mode: SelectionMode, prevMode: SelectionMode ): boolean => { // if creating a new selection from scratch, reset position on mouseup. if (prevMode == SelectionMode.Selecting && mode == SelectionMode.Normal) { return true; } // if re-entering normal mode, reset position if the current position is // outside the new selection on mouseup. if (prevMode != SelectionMode.Normal && mode == SelectionMode.Normal) { const currFrame = currentTimeToFrame(positionRef.current.currentTime); if (currFrame < newSelection.start || currFrame > newSelection.end) { return true; } } return false; }; // handler called when the selection in the main waveform view is changed. const handleWaveformSelectionChange = ({ selection: newSelection, mode, prevMode, }: SelectionChangeEvent) => { setSelection(newSelection); if (setPositionAfterSelectionChange(newSelection, mode, prevMode)) { setPositionFromFrame(newSelection.start); } }; const togglePlay = () => { if (playState == PlayState.Paused) { play(); } else { pause(); } }; const play = () => { audio.play(); video.play(); setPlayState(PlayState.Playing); }; const pause = () => { video.pause(); audio.pause(); setPositionFromFrame(selection.start); setPlayState(PlayState.Paused); }; const handleClip = () => { if (!window.showSaveFilePicker) { downloadClipHTTP(); return; } downloadClipFileSystemAccessAPI(); }; const downloadClipHTTP = () => { (async function () { if (mediaSet == null) { return; } console.debug('clip http', selection); const form = document.createElement('form'); form.method = 'POST'; form.action = `${apiURL}/api/media_sets/${mediaSet.id}/clip`; const startFrameInput = document.createElement('input'); startFrameInput.type = 'hidden'; startFrameInput.name = 'start_frame'; startFrameInput.value = String(selection.start); form.appendChild(startFrameInput); const endFrameInput = document.createElement('input'); endFrameInput.type = 'hidden'; endFrameInput.name = 'end_frame'; endFrameInput.value = String(selection.end); form.appendChild(endFrameInput); const formatInput = document.createElement('input'); formatInput.type = 'hidden'; formatInput.name = 'format'; formatInput.value = 'mp3'; form.appendChild(formatInput); document.body.appendChild(form); form.submit(); })(); }; const downloadClipFileSystemAccessAPI = () => { (async function () { if (mediaSet == null) { return; } console.debug('clip grpc', selection); const h = await window.showSaveFilePicker({ suggestedName: 'clip.mp3' }); const fileStream = await h.createWritable(); const rpc = newRPC(); const service = new MediaSetServiceClientImpl(rpc); const stream = service.GetAudioSegment({ id: mediaSet.id, format: AudioFormat.MP3, startFrame: selection.start, endFrame: selection.end, }); await stream.forEach((p) => fileStream.write(p.audioData)); console.debug('finished writing stream'); await fileStream.close(); console.debug('closed stream'); })(); }; const handleZoomIn = () => { if (mediaSet == null) { return; } const newViewport = zoomViewportIn( viewport, mediaSet.audioFrames, selection, currentTimeToFrame(positionRef.current.currentTime), zoomFactor ); setViewport(newViewport); }; const handleZoomOut = () => { if (mediaSet == null) { return; } const newViewport = zoomViewportOut( viewport, mediaSet.audioFrames, selection, currentTimeToFrame(positionRef.current.currentTime), zoomFactor ); setViewport(newViewport); }; const setPositionFromFrame = useCallback( (frame: number) => { if (mediaSet == null) { return; } const ratio = frame / mediaSet.audioFrames; const currentTime = (mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio; audio.currentTime = currentTime; video.currentTime = currentTime; }, [mediaSet] ); // helpers const currentTimeToFrame = useCallback( (currentTime: number): number => { if (mediaSet == null) { return 0; } const dur = mediaSet.audioFrames / mediaSet.audioSampleRate; const ratio = currentTime / dur; return Math.round(mediaSet.audioFrames * ratio); }, [mediaSet] ); const durationString = useCallback((): string => { if (!mediaSet || !mediaSet.videoDuration) { return ''; } const totalDur = toHHMMSS(mediaSet.videoDuration); if (selection.start == selection.end) { return totalDur; } const clipDur = toHHMMSS( framesToDuration( selection.end - selection.start, mediaSet.audioSampleRate ) ); return `Selected ${clipDur} of ${totalDur}`; }, [mediaSet, selection]); // render component const offsetPixels = Math.floor(thumbnailWidth / 2); const marginClass = 'mx-[88px]'; // offsetPixels if (mediaSet == null) { // TODO: improve return <>; } return ( <>

Clipper

{mediaSet.author} / {mediaSet.title} {durationString()}
{ video.currentTime = position; audio.currentTime = position; }} />
  • Frames: {mediaSet.audioFrames}
  • Viewport (frames): {viewport.start} to {viewport.end}
  • Selection (frames): {selection.start} to {selection.end}
  • Position (frames):{' '} {Math.round(mediaSet.audioFrames * (position.percent / 100))}
  • Position (seconds): {position.currentTime}
); } export default App; export function newRPC(): GrpcWebImpl { return new GrpcWebImpl(apiURL, {}); }