From b876fb915a399503dd1735ae48d7125faf97c60a Mon Sep 17 00:00:00 2001 From: Rob Watson Date: Sat, 11 Dec 2021 17:25:43 +0100 Subject: [PATCH] Update frontend - Add HudCanvas component to Waveform - Allow waveform to be selectable - Fix selection rendering on viewport change - Add spacebar handler --- frontend/src/App.tsx | 130 ++++++++++++++++++++++++++++++------- frontend/src/Helpers.tsx | 36 ---------- frontend/src/HudCanvas.tsx | 46 +++++++++---- frontend/src/Overview.tsx | 19 +++++- frontend/src/Waveform.tsx | 122 +++++++++++++++++++++++----------- 5 files changed, 241 insertions(+), 112 deletions(-) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 340f3c2..eadf263 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -25,7 +25,7 @@ const initialViewportCanvasPixels = 100; const apiURL = process.env.REACT_APP_API_URL || 'http://localhost:8888'; -// Frames represents a selection of audio frames. +// Frames represents a range of audio frames. export interface Frames { start: number; end: number; @@ -41,7 +41,8 @@ const audio = document.createElement('audio'); function App(): JSX.Element { const [mediaSet, setMediaSet] = useState(null); - const [viewport, setViewport] = useState({ start: 0, end: 0 }); + const [viewport, setViewport] = useState({ start: 0, end: 0 }); + const [selection, setSelection] = useState({ start: 0, end: 0 }); const [overviewPeaks, setOverviewPeaks] = useState>( from([]) ); @@ -81,17 +82,35 @@ function App(): JSX.Element { } const intervalID = setInterval(() => { - if (video.currentTime == positionRef.current.currentTime) { + const currTime = audio.currentTime; + if (currTime == positionRef.current.currentTime) { return; } const duration = mediaSet.audioFrames / mediaSet.audioSampleRate; - const percent = (video.currentTime / duration) * 100; + const percent = (currTime / duration) * 100; - setPosition({ currentTime: video.currentTime, percent: percent }); + // check if the end of selection has been passed, and pause if so: + if ( + currentTimeToFrame(position.currentTime) < selection.end && + currentTimeToFrame(currTime) >= selection.end + ) { + handlePause(); + } + + // update the current position + setPosition({ currentTime: audio.currentTime, percent: percent }); }, updatePlayerPositionIntevalMillis); return () => clearInterval(intervalID); - }, [mediaSet]); + }, [mediaSet, selection]); + + // bind to keypress handler. + // selection is a dependency of the handleKeyPress handler, and must be + // included here. + useEffect(() => { + document.addEventListener('keypress', handleKeyPress); + return () => document.removeEventListener('keypress', handleKeyPress); + }, [selection]); // load audio when MediaSet is loaded: useEffect(() => { @@ -161,23 +180,57 @@ function App(): JSX.Element { // handlers - const handleOverviewSelectionChange = (newViewport: Frames) => { - if (mediaSet == null) { - return; - } - console.log('set new viewport', newViewport); - setViewport({ ...newViewport }); + const handleKeyPress = useCallback( + (evt: KeyboardEvent) => { + if (evt.code != 'Space') { + return; + } - if (!audio.paused) { - return; - } + if (audio.paused) { + handlePlay(); + } else { + handlePause(); + } + }, + [selection] + ); - const ratio = newViewport.start / mediaSet.audioFrames; - const currentTime = - (mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio; - audio.currentTime = currentTime; - video.currentTime = currentTime; - }; + // handler called when the selection in the overview (zoom setting) is changed. + const handleOverviewSelectionChange = useCallback( + (newViewport: Frames) => { + if (mediaSet == null) { + return; + } + console.log('set new viewport', newViewport); + setViewport({ ...newViewport }); + + if (!audio.paused) { + return; + } + + setPositionFromFrame(newViewport.start); + }, + [mediaSet, audio, video] + ); + + // handler called when the selection in the main waveform view is changed. + const handleWaveformSelectionChange = useCallback( + (selection: Frames) => { + setSelection(selection); + + if (mediaSet == null) { + return; + } + + // move playback position to start of selection + const ratio = selection.start / mediaSet.audioFrames; + const currentTime = + (mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio; + audio.currentTime = currentTime; + video.currentTime = currentTime; + }, + [mediaSet, audio, video] + ); const handlePlay = useCallback(() => { audio.play(); @@ -187,7 +240,39 @@ function App(): JSX.Element { const handlePause = useCallback(() => { video.pause(); audio.pause(); - }, [audio, video]); + + if (selection.start != selection.end) { + setPositionFromFrame(selection.start); + } + }, [audio, video, selection]); + + const setPositionFromFrame = useCallback( + (frame: number) => { + if (mediaSet == null) { + return; + } + const ratio = frame / mediaSet.audioFrames; + const currentTime = + (mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio; + audio.currentTime = currentTime; + video.currentTime = currentTime; + }, + [mediaSet, audio, video] + ); + + // helpers + + const currentTimeToFrame = useCallback( + (currentTime: number): number => { + if (mediaSet == null) { + return 0; + } + const dur = mediaSet.audioFrames / mediaSet.audioSampleRate; + const ratio = currentTime / dur; + return Math.round(mediaSet.audioFrames * ratio); + }, + [mediaSet] + ); // render component @@ -229,6 +314,7 @@ function App(): JSX.Element { position={position} viewport={viewport} offsetPixels={offsetPixels} + onSelectionChange={handleWaveformSelectionChange} /> -): number => { - const rect = evt.currentTarget.getBoundingClientRect(); - const elementX = evt.clientX - rect.left; - return (elementX * CanvasLogicalWidth) / rect.width; -}; - // TODO: add tests export const mouseEventToCanvasPoint = ( evt: MouseEvent @@ -31,26 +18,3 @@ export const mouseEventToCanvasPoint = ( y: (elementY * evt.currentTarget.height) / rect.height, }; }; - -// TODO: add tests -export const canvasXToFrame = (x: number, numFrames: number): number => { - return Math.floor((x / CanvasLogicalWidth) * numFrames); -}; - -// TODO: add tests -// secsToCanvasX returns the logical x coordinate for a given position -// marker. It is null if the marker is outside of the current viewport. -export const secsToCanvasX = ( - secs: number, - sampleRate: number, - viewport: Frames -): number | null => { - const frame = Math.round(secs * sampleRate); - if (frame < viewport.start || frame > viewport.end) { - return null; - } - - const logicalPixelsPerFrame = - CanvasLogicalWidth / (viewport.end - viewport.start); - return (frame - viewport.start) * logicalPixelsPerFrame; -}; diff --git a/frontend/src/HudCanvas.tsx b/frontend/src/HudCanvas.tsx index d9a1229..e23a1f9 100644 --- a/frontend/src/HudCanvas.tsx +++ b/frontend/src/HudCanvas.tsx @@ -1,11 +1,18 @@ import { useState, useEffect, useRef, MouseEvent } from 'react'; -import { VideoPosition } from './App'; + +interface Styles { + borderLineWidth: number; + borderStrokeStyle: string; + positionLineWidth: number; + positionStrokeStyle: string; +} interface Props { width: number; height: number; zIndex: number; - position: VideoPosition; + styles: Styles; + position: number | null; selection: Selection; onSelectionChange: (selection: Selection) => void; } @@ -36,6 +43,12 @@ export const HudCanvas: React.FC = ({ width, height, zIndex, + styles: { + borderLineWidth, + borderStrokeStyle, + positionLineWidth, + positionStrokeStyle, + }, position, selection, onSelectionChange, @@ -91,26 +104,31 @@ export const HudCanvas: React.FC = ({ } ctx.beginPath(); - ctx.strokeStyle = 'red'; - ctx.lineWidth = 4; + ctx.strokeStyle = borderStrokeStyle; + ctx.lineWidth = borderLineWidth; const alpha = hoverState == HoverState.OverSelection ? '0.15' : '0.13'; ctx.fillStyle = `rgba(255, 255, 255, ${alpha})`; ctx.rect( currentSelection.start, - 2, + borderLineWidth, currentSelection.end - currentSelection.start, - canvas.height - 10 + canvas.height - borderLineWidth * 2 ); ctx.fill(); ctx.stroke(); // draw position marker - const markerX = canvas.width * (position.percent / 100); + if (position == null) { + return; + } + ctx.beginPath(); - ctx.moveTo(markerX, 0); + ctx.strokeStyle = positionStrokeStyle; + ctx.lineWidth = positionLineWidth; + ctx.moveTo(position, 0); ctx.lineWidth = 4; - ctx.lineTo(markerX, canvas.height - 4); + ctx.lineTo(position, canvas.height - 4); ctx.stroke(); }); }, [selection, newSelection, position]); @@ -202,19 +220,19 @@ export const HudCanvas: React.FC = ({ break; } - setNewSelection({ ...newSelection, start: start }); + setNewSelection({ ...selection, start: start }); break; } case Mode.ResizingEnd: { const diff = x - moveOffsetX.current; - const start = constrainXToCanvas(selection.end + diff); + const end = constrainXToCanvas(selection.end + diff); - if (start < selection.start) { - setNewSelection({ start: Math.max(0, start), end: selection.start }); + if (end < selection.start) { + setNewSelection({ start: Math.max(0, end), end: selection.start }); break; } - setNewSelection({ ...newSelection, end: start }); + setNewSelection({ ...selection, end: end }); break; } case Mode.Dragging: { diff --git a/frontend/src/Overview.tsx b/frontend/src/Overview.tsx index 1239fe8..21cb049 100644 --- a/frontend/src/Overview.tsx +++ b/frontend/src/Overview.tsx @@ -33,6 +33,7 @@ export const Overview: React.FC = ({ onSelectionChange, }: Props) => { const [selectedPixels, setSelectedPixels] = useState({ start: 0, end: 0 }); + const [positionPixels, setPositionPixels] = useState(0); // side effects @@ -48,6 +49,14 @@ export const Overview: React.FC = ({ }); }, [viewport, mediaSet]); + // convert position from frames to canvas pixels: + useEffect(() => { + const ratio = + position.currentTime / (mediaSet.audioFrames / mediaSet.audioSampleRate); + setPositionPixels(Math.round(ratio * CanvasLogicalWidth)); + frames; + }, [mediaSet, position]); + // handlers // convert selection change from canvas pixels to frames, and trigger callback. @@ -70,6 +79,13 @@ export const Overview: React.FC = ({ height: `${height}px`, } as React.CSSProperties; + const hudStyles = { + borderLineWidth: 4, + borderStrokeStyle: 'red', + positionLineWidth: 4, + positionStrokeStyle: 'red', + }; + return ( <>
@@ -87,7 +103,8 @@ export const Overview: React.FC = ({ width={CanvasLogicalWidth} height={CanvasLogicalHeight} zIndex={1} - position={position} + styles={hudStyles} + position={positionPixels} selection={selectedPixels} onSelectionChange={handleSelectionChange} /> diff --git a/frontend/src/Waveform.tsx b/frontend/src/Waveform.tsx index e05b14c..2253cd5 100644 --- a/frontend/src/Waveform.tsx +++ b/frontend/src/Waveform.tsx @@ -1,8 +1,8 @@ -import { useEffect, useState, useRef } from 'react'; +import { useEffect, useState, useCallback } from 'react'; import { Frames, VideoPosition, newRPC } from './App'; import { MediaSetServiceClientImpl, MediaSet } from './generated/media_set'; import { WaveformCanvas } from './WaveformCanvas'; -import { secsToCanvasX } from './Helpers'; +import { Selection, HudCanvas } from './HudCanvas'; import { from, Observable } from 'rxjs'; import { bufferCount } from 'rxjs/operators'; @@ -11,6 +11,7 @@ interface Props { position: VideoPosition; viewport: Frames; offsetPixels: number; + onSelectionChange: (selection: Selection) => void; } export const CanvasLogicalWidth = 2000; @@ -21,9 +22,15 @@ export const Waveform: React.FC = ({ position, viewport, offsetPixels, + onSelectionChange, }: Props) => { const [peaks, setPeaks] = useState>(from([])); - const hudCanvasRef = useRef(null); + const [selectedFrames, setSelectedFrames] = useState({ start: 0, end: 0 }); + const [selectedPixels, setSelectedPixels] = useState({ + start: 0, + end: 0, + }); + const [positionPixels, setPositionPixels] = useState(0); // effects @@ -57,41 +64,75 @@ export const Waveform: React.FC = ({ })(); }, [viewport]); - // render HUD + // convert position to canvas pixels useEffect(() => { - const canvas = hudCanvasRef.current; - if (canvas == null) { + const frame = Math.round(position.currentTime * mediaSet.audioSampleRate); + if (frame < viewport.start || frame > viewport.end) { + setPositionPixels(null); return; } + const logicalPixelsPerFrame = + CanvasLogicalWidth / (viewport.end - viewport.start); + const positionPixels = (frame - viewport.start) * logicalPixelsPerFrame; + setPositionPixels(positionPixels); + }, [mediaSet, position, viewport]); - const ctx = canvas.getContext('2d'); - if (ctx == null) { - console.error('no hud 2d context available'); - return; + // update selectedPixels on viewport change + useEffect(() => { + const start = frameToCanvasX(selectedFrames.start); + const end = frameToCanvasX(selectedFrames.end); + + // more verbose than it has to be to make TypeScript happy + if (start == null && end == null) { + setSelectedPixels({ start: 0, end: 0 }); + } else if (start == null && end != null) { + setSelectedPixels({ start: 0, end: end }); + } else if (start != null && end == null) { + setSelectedPixels({ start: 0, end: CanvasLogicalWidth }); + } else if (start != null && end != null) { + setSelectedPixels({ start, end }); + } else { + console.error('unreachable'); } + }, [viewport]); - ctx.clearRect(0, 0, canvas.width, canvas.height); + // handlers - if (mediaSet == null) { - return; - } + const handleSelectionChange = useCallback( + (selection: Selection) => { + setSelectedPixels(selection); - const x = secsToCanvasX( - position.currentTime, - mediaSet.audioSampleRate, - viewport - ); - if (x == null) { - return; - } + const framesPerPixel = + (viewport.end - viewport.start) / CanvasLogicalWidth; + const selectedFrames = { + start: Math.round(viewport.start + selection.start * framesPerPixel), + end: Math.round(viewport.start + selection.end * framesPerPixel), + }; - ctx.strokeStyle = 'red'; - ctx.beginPath(); - ctx.moveTo(x, 0); - ctx.lineWidth = 4; - ctx.lineTo(x, canvas.height); - ctx.stroke(); - }, [mediaSet, position]); + setSelectedFrames(selectedFrames); + onSelectionChange(selectedFrames); + }, + [viewport] + ); + + // helpers + + const frameToCanvasX = useCallback( + (frame: number): number | null => { + if (mediaSet == null) { + return null; + } + + if (frame < viewport.start || frame > viewport.end) { + return null; + } + + const pixelsPerFrame = + CanvasLogicalWidth / (viewport.end - viewport.start); + return (frame - viewport.start) * pixelsPerFrame; + }, + [mediaSet, viewport] + ); // render component @@ -102,12 +143,12 @@ export const Waveform: React.FC = ({ position: 'relative', } as React.CSSProperties; - const canvasStyles = { - position: 'absolute', - width: '100%', - height: '100%', - display: 'block', - } as React.CSSProperties; + const hudStyles = { + borderLineWidth: 0, + borderStrokeStyle: 'transparent', + positionLineWidth: 6, + positionStrokeStyle: 'red', + }; return ( <> @@ -122,12 +163,15 @@ export const Waveform: React.FC = ({ zIndex={0} alpha={1} > - + zIndex={1} + styles={hudStyles} + position={positionPixels} + selection={selectedPixels} + onSelectionChange={handleSelectionChange} + />
);