rob
/
clipper
2
0
Fork 0
Code Issues 11 Pull Requests 1 Projects Releases Wiki Activity

Update frontend 

- Add HudCanvas component to Waveform
- Allow waveform to be selectable
- Fix selection rendering on viewport change
- Add spacebar handler
This commit is contained in:
Rob Watson 2021-12-11 17:25:43 +01:00
parent 65cc365717
commit b876fb915a
5 changed files with 241 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
frontend/src/App.tsx
View File

@ -25,7 +25,7 @@ const initialViewportCanvasPixels = 100;
const apiURL = process.env.REACT_APP_API_URL || 'http://localhost:8888';
// Frames represents a selection of audio frames.
// Frames represents a range of audio frames.
export interface Frames {
start: number;
end: number;
@ -41,7 +41,8 @@ const audio = document.createElement('audio');
function App(): JSX.Element {
const [mediaSet, setMediaSet] = useState<MediaSet | null>(null);
const [viewport, setViewport] = useState({ start: 0, end: 0 });
const [viewport, setViewport] = useState<Frames>({ start: 0, end: 0 });
const [selection, setSelection] = useState<Frames>({ start: 0, end: 0 });
const [overviewPeaks, setOverviewPeaks] = useState<Observable<number[]>>(
from([])
);
@ -81,17 +82,35 @@ function App(): JSX.Element {
}
const intervalID = setInterval(() => {
if (video.currentTime == positionRef.current.currentTime) {
const currTime = audio.currentTime;
if (currTime == positionRef.current.currentTime) {
return;
}
const duration = mediaSet.audioFrames / mediaSet.audioSampleRate;
const percent = (video.currentTime / duration) * 100;
const percent = (currTime / duration) * 100;
setPosition({ currentTime: video.currentTime, percent: percent });
// check if the end of selection has been passed, and pause if so:
if (
currentTimeToFrame(position.currentTime) < selection.end &&
currentTimeToFrame(currTime) >= selection.end
) {
handlePause();
}
// update the current position
setPosition({ currentTime: audio.currentTime, percent: percent });
}, updatePlayerPositionIntevalMillis);
return () => clearInterval(intervalID);
}, [mediaSet]);
}, [mediaSet, selection]);
// bind to keypress handler.
// selection is a dependency of the handleKeyPress handler, and must be
// included here.
useEffect(() => {
document.addEventListener('keypress', handleKeyPress);
return () => document.removeEventListener('keypress', handleKeyPress);
}, [selection]);
// load audio when MediaSet is loaded:
useEffect(() => {
@ -161,7 +180,24 @@ function App(): JSX.Element {
// handlers
const handleOverviewSelectionChange = (newViewport: Frames) => {
const handleKeyPress = useCallback(
(evt: KeyboardEvent) => {
if (evt.code != 'Space') {
return;
}
if (audio.paused) {
handlePlay();
} else {
handlePause();
}
},
[selection]
);
// handler called when the selection in the overview (zoom setting) is changed.
const handleOverviewSelectionChange = useCallback(
(newViewport: Frames) => {
if (mediaSet == null) {
return;
}
@ -172,12 +208,29 @@ function App(): JSX.Element {
return;
}
const ratio = newViewport.start / mediaSet.audioFrames;
setPositionFromFrame(newViewport.start);
},
[mediaSet, audio, video]
);
// handler called when the selection in the main waveform view is changed.
const handleWaveformSelectionChange = useCallback(
(selection: Frames) => {
setSelection(selection);
if (mediaSet == null) {
return;
}
// move playback position to start of selection
const ratio = selection.start / mediaSet.audioFrames;
const currentTime =
(mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio;
audio.currentTime = currentTime;
video.currentTime = currentTime;
};
},
[mediaSet, audio, video]
);
const handlePlay = useCallback(() => {
audio.play();
@ -187,7 +240,39 @@ function App(): JSX.Element {
const handlePause = useCallback(() => {
video.pause();
audio.pause();
}, [audio, video]);
if (selection.start != selection.end) {
setPositionFromFrame(selection.start);
}
}, [audio, video, selection]);
const setPositionFromFrame = useCallback(
(frame: number) => {
if (mediaSet == null) {
return;
}
const ratio = frame / mediaSet.audioFrames;
const currentTime =
(mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio;
audio.currentTime = currentTime;
video.currentTime = currentTime;
},
[mediaSet, audio, video]
);
// helpers
const currentTimeToFrame = useCallback(
(currentTime: number): number => {
if (mediaSet == null) {
return 0;
}
const dur = mediaSet.audioFrames / mediaSet.audioSampleRate;
const ratio = currentTime / dur;
return Math.round(mediaSet.audioFrames * ratio);
},
[mediaSet]
);
// render component
@ -229,6 +314,7 @@ function App(): JSX.Element {
position={position}
viewport={viewport}
offsetPixels={offsetPixels}
onSelectionChange={handleWaveformSelectionChange}
/>
<SeekBar

36
frontend/src/Helpers.tsx
View File

@ -1,23 +1,10 @@
import { MouseEvent } from 'react';
import { Frames } from './App';
// TODO: pass CanvasLogicalWidth as an argument instead.
import { CanvasLogicalWidth } from './Waveform';
interface Point {
x: number;
y: number;
}
// TODO: add tests
export const mouseEventToCanvasX = (
evt: MouseEvent<HTMLCanvasElement>
): number => {
const rect = evt.currentTarget.getBoundingClientRect();
const elementX = evt.clientX - rect.left;
return (elementX * CanvasLogicalWidth) / rect.width;
};
// TODO: add tests
export const mouseEventToCanvasPoint = (
evt: MouseEvent<HTMLCanvasElement>
@ -31,26 +18,3 @@ export const mouseEventToCanvasPoint = (
y: (elementY * evt.currentTarget.height) / rect.height,
};
};
// TODO: add tests
export const canvasXToFrame = (x: number, numFrames: number): number => {
return Math.floor((x / CanvasLogicalWidth) * numFrames);
};
// TODO: add tests
// secsToCanvasX returns the logical x coordinate for a given position
// marker. It is null if the marker is outside of the current viewport.
export const secsToCanvasX = (
secs: number,
sampleRate: number,
viewport: Frames
): number | null => {
const frame = Math.round(secs * sampleRate);
if (frame < viewport.start || frame > viewport.end) {
return null;
}
const logicalPixelsPerFrame =
CanvasLogicalWidth / (viewport.end - viewport.start);
return (frame - viewport.start) * logicalPixelsPerFrame;
};

46
frontend/src/HudCanvas.tsx
View File

@ -1,11 +1,18 @@
import { useState, useEffect, useRef, MouseEvent } from 'react';
import { VideoPosition } from './App';
interface Styles {
borderLineWidth: number;
borderStrokeStyle: string;
positionLineWidth: number;
positionStrokeStyle: string;
}
interface Props {
width: number;
height: number;
zIndex: number;
position: VideoPosition;
styles: Styles;
position: number | null;
selection: Selection;
onSelectionChange: (selection: Selection) => void;
}
@ -36,6 +43,12 @@ export const HudCanvas: React.FC<Props> = ({
width,
height,
zIndex,
styles: {
borderLineWidth,
borderStrokeStyle,
positionLineWidth,
positionStrokeStyle,
},
position,
selection,
onSelectionChange,
@ -91,26 +104,31 @@ export const HudCanvas: React.FC<Props> = ({
}
ctx.beginPath();
ctx.strokeStyle = 'red';
ctx.lineWidth = 4;
ctx.strokeStyle = borderStrokeStyle;
ctx.lineWidth = borderLineWidth;
const alpha = hoverState == HoverState.OverSelection ? '0.15' : '0.13';
ctx.fillStyle = `rgba(255, 255, 255, ${alpha})`;
ctx.rect(
currentSelection.start,
2,
borderLineWidth,
currentSelection.end - currentSelection.start,
canvas.height - 10
canvas.height - borderLineWidth * 2
);
ctx.fill();
ctx.stroke();
// draw position marker
const markerX = canvas.width * (position.percent / 100);
if (position == null) {
return;
}
ctx.beginPath();
ctx.moveTo(markerX, 0);
ctx.strokeStyle = positionStrokeStyle;
ctx.lineWidth = positionLineWidth;
ctx.moveTo(position, 0);
ctx.lineWidth = 4;
ctx.lineTo(markerX, canvas.height - 4);
ctx.lineTo(position, canvas.height - 4);
ctx.stroke();
});
}, [selection, newSelection, position]);
@ -202,19 +220,19 @@ export const HudCanvas: React.FC<Props> = ({
break;
}
setNewSelection({ ...newSelection, start: start });
setNewSelection({ ...selection, start: start });
break;
}
case Mode.ResizingEnd: {
const diff = x - moveOffsetX.current;
const start = constrainXToCanvas(selection.end + diff);
const end = constrainXToCanvas(selection.end + diff);
if (start < selection.start) {
setNewSelection({ start: Math.max(0, start), end: selection.start });
if (end < selection.start) {
setNewSelection({ start: Math.max(0, end), end: selection.start });
break;
}
setNewSelection({ ...newSelection, end: start });
setNewSelection({ ...selection, end: end });
break;
}
case Mode.Dragging: {

19
frontend/src/Overview.tsx
View File

@ -33,6 +33,7 @@ export const Overview: React.FC<Props> = ({
onSelectionChange,
}: Props) => {
const [selectedPixels, setSelectedPixels] = useState({ start: 0, end: 0 });
const [positionPixels, setPositionPixels] = useState(0);
// side effects
@ -48,6 +49,14 @@ export const Overview: React.FC<Props> = ({
});
}, [viewport, mediaSet]);
// convert position from frames to canvas pixels:
useEffect(() => {
const ratio =
position.currentTime / (mediaSet.audioFrames / mediaSet.audioSampleRate);
setPositionPixels(Math.round(ratio * CanvasLogicalWidth));
frames;
}, [mediaSet, position]);
// handlers
// convert selection change from canvas pixels to frames, and trigger callback.
@ -70,6 +79,13 @@ export const Overview: React.FC<Props> = ({
height: `${height}px`,
} as React.CSSProperties;
const hudStyles = {
borderLineWidth: 4,
borderStrokeStyle: 'red',
positionLineWidth: 4,
positionStrokeStyle: 'red',
};
return (
<>
<div style={containerStyles}>
@ -87,7 +103,8 @@ export const Overview: React.FC<Props> = ({
width={CanvasLogicalWidth}
height={CanvasLogicalHeight}
zIndex={1}
position={position}
styles={hudStyles}
position={positionPixels}
selection={selectedPixels}
onSelectionChange={handleSelectionChange}
/>

118
frontend/src/Waveform.tsx
View File

@ -1,8 +1,8 @@
import { useEffect, useState, useRef } from 'react';
import { useEffect, useState, useCallback } from 'react';
import { Frames, VideoPosition, newRPC } from './App';
import { MediaSetServiceClientImpl, MediaSet } from './generated/media_set';
import { WaveformCanvas } from './WaveformCanvas';
import { secsToCanvasX } from './Helpers';
import { Selection, HudCanvas } from './HudCanvas';
import { from, Observable } from 'rxjs';
import { bufferCount } from 'rxjs/operators';
@ -11,6 +11,7 @@ interface Props {
position: VideoPosition;
viewport: Frames;
offsetPixels: number;
onSelectionChange: (selection: Selection) => void;
}
export const CanvasLogicalWidth = 2000;
@ -21,9 +22,15 @@ export const Waveform: React.FC<Props> = ({
position,
viewport,
offsetPixels,
onSelectionChange,
}: Props) => {
const [peaks, setPeaks] = useState<Observable<number[]>>(from([]));
const hudCanvasRef = useRef<HTMLCanvasElement>(null);
const [selectedFrames, setSelectedFrames] = useState({ start: 0, end: 0 });
const [selectedPixels, setSelectedPixels] = useState({
start: 0,
end: 0,
});
const [positionPixels, setPositionPixels] = useState<number | null>(0);
// effects
@ -57,41 +64,75 @@ export const Waveform: React.FC<Props> = ({
})();
}, [viewport]);
// render HUD
// convert position to canvas pixels
useEffect(() => {
const canvas = hudCanvasRef.current;
if (canvas == null) {
const frame = Math.round(position.currentTime * mediaSet.audioSampleRate);
if (frame < viewport.start || frame > viewport.end) {
setPositionPixels(null);
return;
}
const logicalPixelsPerFrame =
CanvasLogicalWidth / (viewport.end - viewport.start);
const positionPixels = (frame - viewport.start) * logicalPixelsPerFrame;
setPositionPixels(positionPixels);
}, [mediaSet, position, viewport]);
const ctx = canvas.getContext('2d');
if (ctx == null) {
console.error('no hud 2d context available');
return;
// update selectedPixels on viewport change
useEffect(() => {
const start = frameToCanvasX(selectedFrames.start);
const end = frameToCanvasX(selectedFrames.end);
// more verbose than it has to be to make TypeScript happy
if (start == null && end == null) {
setSelectedPixels({ start: 0, end: 0 });
} else if (start == null && end != null) {
setSelectedPixels({ start: 0, end: end });
} else if (start != null && end == null) {
setSelectedPixels({ start: 0, end: CanvasLogicalWidth });
} else if (start != null && end != null) {
setSelectedPixels({ start, end });
} else {
console.error('unreachable');
}
}, [viewport]);
ctx.clearRect(0, 0, canvas.width, canvas.height);
// handlers
if (mediaSet == null) {
return;
}
const handleSelectionChange = useCallback(
(selection: Selection) => {
setSelectedPixels(selection);
const x = secsToCanvasX(
position.currentTime,
mediaSet.audioSampleRate,
viewport
const framesPerPixel =
(viewport.end - viewport.start) / CanvasLogicalWidth;
const selectedFrames = {
start: Math.round(viewport.start + selection.start * framesPerPixel),
end: Math.round(viewport.start + selection.end * framesPerPixel),
};
setSelectedFrames(selectedFrames);
onSelectionChange(selectedFrames);
},
[viewport]
);
if (x == null) {
return;
// helpers
const frameToCanvasX = useCallback(
(frame: number): number | null => {
if (mediaSet == null) {
return null;
}
ctx.strokeStyle = 'red';
ctx.beginPath();
ctx.moveTo(x, 0);
ctx.lineWidth = 4;
ctx.lineTo(x, canvas.height);
ctx.stroke();
}, [mediaSet, position]);
if (frame < viewport.start || frame > viewport.end) {
return null;
}
const pixelsPerFrame =
CanvasLogicalWidth / (viewport.end - viewport.start);
return (frame - viewport.start) * pixelsPerFrame;
},
[mediaSet, viewport]
);
// render component
@ -102,12 +143,12 @@ export const Waveform: React.FC<Props> = ({
position: 'relative',
} as React.CSSProperties;
const canvasStyles = {
position: 'absolute',
width: '100%',
height: '100%',
display: 'block',
} as React.CSSProperties;
const hudStyles = {
borderLineWidth: 0,
borderStrokeStyle: 'transparent',
positionLineWidth: 6,
positionStrokeStyle: 'red',
};
return (
<>
@ -122,12 +163,15 @@ export const Waveform: React.FC<Props> = ({
zIndex={0}
alpha={1}
></WaveformCanvas>
<canvas
<HudCanvas
width={CanvasLogicalWidth}
height={CanvasLogicalHeight}
ref={hudCanvasRef}
style={canvasStyles}
></canvas>
zIndex={1}
styles={hudStyles}
position={positionPixels}
selection={selectedPixels}
onSelectionChange={handleSelectionChange}
/>
</div>
</>
);