Update frontend
- Add HudCanvas component to Waveform - Allow waveform to be selectable - Fix selection rendering on viewport change - Add spacebar handler
This commit is contained in:
parent
65cc365717
commit
b876fb915a
|
@ -25,7 +25,7 @@ const initialViewportCanvasPixels = 100;
|
|||
|
||||
const apiURL = process.env.REACT_APP_API_URL || 'http://localhost:8888';
|
||||
|
||||
// Frames represents a selection of audio frames.
|
||||
// Frames represents a range of audio frames.
|
||||
export interface Frames {
|
||||
start: number;
|
||||
end: number;
|
||||
|
@ -41,7 +41,8 @@ const audio = document.createElement('audio');
|
|||
|
||||
function App(): JSX.Element {
|
||||
const [mediaSet, setMediaSet] = useState<MediaSet | null>(null);
|
||||
const [viewport, setViewport] = useState({ start: 0, end: 0 });
|
||||
const [viewport, setViewport] = useState<Frames>({ start: 0, end: 0 });
|
||||
const [selection, setSelection] = useState<Frames>({ start: 0, end: 0 });
|
||||
const [overviewPeaks, setOverviewPeaks] = useState<Observable<number[]>>(
|
||||
from([])
|
||||
);
|
||||
|
@ -81,17 +82,35 @@ function App(): JSX.Element {
|
|||
}
|
||||
|
||||
const intervalID = setInterval(() => {
|
||||
if (video.currentTime == positionRef.current.currentTime) {
|
||||
const currTime = audio.currentTime;
|
||||
if (currTime == positionRef.current.currentTime) {
|
||||
return;
|
||||
}
|
||||
const duration = mediaSet.audioFrames / mediaSet.audioSampleRate;
|
||||
const percent = (video.currentTime / duration) * 100;
|
||||
const percent = (currTime / duration) * 100;
|
||||
|
||||
setPosition({ currentTime: video.currentTime, percent: percent });
|
||||
// check if the end of selection has been passed, and pause if so:
|
||||
if (
|
||||
currentTimeToFrame(position.currentTime) < selection.end &&
|
||||
currentTimeToFrame(currTime) >= selection.end
|
||||
) {
|
||||
handlePause();
|
||||
}
|
||||
|
||||
// update the current position
|
||||
setPosition({ currentTime: audio.currentTime, percent: percent });
|
||||
}, updatePlayerPositionIntevalMillis);
|
||||
|
||||
return () => clearInterval(intervalID);
|
||||
}, [mediaSet]);
|
||||
}, [mediaSet, selection]);
|
||||
|
||||
// bind to keypress handler.
|
||||
// selection is a dependency of the handleKeyPress handler, and must be
|
||||
// included here.
|
||||
useEffect(() => {
|
||||
document.addEventListener('keypress', handleKeyPress);
|
||||
return () => document.removeEventListener('keypress', handleKeyPress);
|
||||
}, [selection]);
|
||||
|
||||
// load audio when MediaSet is loaded:
|
||||
useEffect(() => {
|
||||
|
@ -161,7 +180,24 @@ function App(): JSX.Element {
|
|||
|
||||
// handlers
|
||||
|
||||
const handleOverviewSelectionChange = (newViewport: Frames) => {
|
||||
const handleKeyPress = useCallback(
|
||||
(evt: KeyboardEvent) => {
|
||||
if (evt.code != 'Space') {
|
||||
return;
|
||||
}
|
||||
|
||||
if (audio.paused) {
|
||||
handlePlay();
|
||||
} else {
|
||||
handlePause();
|
||||
}
|
||||
},
|
||||
[selection]
|
||||
);
|
||||
|
||||
// handler called when the selection in the overview (zoom setting) is changed.
|
||||
const handleOverviewSelectionChange = useCallback(
|
||||
(newViewport: Frames) => {
|
||||
if (mediaSet == null) {
|
||||
return;
|
||||
}
|
||||
|
@ -172,12 +208,29 @@ function App(): JSX.Element {
|
|||
return;
|
||||
}
|
||||
|
||||
const ratio = newViewport.start / mediaSet.audioFrames;
|
||||
setPositionFromFrame(newViewport.start);
|
||||
},
|
||||
[mediaSet, audio, video]
|
||||
);
|
||||
|
||||
// handler called when the selection in the main waveform view is changed.
|
||||
const handleWaveformSelectionChange = useCallback(
|
||||
(selection: Frames) => {
|
||||
setSelection(selection);
|
||||
|
||||
if (mediaSet == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// move playback position to start of selection
|
||||
const ratio = selection.start / mediaSet.audioFrames;
|
||||
const currentTime =
|
||||
(mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio;
|
||||
audio.currentTime = currentTime;
|
||||
video.currentTime = currentTime;
|
||||
};
|
||||
},
|
||||
[mediaSet, audio, video]
|
||||
);
|
||||
|
||||
const handlePlay = useCallback(() => {
|
||||
audio.play();
|
||||
|
@ -187,7 +240,39 @@ function App(): JSX.Element {
|
|||
const handlePause = useCallback(() => {
|
||||
video.pause();
|
||||
audio.pause();
|
||||
}, [audio, video]);
|
||||
|
||||
if (selection.start != selection.end) {
|
||||
setPositionFromFrame(selection.start);
|
||||
}
|
||||
}, [audio, video, selection]);
|
||||
|
||||
const setPositionFromFrame = useCallback(
|
||||
(frame: number) => {
|
||||
if (mediaSet == null) {
|
||||
return;
|
||||
}
|
||||
const ratio = frame / mediaSet.audioFrames;
|
||||
const currentTime =
|
||||
(mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio;
|
||||
audio.currentTime = currentTime;
|
||||
video.currentTime = currentTime;
|
||||
},
|
||||
[mediaSet, audio, video]
|
||||
);
|
||||
|
||||
// helpers
|
||||
|
||||
const currentTimeToFrame = useCallback(
|
||||
(currentTime: number): number => {
|
||||
if (mediaSet == null) {
|
||||
return 0;
|
||||
}
|
||||
const dur = mediaSet.audioFrames / mediaSet.audioSampleRate;
|
||||
const ratio = currentTime / dur;
|
||||
return Math.round(mediaSet.audioFrames * ratio);
|
||||
},
|
||||
[mediaSet]
|
||||
);
|
||||
|
||||
// render component
|
||||
|
||||
|
@ -229,6 +314,7 @@ function App(): JSX.Element {
|
|||
position={position}
|
||||
viewport={viewport}
|
||||
offsetPixels={offsetPixels}
|
||||
onSelectionChange={handleWaveformSelectionChange}
|
||||
/>
|
||||
|
||||
<SeekBar
|
||||
|
|
|
@ -1,23 +1,10 @@
|
|||
import { MouseEvent } from 'react';
|
||||
import { Frames } from './App';
|
||||
|
||||
// TODO: pass CanvasLogicalWidth as an argument instead.
|
||||
import { CanvasLogicalWidth } from './Waveform';
|
||||
|
||||
interface Point {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
// TODO: add tests
|
||||
export const mouseEventToCanvasX = (
|
||||
evt: MouseEvent<HTMLCanvasElement>
|
||||
): number => {
|
||||
const rect = evt.currentTarget.getBoundingClientRect();
|
||||
const elementX = evt.clientX - rect.left;
|
||||
return (elementX * CanvasLogicalWidth) / rect.width;
|
||||
};
|
||||
|
||||
// TODO: add tests
|
||||
export const mouseEventToCanvasPoint = (
|
||||
evt: MouseEvent<HTMLCanvasElement>
|
||||
|
@ -31,26 +18,3 @@ export const mouseEventToCanvasPoint = (
|
|||
y: (elementY * evt.currentTarget.height) / rect.height,
|
||||
};
|
||||
};
|
||||
|
||||
// TODO: add tests
|
||||
export const canvasXToFrame = (x: number, numFrames: number): number => {
|
||||
return Math.floor((x / CanvasLogicalWidth) * numFrames);
|
||||
};
|
||||
|
||||
// TODO: add tests
|
||||
// secsToCanvasX returns the logical x coordinate for a given position
|
||||
// marker. It is null if the marker is outside of the current viewport.
|
||||
export const secsToCanvasX = (
|
||||
secs: number,
|
||||
sampleRate: number,
|
||||
viewport: Frames
|
||||
): number | null => {
|
||||
const frame = Math.round(secs * sampleRate);
|
||||
if (frame < viewport.start || frame > viewport.end) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const logicalPixelsPerFrame =
|
||||
CanvasLogicalWidth / (viewport.end - viewport.start);
|
||||
return (frame - viewport.start) * logicalPixelsPerFrame;
|
||||
};
|
||||
|
|
|
@ -1,11 +1,18 @@
|
|||
import { useState, useEffect, useRef, MouseEvent } from 'react';
|
||||
import { VideoPosition } from './App';
|
||||
|
||||
interface Styles {
|
||||
borderLineWidth: number;
|
||||
borderStrokeStyle: string;
|
||||
positionLineWidth: number;
|
||||
positionStrokeStyle: string;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
width: number;
|
||||
height: number;
|
||||
zIndex: number;
|
||||
position: VideoPosition;
|
||||
styles: Styles;
|
||||
position: number | null;
|
||||
selection: Selection;
|
||||
onSelectionChange: (selection: Selection) => void;
|
||||
}
|
||||
|
@ -36,6 +43,12 @@ export const HudCanvas: React.FC<Props> = ({
|
|||
width,
|
||||
height,
|
||||
zIndex,
|
||||
styles: {
|
||||
borderLineWidth,
|
||||
borderStrokeStyle,
|
||||
positionLineWidth,
|
||||
positionStrokeStyle,
|
||||
},
|
||||
position,
|
||||
selection,
|
||||
onSelectionChange,
|
||||
|
@ -91,26 +104,31 @@ export const HudCanvas: React.FC<Props> = ({
|
|||
}
|
||||
|
||||
ctx.beginPath();
|
||||
ctx.strokeStyle = 'red';
|
||||
ctx.lineWidth = 4;
|
||||
ctx.strokeStyle = borderStrokeStyle;
|
||||
ctx.lineWidth = borderLineWidth;
|
||||
const alpha = hoverState == HoverState.OverSelection ? '0.15' : '0.13';
|
||||
ctx.fillStyle = `rgba(255, 255, 255, ${alpha})`;
|
||||
ctx.rect(
|
||||
currentSelection.start,
|
||||
2,
|
||||
borderLineWidth,
|
||||
currentSelection.end - currentSelection.start,
|
||||
canvas.height - 10
|
||||
canvas.height - borderLineWidth * 2
|
||||
);
|
||||
ctx.fill();
|
||||
ctx.stroke();
|
||||
|
||||
// draw position marker
|
||||
|
||||
const markerX = canvas.width * (position.percent / 100);
|
||||
if (position == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(markerX, 0);
|
||||
ctx.strokeStyle = positionStrokeStyle;
|
||||
ctx.lineWidth = positionLineWidth;
|
||||
ctx.moveTo(position, 0);
|
||||
ctx.lineWidth = 4;
|
||||
ctx.lineTo(markerX, canvas.height - 4);
|
||||
ctx.lineTo(position, canvas.height - 4);
|
||||
ctx.stroke();
|
||||
});
|
||||
}, [selection, newSelection, position]);
|
||||
|
@ -202,19 +220,19 @@ export const HudCanvas: React.FC<Props> = ({
|
|||
break;
|
||||
}
|
||||
|
||||
setNewSelection({ ...newSelection, start: start });
|
||||
setNewSelection({ ...selection, start: start });
|
||||
break;
|
||||
}
|
||||
case Mode.ResizingEnd: {
|
||||
const diff = x - moveOffsetX.current;
|
||||
const start = constrainXToCanvas(selection.end + diff);
|
||||
const end = constrainXToCanvas(selection.end + diff);
|
||||
|
||||
if (start < selection.start) {
|
||||
setNewSelection({ start: Math.max(0, start), end: selection.start });
|
||||
if (end < selection.start) {
|
||||
setNewSelection({ start: Math.max(0, end), end: selection.start });
|
||||
break;
|
||||
}
|
||||
|
||||
setNewSelection({ ...newSelection, end: start });
|
||||
setNewSelection({ ...selection, end: end });
|
||||
break;
|
||||
}
|
||||
case Mode.Dragging: {
|
||||
|
|
|
@ -33,6 +33,7 @@ export const Overview: React.FC<Props> = ({
|
|||
onSelectionChange,
|
||||
}: Props) => {
|
||||
const [selectedPixels, setSelectedPixels] = useState({ start: 0, end: 0 });
|
||||
const [positionPixels, setPositionPixels] = useState(0);
|
||||
|
||||
// side effects
|
||||
|
||||
|
@ -48,6 +49,14 @@ export const Overview: React.FC<Props> = ({
|
|||
});
|
||||
}, [viewport, mediaSet]);
|
||||
|
||||
// convert position from frames to canvas pixels:
|
||||
useEffect(() => {
|
||||
const ratio =
|
||||
position.currentTime / (mediaSet.audioFrames / mediaSet.audioSampleRate);
|
||||
setPositionPixels(Math.round(ratio * CanvasLogicalWidth));
|
||||
frames;
|
||||
}, [mediaSet, position]);
|
||||
|
||||
// handlers
|
||||
|
||||
// convert selection change from canvas pixels to frames, and trigger callback.
|
||||
|
@ -70,6 +79,13 @@ export const Overview: React.FC<Props> = ({
|
|||
height: `${height}px`,
|
||||
} as React.CSSProperties;
|
||||
|
||||
const hudStyles = {
|
||||
borderLineWidth: 4,
|
||||
borderStrokeStyle: 'red',
|
||||
positionLineWidth: 4,
|
||||
positionStrokeStyle: 'red',
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<div style={containerStyles}>
|
||||
|
@ -87,7 +103,8 @@ export const Overview: React.FC<Props> = ({
|
|||
width={CanvasLogicalWidth}
|
||||
height={CanvasLogicalHeight}
|
||||
zIndex={1}
|
||||
position={position}
|
||||
styles={hudStyles}
|
||||
position={positionPixels}
|
||||
selection={selectedPixels}
|
||||
onSelectionChange={handleSelectionChange}
|
||||
/>
|
||||
|
|
|
@ -1,8 +1,8 @@
|
|||
import { useEffect, useState, useRef } from 'react';
|
||||
import { useEffect, useState, useCallback } from 'react';
|
||||
import { Frames, VideoPosition, newRPC } from './App';
|
||||
import { MediaSetServiceClientImpl, MediaSet } from './generated/media_set';
|
||||
import { WaveformCanvas } from './WaveformCanvas';
|
||||
import { secsToCanvasX } from './Helpers';
|
||||
import { Selection, HudCanvas } from './HudCanvas';
|
||||
import { from, Observable } from 'rxjs';
|
||||
import { bufferCount } from 'rxjs/operators';
|
||||
|
||||
|
@ -11,6 +11,7 @@ interface Props {
|
|||
position: VideoPosition;
|
||||
viewport: Frames;
|
||||
offsetPixels: number;
|
||||
onSelectionChange: (selection: Selection) => void;
|
||||
}
|
||||
|
||||
export const CanvasLogicalWidth = 2000;
|
||||
|
@ -21,9 +22,15 @@ export const Waveform: React.FC<Props> = ({
|
|||
position,
|
||||
viewport,
|
||||
offsetPixels,
|
||||
onSelectionChange,
|
||||
}: Props) => {
|
||||
const [peaks, setPeaks] = useState<Observable<number[]>>(from([]));
|
||||
const hudCanvasRef = useRef<HTMLCanvasElement>(null);
|
||||
const [selectedFrames, setSelectedFrames] = useState({ start: 0, end: 0 });
|
||||
const [selectedPixels, setSelectedPixels] = useState({
|
||||
start: 0,
|
||||
end: 0,
|
||||
});
|
||||
const [positionPixels, setPositionPixels] = useState<number | null>(0);
|
||||
|
||||
// effects
|
||||
|
||||
|
@ -57,41 +64,75 @@ export const Waveform: React.FC<Props> = ({
|
|||
})();
|
||||
}, [viewport]);
|
||||
|
||||
// render HUD
|
||||
// convert position to canvas pixels
|
||||
useEffect(() => {
|
||||
const canvas = hudCanvasRef.current;
|
||||
if (canvas == null) {
|
||||
const frame = Math.round(position.currentTime * mediaSet.audioSampleRate);
|
||||
if (frame < viewport.start || frame > viewport.end) {
|
||||
setPositionPixels(null);
|
||||
return;
|
||||
}
|
||||
const logicalPixelsPerFrame =
|
||||
CanvasLogicalWidth / (viewport.end - viewport.start);
|
||||
const positionPixels = (frame - viewport.start) * logicalPixelsPerFrame;
|
||||
setPositionPixels(positionPixels);
|
||||
}, [mediaSet, position, viewport]);
|
||||
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (ctx == null) {
|
||||
console.error('no hud 2d context available');
|
||||
return;
|
||||
// update selectedPixels on viewport change
|
||||
useEffect(() => {
|
||||
const start = frameToCanvasX(selectedFrames.start);
|
||||
const end = frameToCanvasX(selectedFrames.end);
|
||||
|
||||
// more verbose than it has to be to make TypeScript happy
|
||||
if (start == null && end == null) {
|
||||
setSelectedPixels({ start: 0, end: 0 });
|
||||
} else if (start == null && end != null) {
|
||||
setSelectedPixels({ start: 0, end: end });
|
||||
} else if (start != null && end == null) {
|
||||
setSelectedPixels({ start: 0, end: CanvasLogicalWidth });
|
||||
} else if (start != null && end != null) {
|
||||
setSelectedPixels({ start, end });
|
||||
} else {
|
||||
console.error('unreachable');
|
||||
}
|
||||
}, [viewport]);
|
||||
|
||||
ctx.clearRect(0, 0, canvas.width, canvas.height);
|
||||
// handlers
|
||||
|
||||
if (mediaSet == null) {
|
||||
return;
|
||||
}
|
||||
const handleSelectionChange = useCallback(
|
||||
(selection: Selection) => {
|
||||
setSelectedPixels(selection);
|
||||
|
||||
const x = secsToCanvasX(
|
||||
position.currentTime,
|
||||
mediaSet.audioSampleRate,
|
||||
viewport
|
||||
const framesPerPixel =
|
||||
(viewport.end - viewport.start) / CanvasLogicalWidth;
|
||||
const selectedFrames = {
|
||||
start: Math.round(viewport.start + selection.start * framesPerPixel),
|
||||
end: Math.round(viewport.start + selection.end * framesPerPixel),
|
||||
};
|
||||
|
||||
setSelectedFrames(selectedFrames);
|
||||
onSelectionChange(selectedFrames);
|
||||
},
|
||||
[viewport]
|
||||
);
|
||||
if (x == null) {
|
||||
return;
|
||||
|
||||
// helpers
|
||||
|
||||
const frameToCanvasX = useCallback(
|
||||
(frame: number): number | null => {
|
||||
if (mediaSet == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
ctx.strokeStyle = 'red';
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(x, 0);
|
||||
ctx.lineWidth = 4;
|
||||
ctx.lineTo(x, canvas.height);
|
||||
ctx.stroke();
|
||||
}, [mediaSet, position]);
|
||||
if (frame < viewport.start || frame > viewport.end) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const pixelsPerFrame =
|
||||
CanvasLogicalWidth / (viewport.end - viewport.start);
|
||||
return (frame - viewport.start) * pixelsPerFrame;
|
||||
},
|
||||
[mediaSet, viewport]
|
||||
);
|
||||
|
||||
// render component
|
||||
|
||||
|
@ -102,12 +143,12 @@ export const Waveform: React.FC<Props> = ({
|
|||
position: 'relative',
|
||||
} as React.CSSProperties;
|
||||
|
||||
const canvasStyles = {
|
||||
position: 'absolute',
|
||||
width: '100%',
|
||||
height: '100%',
|
||||
display: 'block',
|
||||
} as React.CSSProperties;
|
||||
const hudStyles = {
|
||||
borderLineWidth: 0,
|
||||
borderStrokeStyle: 'transparent',
|
||||
positionLineWidth: 6,
|
||||
positionStrokeStyle: 'red',
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
|
@ -122,12 +163,15 @@ export const Waveform: React.FC<Props> = ({
|
|||
zIndex={0}
|
||||
alpha={1}
|
||||
></WaveformCanvas>
|
||||
<canvas
|
||||
<HudCanvas
|
||||
width={CanvasLogicalWidth}
|
||||
height={CanvasLogicalHeight}
|
||||
ref={hudCanvasRef}
|
||||
style={canvasStyles}
|
||||
></canvas>
|
||||
zIndex={1}
|
||||
styles={hudStyles}
|
||||
position={positionPixels}
|
||||
selection={selectedPixels}
|
||||
onSelectionChange={handleSelectionChange}
|
||||
/>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
|
|
Loading…
Reference in New Issue