rob
/
clipper
2
0
Fork 0
Code Issues 11 Pull Requests 1 Projects Releases Wiki Activity

Frontend fixes

This commit is contained in:
Rob Watson 2021-11-30 20:41:34 +01:00
parent 642ce6e349
commit 0cc1fd8272
7 changed files with 136 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
backend/media/service.go
View File

@ -371,6 +371,11 @@ outer:
}
func (s *MediaSetService) GetAudioSegment(ctx context.Context, id uuid.UUID, startFrame, endFrame int64, numBins int) ([]int16, error) {
if startFrame < 0 || endFrame < 0 || numBins <= 0 {
s.logger.With("startFrame", startFrame, "endFrame", endFrame, "numBins", numBins).Error("invalid arguments")
return nil, errors.New("invalid arguments")
}
mediaSet, err := s.store.GetMediaSet(ctx, id)
if err != nil {
return nil, fmt.Errorf("error getting media set: %v", err)

2
deploy.sh
View File

@ -15,6 +15,8 @@
# TODO: production build
set -ex
export DOCKER_BUILDKIT=1
docker build \
-t netfluxio/clipper-staging:latest \
--build-arg API_URL=https://clipper-staging.netflux.io \

85
frontend/src/App.tsx
View File

@ -6,7 +6,7 @@ import {
GetAudioProgress,
} from './generated/media_set';
import { useState, useEffect } from 'react';
import { useState, useEffect, useRef } from 'react';
import { VideoPreview } from './VideoPreview';
import { Overview, CanvasLogicalWidth } from './Overview';
import { Waveform } from './Waveform';
@ -14,8 +14,8 @@ import { ControlBar } from './ControlBar';
import { SeekBar } from './SeekBar';
import './App.css';
import { Duration } from './generated/google/protobuf/duration';
import { from, Observable } from 'rxjs';
import { map } from 'rxjs/operators';
import { firstValueFrom, from, Observable } from 'rxjs';
import { first, map } from 'rxjs/operators';
// ported from backend, where should they live?
const thumbnailWidth = 177;
@ -36,16 +36,22 @@ export interface VideoPosition {
percent: number;
}
const video = document.createElement('video');
const audio = document.createElement('audio');
function App(): JSX.Element {
const [mediaSet, setMediaSet] = useState<MediaSet | null>(null);
const [video, _setVideo] = useState(document.createElement('video'));
const [audio, _setAudio] = useState(document.createElement('audio'));
const [position, setPosition] = useState({ currentTime: 0, percent: 0 });
const [viewport, setViewport] = useState({ start: 0, end: 0 });
const [overviewPeaks, setOverviewPeaks] = useState<Observable<number[]>>(
from([])
);
// position stores the current playback position. positionRef makes it
// available inside a setInterval callback.
const [position, setPosition] = useState({ currentTime: 0, percent: 0 });
const positionRef = useRef(position);
positionRef.current = position;
// effects
// TODO: error handling
@ -66,22 +72,25 @@ function App(): JSX.Element {
})();
}, []);
const updatePlayerPositionIntevalMillis = 30;
// setup player on first page load only:
useEffect(() => {
if (mediaSet == null) {
return;
}
// assume mediaSet never changes once loaded
setInterval(() => {
if (video.currentTime == position.currentTime) {
const intervalID = setInterval(() => {
if (video.currentTime == positionRef.current.currentTime) {
return;
}
const duration = mediaSet.audioFrames / mediaSet.audioSampleRate;
const percent = (video.currentTime / duration) * 100;
setPosition({ currentTime: video.currentTime, percent: percent });
}, 100);
}, updatePlayerPositionIntevalMillis);
return () => clearInterval(intervalID);
}, [mediaSet]);
// load audio when MediaSet is loaded:
@ -91,7 +100,6 @@ function App(): JSX.Element {
return;
}
console.log('fetching audio...');
// TODO move this call to app.tsx, pass the stream in as a prop.
const service = new MediaSetServiceClientImpl(newRPC());
const audioProgressStream = service.GetAudio({
id: mediaSet.id,
@ -100,18 +108,15 @@ function App(): JSX.Element {
const peaks = audioProgressStream.pipe(map((progress) => progress.peaks));
setOverviewPeaks(peaks);
let url = '';
// TODO: probably a nicer way to do this.
await audioProgressStream.forEach((progress: GetAudioProgress) => {
if (progress.url != '') {
url = progress.url;
}
});
const pipe = audioProgressStream.pipe(
first((progress: GetAudioProgress) => progress.url != '')
);
const progressWithURL = await firstValueFrom(pipe);
audio.src = url;
audio.src = progressWithURL.url;
audio.muted = false;
audio.volume = 1;
console.log('got audio URL', url);
console.log('set audio src', progressWithURL.url);
})();
}, [mediaSet]);
@ -122,21 +127,16 @@ function App(): JSX.Element {
return;
}
console.log('getting video...');
const rpc = newRPC();
const service = new MediaSetServiceClientImpl(rpc);
console.log('fetching video...');
const service = new MediaSetServiceClientImpl(newRPC());
const videoProgressStream = service.GetVideo({ id: mediaSet.id });
const pipe = videoProgressStream.pipe(
first((progress: GetVideoProgress) => progress.url != '')
);
const progressWithURL = await firstValueFrom(pipe);
let url = '';
// TODO: probably a nicer way to do this.
await videoProgressStream.forEach((progress: GetVideoProgress) => {
if (progress.url != '') {
url = progress.url;
}
});
video.src = url;
console.log('set video src', video.src);
video.src = progressWithURL.url;
console.log('set video src', progressWithURL.url);
})();
}, [mediaSet]);
@ -160,16 +160,22 @@ function App(): JSX.Element {
// handlers
const handleOverviewSelectionChange = (selection: Frames) => {
console.log('in handleOverviewSelectionChange', selection);
const handleOverviewSelectionChange = (newViewport: Frames) => {
if (mediaSet == null) {
return;
}
console.log('set new viewport', newViewport);
setViewport({ ...newViewport });
setViewport({
start: mediaSet.audioFrames * (selection.start / 100),
end: mediaSet.audioFrames * (selection.end / 100),
});
if (!audio.paused) {
return;
}
const ratio = newViewport.start / mediaSet.audioFrames;
const currentTime =
(mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio;
audio.currentTime = currentTime;
video.currentTime = currentTime;
};
// render component
@ -211,6 +217,7 @@ function App(): JSX.Element {
mediaSet={mediaSet}
offsetPixels={offsetPixels}
height={80}
viewport={viewport}
position={position}
onSelectionChange={handleOverviewSelectionChange}
/>

2
frontend/src/Helpers.tsx
View File

@ -45,7 +45,7 @@ export const secsToCanvasX = (
sampleRate: number,
viewport: Frames
): number | null => {
const frame = Math.floor(secs * sampleRate);
const frame = Math.round(secs * sampleRate);
if (frame < viewport.start || frame > viewport.end) {
return null;
}

137
frontend/src/Overview.tsx
View File

@ -15,6 +15,7 @@ interface Props {
height: number;
offsetPixels: number;
position: VideoPosition;
viewport: Frames;
onSelectionChange: (selection: Frames) => void;
}
@ -44,20 +45,23 @@ export const Overview: React.FC<Props> = ({
height,
offsetPixels,
position,
viewport,
onSelectionChange,
}: Props) => {
const hudCanvasRef = useRef<HTMLCanvasElement>(null);
const [mode, setMode] = useState(Mode.Normal);
const [hoverState, setHoverState] = useState(HoverState.Normal);
const [cursor, setCursor] = useState('auto');
// selection and newSelection relate to canvas logical pixels:
const [selection, setSelection] = useState({ ...emptySelection });
const [newSelection, setNewSelection] = useState({
...emptySelection,
});
const [selection, setSelection] = useState({ start: 0, end: 100 });
const [cursor, setCursor] = useState('auto');
const moveOffsetX = useRef(0);
// effects
// side effects
// handle global mouse up.
useEffect(() => {
@ -67,30 +71,21 @@ export const Overview: React.FC<Props> = ({
};
}, [mode, newSelection]);
// publish onSelectionChange event
// set selection state on viewport change
useEffect(() => {
if (mediaSet == null) {
return;
}
const canvas = hudCanvasRef.current;
if (canvas == null) {
console.error('no hud canvas ref available');
return;
}
const ctx = canvas.getContext('2d');
if (ctx == null) {
console.error('no hud 2d context available');
return;
}
const width = canvas.getBoundingClientRect().width;
const selectionPercent = {
start: (selection.start / width) * 100,
end: (selection.end / width) * 100,
};
onSelectionChange(selectionPercent);
}, [selection]);
setSelection({
start: Math.round(
(viewport.start / mediaSet.audioFrames) * CanvasLogicalWidth
),
end: Math.round(
(viewport.end / mediaSet.audioFrames) * CanvasLogicalWidth
),
});
}, [mediaSet, viewport]);
// load peaks on mediaset change
useEffect(() => {
@ -140,44 +135,65 @@ export const Overview: React.FC<Props> = ({
currentSelection = selection;
}
const elementWidth = canvas.getBoundingClientRect().width;
const start =
(currentSelection.start / elementWidth) * CanvasLogicalWidth;
const end = (currentSelection.end / elementWidth) * CanvasLogicalWidth;
ctx.beginPath();
ctx.strokeStyle = 'red';
ctx.lineWidth = 4;
const alpha = hoverState == HoverState.OverSelection ? '0.15' : '0.13';
ctx.fillStyle = `rgba(255, 255, 255, ${alpha})`;
ctx.rect(start, 2, end - start, canvas.height - 10);
ctx.rect(
currentSelection.start,
2,
currentSelection.end - currentSelection.start,
canvas.height - 10
);
ctx.fill();
ctx.stroke();
// draw position marker
const markerX = canvas.width * (position.percent / 100);
ctx.beginPath();
ctx.moveTo(markerX, 0);
ctx.lineWidth = 4;
ctx.lineTo(markerX, canvas.height - 4);
ctx.stroke();
});
});
}, [mediaSet, selection, newSelection, position]);
// handlers
const isHoveringSelectionStart = (elementX: number): boolean => {
return elementX > selection.start - 10 && elementX < selection.start + 10;
const hoverOffset = 10;
const isHoveringSelectionStart = (x: number): boolean => {
return (
x > selection.start - hoverOffset && x < selection.start + hoverOffset
);
};
const isHoveringSelectionEnd = (elementX: number): boolean => {
return elementX > selection.end - 10 && elementX < selection.end + 10;
const isHoveringSelectionEnd = (x: number): boolean => {
return x > selection.end - hoverOffset && x < selection.end + hoverOffset;
};
const isHoveringSelection = (elementX: number): boolean => {
return elementX >= selection.start && elementX <= selection.end;
const isHoveringSelection = (x: number): boolean => {
return x >= selection.start && x <= selection.end;
};
const getCanvasX = (evt: MouseEvent<HTMLCanvasElement>): number => {
const rect = evt.currentTarget.getBoundingClientRect();
const x = Math.round(
((evt.clientX - rect.left) / rect.width) * CanvasLogicalWidth
);
return constrainXToCanvas(x);
};
const constrainXToCanvas = (x: number): number => {
if (x < 0) {
return 0;
}
if (x > CanvasLogicalWidth) {
return CanvasLogicalWidth;
}
return x;
};
const handleMouseDown = (evt: MouseEvent<HTMLCanvasElement>) => {
@ -185,35 +201,31 @@ export const Overview: React.FC<Props> = ({
return;
}
const elementX = Math.round(
evt.clientX - evt.currentTarget.getBoundingClientRect().x
);
const x = getCanvasX(evt);
if (isHoveringSelectionStart(elementX)) {
if (isHoveringSelectionStart(x)) {
setMode(Mode.ResizingStart);
moveOffsetX.current = elementX;
moveOffsetX.current = x;
return;
} else if (isHoveringSelectionEnd(elementX)) {
} else if (isHoveringSelectionEnd(x)) {
setMode(Mode.ResizingEnd);
moveOffsetX.current = elementX;
moveOffsetX.current = x;
return;
} else if (isHoveringSelection(elementX)) {
} else if (isHoveringSelection(x)) {
setMode(Mode.Dragging);
setCursor('pointer');
moveOffsetX.current = elementX;
moveOffsetX.current = x;
return;
}
setMode(Mode.Selecting);
setCursor('col-resize');
moveOffsetX.current = elementX;
setNewSelection({ start: elementX, end: elementX });
moveOffsetX.current = x;
setNewSelection({ start: x, end: x });
};
const handleMouseMove = (evt: MouseEvent<HTMLCanvasElement>) => {
const x = Math.round(
evt.clientX - evt.currentTarget.getBoundingClientRect().x
);
const x = getCanvasX(evt);
switch (mode) {
case Mode.Normal: {
@ -233,7 +245,7 @@ export const Overview: React.FC<Props> = ({
}
case Mode.ResizingStart: {
const diff = x - moveOffsetX.current;
const start = selection.start + diff;
const start = constrainXToCanvas(selection.start + diff);
if (start > selection.end) {
setNewSelection({ start: selection.end, end: start });
@ -245,7 +257,7 @@ export const Overview: React.FC<Props> = ({
}
case Mode.ResizingEnd: {
const diff = x - moveOffsetX.current;
const start = selection.end + diff;
const start = constrainXToCanvas(selection.end + diff);
if (start < selection.start) {
setNewSelection({ start: Math.max(0, start), end: selection.start });
@ -260,8 +272,8 @@ export const Overview: React.FC<Props> = ({
const width = selection.end - selection.start;
let start = Math.max(0, selection.start + diff);
let end = start + width;
if (end > evt.currentTarget.getBoundingClientRect().width) {
end = evt.currentTarget.getBoundingClientRect().width;
if (end > CanvasLogicalWidth) {
end = CanvasLogicalWidth;
start = end - width;
}
@ -290,16 +302,13 @@ export const Overview: React.FC<Props> = ({
setMode(Mode.Normal);
setCursor('auto');
if (newSelection.start == newSelection.end) {
setSelection({ start: newSelection.start, end: newSelection.end + 5 });
return;
}
if (newSelection.start == newSelection.end) {
setSelection({ ...emptySelection });
return;
}
setSelection({ ...newSelection });
const start = Math.round(
(newSelection.start / CanvasLogicalWidth) * mediaSet.audioFrames
);
const end = Math.round(
(newSelection.end / CanvasLogicalWidth) * mediaSet.audioFrames
);
onSelectionChange({ start, end });
};
const handleMouseLeave = (_evt: MouseEvent<HTMLCanvasElement>) => {

10
frontend/src/Waveform.tsx
View File

@ -38,7 +38,7 @@ export const Waveform: React.FC<Props> = ({
return;
}
console.log('fetch audio segment...');
console.log('fetch audio segment, frames', viewport);
const service = new MediaSetServiceClientImpl(newRPC());
const segment = await service.GetAudioSegment({
@ -76,7 +76,11 @@ export const Waveform: React.FC<Props> = ({
return;
}
const x = secsToCanvasX(position.currentTime, mediaSet.audioSampleRate, viewport);
const x = secsToCanvasX(
position.currentTime,
mediaSet.audioSampleRate,
viewport
);
if (x == null) {
return;
}
@ -87,7 +91,7 @@ export const Waveform: React.FC<Props> = ({
ctx.lineWidth = 4;
ctx.lineTo(x, canvas.height);
ctx.stroke();
}, [viewport, position]);
}, [mediaSet, position]);
// render component

2
proto/media_set.proto
View File

@ -5,6 +5,8 @@ option go_package = "pb/media_set";
import "google/protobuf/duration.proto";
// TODO: use uints where appropriate.
message MediaSet {
string id = 1;
string youtube_id = 2;