From 0cc1fd8272a77f7defcc4ab6db0769ab5b3897ce Mon Sep 17 00:00:00 2001 From: Rob Watson Date: Tue, 30 Nov 2021 20:41:34 +0100 Subject: [PATCH] Frontend fixes --- backend/media/service.go | 5 ++ deploy.sh | 2 + frontend/src/App.tsx | 85 ++++++++++++----------- frontend/src/Helpers.tsx | 2 +- frontend/src/Overview.tsx | 137 ++++++++++++++++++++------------------ frontend/src/Waveform.tsx | 10 ++- proto/media_set.proto | 2 + 7 files changed, 136 insertions(+), 107 deletions(-) diff --git a/backend/media/service.go b/backend/media/service.go index fa6d8f9..5e6e241 100644 --- a/backend/media/service.go +++ b/backend/media/service.go @@ -371,6 +371,11 @@ outer: } func (s *MediaSetService) GetAudioSegment(ctx context.Context, id uuid.UUID, startFrame, endFrame int64, numBins int) ([]int16, error) { + if startFrame < 0 || endFrame < 0 || numBins <= 0 { + s.logger.With("startFrame", startFrame, "endFrame", endFrame, "numBins", numBins).Error("invalid arguments") + return nil, errors.New("invalid arguments") + } + mediaSet, err := s.store.GetMediaSet(ctx, id) if err != nil { return nil, fmt.Errorf("error getting media set: %v", err) diff --git a/deploy.sh b/deploy.sh index 3dcda5a..d2169d4 100755 --- a/deploy.sh +++ b/deploy.sh @@ -15,6 +15,8 @@ # TODO: production build set -ex +export DOCKER_BUILDKIT=1 + docker build \ -t netfluxio/clipper-staging:latest \ --build-arg API_URL=https://clipper-staging.netflux.io \ diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 88825c7..168fbcb 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -6,7 +6,7 @@ import { GetAudioProgress, } from './generated/media_set'; -import { useState, useEffect } from 'react'; +import { useState, useEffect, useRef } from 'react'; import { VideoPreview } from './VideoPreview'; import { Overview, CanvasLogicalWidth } from './Overview'; import { Waveform } from './Waveform'; @@ -14,8 +14,8 @@ import { ControlBar } from './ControlBar'; import { SeekBar } from './SeekBar'; import './App.css'; import { Duration } from './generated/google/protobuf/duration'; -import { from, Observable } from 'rxjs'; -import { map } from 'rxjs/operators'; +import { firstValueFrom, from, Observable } from 'rxjs'; +import { first, map } from 'rxjs/operators'; // ported from backend, where should they live? const thumbnailWidth = 177; @@ -36,16 +36,22 @@ export interface VideoPosition { percent: number; } +const video = document.createElement('video'); +const audio = document.createElement('audio'); + function App(): JSX.Element { const [mediaSet, setMediaSet] = useState(null); - const [video, _setVideo] = useState(document.createElement('video')); - const [audio, _setAudio] = useState(document.createElement('audio')); - const [position, setPosition] = useState({ currentTime: 0, percent: 0 }); const [viewport, setViewport] = useState({ start: 0, end: 0 }); const [overviewPeaks, setOverviewPeaks] = useState>( from([]) ); + // position stores the current playback position. positionRef makes it + // available inside a setInterval callback. + const [position, setPosition] = useState({ currentTime: 0, percent: 0 }); + const positionRef = useRef(position); + positionRef.current = position; + // effects // TODO: error handling @@ -66,22 +72,25 @@ function App(): JSX.Element { })(); }, []); + const updatePlayerPositionIntevalMillis = 30; + // setup player on first page load only: useEffect(() => { if (mediaSet == null) { return; } - // assume mediaSet never changes once loaded - setInterval(() => { - if (video.currentTime == position.currentTime) { + const intervalID = setInterval(() => { + if (video.currentTime == positionRef.current.currentTime) { return; } const duration = mediaSet.audioFrames / mediaSet.audioSampleRate; const percent = (video.currentTime / duration) * 100; setPosition({ currentTime: video.currentTime, percent: percent }); - }, 100); + }, updatePlayerPositionIntevalMillis); + + return () => clearInterval(intervalID); }, [mediaSet]); // load audio when MediaSet is loaded: @@ -91,7 +100,6 @@ function App(): JSX.Element { return; } console.log('fetching audio...'); - // TODO move this call to app.tsx, pass the stream in as a prop. const service = new MediaSetServiceClientImpl(newRPC()); const audioProgressStream = service.GetAudio({ id: mediaSet.id, @@ -100,18 +108,15 @@ function App(): JSX.Element { const peaks = audioProgressStream.pipe(map((progress) => progress.peaks)); setOverviewPeaks(peaks); - let url = ''; - // TODO: probably a nicer way to do this. - await audioProgressStream.forEach((progress: GetAudioProgress) => { - if (progress.url != '') { - url = progress.url; - } - }); + const pipe = audioProgressStream.pipe( + first((progress: GetAudioProgress) => progress.url != '') + ); + const progressWithURL = await firstValueFrom(pipe); - audio.src = url; + audio.src = progressWithURL.url; audio.muted = false; audio.volume = 1; - console.log('got audio URL', url); + console.log('set audio src', progressWithURL.url); })(); }, [mediaSet]); @@ -122,21 +127,16 @@ function App(): JSX.Element { return; } - console.log('getting video...'); - const rpc = newRPC(); - const service = new MediaSetServiceClientImpl(rpc); + console.log('fetching video...'); + const service = new MediaSetServiceClientImpl(newRPC()); const videoProgressStream = service.GetVideo({ id: mediaSet.id }); + const pipe = videoProgressStream.pipe( + first((progress: GetVideoProgress) => progress.url != '') + ); + const progressWithURL = await firstValueFrom(pipe); - let url = ''; - // TODO: probably a nicer way to do this. - await videoProgressStream.forEach((progress: GetVideoProgress) => { - if (progress.url != '') { - url = progress.url; - } - }); - - video.src = url; - console.log('set video src', video.src); + video.src = progressWithURL.url; + console.log('set video src', progressWithURL.url); })(); }, [mediaSet]); @@ -160,16 +160,22 @@ function App(): JSX.Element { // handlers - const handleOverviewSelectionChange = (selection: Frames) => { - console.log('in handleOverviewSelectionChange', selection); + const handleOverviewSelectionChange = (newViewport: Frames) => { if (mediaSet == null) { return; } + console.log('set new viewport', newViewport); + setViewport({ ...newViewport }); - setViewport({ - start: mediaSet.audioFrames * (selection.start / 100), - end: mediaSet.audioFrames * (selection.end / 100), - }); + if (!audio.paused) { + return; + } + + const ratio = newViewport.start / mediaSet.audioFrames; + const currentTime = + (mediaSet.audioFrames / mediaSet.audioSampleRate) * ratio; + audio.currentTime = currentTime; + video.currentTime = currentTime; }; // render component @@ -211,6 +217,7 @@ function App(): JSX.Element { mediaSet={mediaSet} offsetPixels={offsetPixels} height={80} + viewport={viewport} position={position} onSelectionChange={handleOverviewSelectionChange} /> diff --git a/frontend/src/Helpers.tsx b/frontend/src/Helpers.tsx index d5ad19e..2eb2cd6 100644 --- a/frontend/src/Helpers.tsx +++ b/frontend/src/Helpers.tsx @@ -45,7 +45,7 @@ export const secsToCanvasX = ( sampleRate: number, viewport: Frames ): number | null => { - const frame = Math.floor(secs * sampleRate); + const frame = Math.round(secs * sampleRate); if (frame < viewport.start || frame > viewport.end) { return null; } diff --git a/frontend/src/Overview.tsx b/frontend/src/Overview.tsx index e246add..61dd332 100644 --- a/frontend/src/Overview.tsx +++ b/frontend/src/Overview.tsx @@ -15,6 +15,7 @@ interface Props { height: number; offsetPixels: number; position: VideoPosition; + viewport: Frames; onSelectionChange: (selection: Frames) => void; } @@ -44,20 +45,23 @@ export const Overview: React.FC = ({ height, offsetPixels, position, + viewport, onSelectionChange, }: Props) => { const hudCanvasRef = useRef(null); const [mode, setMode] = useState(Mode.Normal); const [hoverState, setHoverState] = useState(HoverState.Normal); + const [cursor, setCursor] = useState('auto'); + + // selection and newSelection relate to canvas logical pixels: + const [selection, setSelection] = useState({ ...emptySelection }); const [newSelection, setNewSelection] = useState({ ...emptySelection, }); - const [selection, setSelection] = useState({ start: 0, end: 100 }); - const [cursor, setCursor] = useState('auto'); const moveOffsetX = useRef(0); - // effects + // side effects // handle global mouse up. useEffect(() => { @@ -67,30 +71,21 @@ export const Overview: React.FC = ({ }; }, [mode, newSelection]); - // publish onSelectionChange event + // set selection state on viewport change useEffect(() => { if (mediaSet == null) { return; } - const canvas = hudCanvasRef.current; - if (canvas == null) { - console.error('no hud canvas ref available'); - return; - } - const ctx = canvas.getContext('2d'); - if (ctx == null) { - console.error('no hud 2d context available'); - return; - } - const width = canvas.getBoundingClientRect().width; - const selectionPercent = { - start: (selection.start / width) * 100, - end: (selection.end / width) * 100, - }; - - onSelectionChange(selectionPercent); - }, [selection]); + setSelection({ + start: Math.round( + (viewport.start / mediaSet.audioFrames) * CanvasLogicalWidth + ), + end: Math.round( + (viewport.end / mediaSet.audioFrames) * CanvasLogicalWidth + ), + }); + }, [mediaSet, viewport]); // load peaks on mediaset change useEffect(() => { @@ -140,44 +135,65 @@ export const Overview: React.FC = ({ currentSelection = selection; } - const elementWidth = canvas.getBoundingClientRect().width; - const start = - (currentSelection.start / elementWidth) * CanvasLogicalWidth; - const end = (currentSelection.end / elementWidth) * CanvasLogicalWidth; - ctx.beginPath(); ctx.strokeStyle = 'red'; ctx.lineWidth = 4; const alpha = hoverState == HoverState.OverSelection ? '0.15' : '0.13'; ctx.fillStyle = `rgba(255, 255, 255, ${alpha})`; - ctx.rect(start, 2, end - start, canvas.height - 10); + ctx.rect( + currentSelection.start, + 2, + currentSelection.end - currentSelection.start, + canvas.height - 10 + ); ctx.fill(); ctx.stroke(); // draw position marker const markerX = canvas.width * (position.percent / 100); - ctx.beginPath(); ctx.moveTo(markerX, 0); ctx.lineWidth = 4; ctx.lineTo(markerX, canvas.height - 4); ctx.stroke(); }); - }); + }, [mediaSet, selection, newSelection, position]); // handlers - const isHoveringSelectionStart = (elementX: number): boolean => { - return elementX > selection.start - 10 && elementX < selection.start + 10; + const hoverOffset = 10; + + const isHoveringSelectionStart = (x: number): boolean => { + return ( + x > selection.start - hoverOffset && x < selection.start + hoverOffset + ); }; - const isHoveringSelectionEnd = (elementX: number): boolean => { - return elementX > selection.end - 10 && elementX < selection.end + 10; + const isHoveringSelectionEnd = (x: number): boolean => { + return x > selection.end - hoverOffset && x < selection.end + hoverOffset; }; - const isHoveringSelection = (elementX: number): boolean => { - return elementX >= selection.start && elementX <= selection.end; + const isHoveringSelection = (x: number): boolean => { + return x >= selection.start && x <= selection.end; + }; + + const getCanvasX = (evt: MouseEvent): number => { + const rect = evt.currentTarget.getBoundingClientRect(); + const x = Math.round( + ((evt.clientX - rect.left) / rect.width) * CanvasLogicalWidth + ); + return constrainXToCanvas(x); + }; + + const constrainXToCanvas = (x: number): number => { + if (x < 0) { + return 0; + } + if (x > CanvasLogicalWidth) { + return CanvasLogicalWidth; + } + return x; }; const handleMouseDown = (evt: MouseEvent) => { @@ -185,35 +201,31 @@ export const Overview: React.FC = ({ return; } - const elementX = Math.round( - evt.clientX - evt.currentTarget.getBoundingClientRect().x - ); + const x = getCanvasX(evt); - if (isHoveringSelectionStart(elementX)) { + if (isHoveringSelectionStart(x)) { setMode(Mode.ResizingStart); - moveOffsetX.current = elementX; + moveOffsetX.current = x; return; - } else if (isHoveringSelectionEnd(elementX)) { + } else if (isHoveringSelectionEnd(x)) { setMode(Mode.ResizingEnd); - moveOffsetX.current = elementX; + moveOffsetX.current = x; return; - } else if (isHoveringSelection(elementX)) { + } else if (isHoveringSelection(x)) { setMode(Mode.Dragging); setCursor('pointer'); - moveOffsetX.current = elementX; + moveOffsetX.current = x; return; } setMode(Mode.Selecting); setCursor('col-resize'); - moveOffsetX.current = elementX; - setNewSelection({ start: elementX, end: elementX }); + moveOffsetX.current = x; + setNewSelection({ start: x, end: x }); }; const handleMouseMove = (evt: MouseEvent) => { - const x = Math.round( - evt.clientX - evt.currentTarget.getBoundingClientRect().x - ); + const x = getCanvasX(evt); switch (mode) { case Mode.Normal: { @@ -233,7 +245,7 @@ export const Overview: React.FC = ({ } case Mode.ResizingStart: { const diff = x - moveOffsetX.current; - const start = selection.start + diff; + const start = constrainXToCanvas(selection.start + diff); if (start > selection.end) { setNewSelection({ start: selection.end, end: start }); @@ -245,7 +257,7 @@ export const Overview: React.FC = ({ } case Mode.ResizingEnd: { const diff = x - moveOffsetX.current; - const start = selection.end + diff; + const start = constrainXToCanvas(selection.end + diff); if (start < selection.start) { setNewSelection({ start: Math.max(0, start), end: selection.start }); @@ -260,8 +272,8 @@ export const Overview: React.FC = ({ const width = selection.end - selection.start; let start = Math.max(0, selection.start + diff); let end = start + width; - if (end > evt.currentTarget.getBoundingClientRect().width) { - end = evt.currentTarget.getBoundingClientRect().width; + if (end > CanvasLogicalWidth) { + end = CanvasLogicalWidth; start = end - width; } @@ -290,16 +302,13 @@ export const Overview: React.FC = ({ setMode(Mode.Normal); setCursor('auto'); - if (newSelection.start == newSelection.end) { - setSelection({ start: newSelection.start, end: newSelection.end + 5 }); - return; - } - - if (newSelection.start == newSelection.end) { - setSelection({ ...emptySelection }); - return; - } - setSelection({ ...newSelection }); + const start = Math.round( + (newSelection.start / CanvasLogicalWidth) * mediaSet.audioFrames + ); + const end = Math.round( + (newSelection.end / CanvasLogicalWidth) * mediaSet.audioFrames + ); + onSelectionChange({ start, end }); }; const handleMouseLeave = (_evt: MouseEvent) => { diff --git a/frontend/src/Waveform.tsx b/frontend/src/Waveform.tsx index be2c3ad..e05b14c 100644 --- a/frontend/src/Waveform.tsx +++ b/frontend/src/Waveform.tsx @@ -38,7 +38,7 @@ export const Waveform: React.FC = ({ return; } - console.log('fetch audio segment...'); + console.log('fetch audio segment, frames', viewport); const service = new MediaSetServiceClientImpl(newRPC()); const segment = await service.GetAudioSegment({ @@ -76,7 +76,11 @@ export const Waveform: React.FC = ({ return; } - const x = secsToCanvasX(position.currentTime, mediaSet.audioSampleRate, viewport); + const x = secsToCanvasX( + position.currentTime, + mediaSet.audioSampleRate, + viewport + ); if (x == null) { return; } @@ -87,7 +91,7 @@ export const Waveform: React.FC = ({ ctx.lineWidth = 4; ctx.lineTo(x, canvas.height); ctx.stroke(); - }, [viewport, position]); + }, [mediaSet, position]); // render component diff --git a/proto/media_set.proto b/proto/media_set.proto index a7754d9..c165020 100644 --- a/proto/media_set.proto +++ b/proto/media_set.proto @@ -5,6 +5,8 @@ option go_package = "pb/media_set"; import "google/protobuf/duration.proto"; +// TODO: use uints where appropriate. + message MediaSet { string id = 1; string youtube_id = 2;