diff --git a/backend/cmd/clipper/main.go b/backend/cmd/clipper/main.go index ff2925b..712a021 100644 --- a/backend/cmd/clipper/main.go +++ b/backend/cmd/clipper/main.go @@ -25,7 +25,10 @@ func main() { // Create a store databaseURL := os.Getenv("DATABASE_URL") - log.Printf("DATABASE_URL = %s", databaseURL) + if databaseURL == "" { + log.Fatal("DATABASE_URL not set") + } + db, err := sql.Open("postgres", databaseURL) if err != nil { log.Fatal(err) diff --git a/backend/media/audio_progress.go b/backend/media/audio_progress.go index f466f79..a46022c 100644 --- a/backend/media/audio_progress.go +++ b/backend/media/audio_progress.go @@ -5,6 +5,7 @@ import ( "encoding/binary" "fmt" "io" + "math" ) type GetAudioProgress struct { @@ -21,6 +22,7 @@ type GetAudioProgressReader interface { // signed int16s and, given a target number of bins, emits a stream of peaks // corresponding to each channel of the audio data. type fetchAudioProgressReader struct { + byteOrder binary.ByteOrder framesExpected int64 channels int framesPerBin int @@ -34,11 +36,12 @@ type fetchAudioProgressReader struct { } // TODO: validate inputs, debugging is confusing otherwise -func newGetAudioProgressReader(framesExpected int64, channels, numBins int) *fetchAudioProgressReader { +func newGetAudioProgressReader(byteOrder binary.ByteOrder, framesExpected int64, channels, numBins int) *fetchAudioProgressReader { return &fetchAudioProgressReader{ + byteOrder: byteOrder, channels: channels, framesExpected: framesExpected, - framesPerBin: int(framesExpected / int64(numBins)), + framesPerBin: int(math.Ceil(float64(framesExpected) / float64(numBins))), samples: make([]int16, 8_192), currPeaks: make([]int16, channels), progress: make(chan GetAudioProgress), @@ -55,6 +58,20 @@ func (w *fetchAudioProgressReader) Close() error { return nil } +func (w *fetchAudioProgressReader) Read() (GetAudioProgress, error) { + for { + select { + case progress, ok := <-w.progress: + if !ok { + return GetAudioProgress{}, io.EOF + } + return progress, nil + case err := <-w.errorChan: + return GetAudioProgress{}, fmt.Errorf("error waiting for progress: %v", err) + } + } +} + func (w *fetchAudioProgressReader) Write(p []byte) (int, error) { // expand our target slice if it is of insufficient size: numSamples := len(p) / SizeOfInt16 @@ -64,7 +81,7 @@ func (w *fetchAudioProgressReader) Write(p []byte) (int, error) { samples := w.samples[:numSamples] - if err := binary.Read(bytes.NewReader(p), binary.LittleEndian, samples); err != nil { + if err := binary.Read(bytes.NewReader(p), w.byteOrder, samples); err != nil { return 0, fmt.Errorf("error parsing samples: %v", err) } @@ -103,17 +120,3 @@ func (w *fetchAudioProgressReader) nextBin() { } w.framesProcessed++ } - -func (w *fetchAudioProgressReader) Read() (GetAudioProgress, error) { - for { - select { - case progress, ok := <-w.progress: - if !ok { - return GetAudioProgress{}, io.EOF - } - return progress, nil - case err := <-w.errorChan: - return GetAudioProgress{}, fmt.Errorf("error waiting for progress: %v", err) - } - } -} diff --git a/backend/media/service.go b/backend/media/service.go index a58d3cc..50c4cef 100644 --- a/backend/media/service.go +++ b/backend/media/service.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "database/sql" + "encoding/binary" "errors" "fmt" "io" @@ -259,6 +260,7 @@ func (s *MediaSetService) getAudioFromS3(ctx context.Context, mediaSet store.Med } fetchAudioProgressReader := newGetAudioProgressReader( + binary.BigEndian, int64(mediaSet.AudioFrames.Int64), int(mediaSet.AudioChannels), numBins, @@ -340,7 +342,7 @@ func (s *MediaSetService) getAudioFromYoutube(ctx context.Context, mediaSet stor return nil, fmt.Errorf("error creating ffmpegreader: %v", err) } - s3Key := fmt.Sprintf("media_sets/%s/audio.webm", mediaSet.ID) + s3Key := fmt.Sprintf("media_sets/%s/audio.raw", mediaSet.ID) uploader, err := newMultipartUploadWriter( ctx, s.s3, @@ -353,6 +355,7 @@ func (s *MediaSetService) getAudioFromYoutube(ctx context.Context, mediaSet stor } fetchAudioProgressReader := newGetAudioProgressReader( + binary.LittleEndian, int64(mediaSet.AudioFramesApprox), format.AudioChannels, numBins, diff --git a/frontend/src/Overview.tsx b/frontend/src/Overview.tsx index 54137d1..15026f6 100644 --- a/frontend/src/Overview.tsx +++ b/frontend/src/Overview.tsx @@ -1,13 +1,11 @@ import { useState, useEffect, useRef, MouseEvent } from 'react'; -import { - MediaSetServiceClientImpl, - MediaSet, - GetAudioProgress, -} from './generated/media_set'; +import { MediaSetServiceClientImpl, MediaSet } from './generated/media_set'; import { Frames, newRPC } from './App'; import { WaveformCanvas } from './WaveformCanvas'; import { mouseEventToCanvasX } from './Helpers'; import { secsToCanvasX } from './Helpers'; +import { from, Observable } from 'rxjs'; +import { map } from 'rxjs/operators'; interface Props { mediaSet: MediaSet; @@ -24,7 +22,7 @@ enum Mode { Dragging, } -const CanvasLogicalWidth = 2000; +const CanvasLogicalWidth = 2_000; const CanvasLogicalHeight = 500; const emptySelection = { start: 0, end: 0 }; @@ -39,7 +37,7 @@ export const Overview: React.FC = ({ onSelectionChange, }: Props) => { const hudCanvasRef = useRef(null); - const [peaks, setPeaks] = useState([[], []]); + const [peaks, setPeaks] = useState>(from([])); const [mode, setMode] = useState(Mode.Normal); const [selection, setSelection] = useState({ ...emptySelection }); const [newSelection, setNewSelection] = useState({ ...emptySelection }); @@ -54,26 +52,31 @@ export const Overview: React.FC = ({ return; } + const canvas = hudCanvasRef.current; + if (canvas == null) { + console.error('no hud canvas ref available'); + return; + } + + const ctx = canvas.getContext('2d'); + if (ctx == null) { + console.error('no hud 2d context available'); + return; + } + console.log('fetching audio...'); const service = new MediaSetServiceClientImpl(newRPC()); - const observable = service.GetAudio({ id: mediaSet.id, numBins: 2_000 }); - - console.log('calling forEach...'); - await observable.forEach((progress: GetAudioProgress) => { - console.log('got progress', progress.percentCompleted); + const audioProgressStream = service.GetAudio({ + id: mediaSet.id, + numBins: CanvasLogicalWidth, }); - console.log('done'); - - // const resp = await fetch( - // `http://localhost:8888/api/media_sets/${mediaSet.id}/peaks?start=0&end=${mediaSet.audioFrames}&bins=${CanvasLogicalWidth}` - // ); - // const peaks = await resp.json(); - // setPeaks(peaks); + const peaks = audioProgressStream.pipe(map((progress) => progress.peaks)); + setPeaks(peaks); })(); }, [mediaSet]); - // draw the overview waveform + // draw the overview HUD useEffect(() => { (async function () { const canvas = hudCanvasRef.current; @@ -229,6 +232,7 @@ export const Overview: React.FC = ({
= ({ viewport, offsetPixels, }: Props) => { - const [peaks, setPeaks] = useState([[], []]); + const [peaks, setPeaks] = useState>(from([])); const hudCanvasRef = useRef(null); // effects @@ -98,6 +99,7 @@ export const Waveform: React.FC = ({
; + channels: number; strokeStyle: string; fillStyle: string; zIndex: number; @@ -44,24 +46,25 @@ export const WaveformCanvas: React.FC = (props: Props) => { return; } - const numChannels = props.peaks.length; - const chanHeight = canvas.height / numChannels; - for (let i = 0; i < numChannels; i++) { - const yOffset = chanHeight * i; - // props.peaks[n].length must equal canvasLogicalWidth: - for (let j = 0; j < props.peaks[i].length; j++) { - const val = props.peaks[i][j]; + const chanHeight = canvas.height / props.channels; + + let frameIndex = 0; + props.peaks.forEach((peaks) => { + for (let chanIndex = 0; chanIndex < peaks.length; chanIndex++) { + const yOffset = chanHeight * chanIndex; + const val = peaks[chanIndex]; const height = Math.floor((val / maxPeakValue) * chanHeight); const y1 = (chanHeight - height) / 2 + yOffset; const y2 = y1 + height; ctx.beginPath(); ctx.globalAlpha = props.alpha; - ctx.moveTo(j, y1); - ctx.lineTo(j, y2); + ctx.moveTo(frameIndex, y1); + ctx.lineTo(frameIndex, y2); ctx.stroke(); ctx.globalAlpha = 1; } - } + frameIndex++; + }); }, [props.peaks]); const canvasStyles = {