rob
/
clipper
2
0
Fork 0
Code Issues 11 Pull Requests 1 Projects Releases Wiki Activity

Render WaveformCanvas via peaks

This commit is contained in:
Rob Watson 2021-11-06 21:52:47 +01:00
parent 97a55632ef
commit c1ac075a88
6 changed files with 69 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
backend/cmd/clipper/main.go
View File

@ -25,7 +25,10 @@ func main() {
// Create a store
databaseURL := os.Getenv("DATABASE_URL")
log.Printf("DATABASE_URL = %s", databaseURL)
if databaseURL == "" {
log.Fatal("DATABASE_URL not set")
}
db, err := sql.Open("postgres", databaseURL)
if err != nil {
log.Fatal(err)

37
backend/media/audio_progress.go
View File

@ -5,6 +5,7 @@ import (
"encoding/binary"
"fmt"
"io"
"math"
)
type GetAudioProgress struct {
@ -21,6 +22,7 @@ type GetAudioProgressReader interface {
// signed int16s and, given a target number of bins, emits a stream of peaks
// corresponding to each channel of the audio data.
type fetchAudioProgressReader struct {
byteOrder binary.ByteOrder
framesExpected int64
channels int
framesPerBin int
@ -34,11 +36,12 @@ type fetchAudioProgressReader struct {
}
// TODO: validate inputs, debugging is confusing otherwise
func newGetAudioProgressReader(framesExpected int64, channels, numBins int) *fetchAudioProgressReader {
func newGetAudioProgressReader(byteOrder binary.ByteOrder, framesExpected int64, channels, numBins int) *fetchAudioProgressReader {
return &fetchAudioProgressReader{
byteOrder: byteOrder,
channels: channels,
framesExpected: framesExpected,
framesPerBin: int(framesExpected / int64(numBins)),
framesPerBin: int(math.Ceil(float64(framesExpected) / float64(numBins))),
samples: make([]int16, 8_192),
currPeaks: make([]int16, channels),
progress: make(chan GetAudioProgress),
@ -55,6 +58,20 @@ func (w *fetchAudioProgressReader) Close() error {
return nil
}
func (w *fetchAudioProgressReader) Read() (GetAudioProgress, error) {
for {
select {
case progress, ok := <-w.progress:
if !ok {
return GetAudioProgress{}, io.EOF
}
return progress, nil
case err := <-w.errorChan:
return GetAudioProgress{}, fmt.Errorf("error waiting for progress: %v", err)
}
}
}
func (w *fetchAudioProgressReader) Write(p []byte) (int, error) {
// expand our target slice if it is of insufficient size:
numSamples := len(p) / SizeOfInt16
@ -64,7 +81,7 @@ func (w *fetchAudioProgressReader) Write(p []byte) (int, error) {
samples := w.samples[:numSamples]
if err := binary.Read(bytes.NewReader(p), binary.LittleEndian, samples); err != nil {
if err := binary.Read(bytes.NewReader(p), w.byteOrder, samples); err != nil {
return 0, fmt.Errorf("error parsing samples: %v", err)
}
@ -103,17 +120,3 @@ func (w *fetchAudioProgressReader) nextBin() {
}
w.framesProcessed++
}
func (w *fetchAudioProgressReader) Read() (GetAudioProgress, error) {
for {
select {
case progress, ok := <-w.progress:
if !ok {
return GetAudioProgress{}, io.EOF
}
return progress, nil
case err := <-w.errorChan:
return GetAudioProgress{}, fmt.Errorf("error waiting for progress: %v", err)
}
}
}

5
backend/media/service.go
View File

@ -4,6 +4,7 @@ import (
"bytes"
"context"
"database/sql"
"encoding/binary"
"errors"
"fmt"
"io"
@ -259,6 +260,7 @@ func (s *MediaSetService) getAudioFromS3(ctx context.Context, mediaSet store.Med
}
fetchAudioProgressReader := newGetAudioProgressReader(
binary.BigEndian,
int64(mediaSet.AudioFrames.Int64),
int(mediaSet.AudioChannels),
numBins,
@ -340,7 +342,7 @@ func (s *MediaSetService) getAudioFromYoutube(ctx context.Context, mediaSet stor
return nil, fmt.Errorf("error creating ffmpegreader: %v", err)
}
s3Key := fmt.Sprintf("media_sets/%s/audio.webm", mediaSet.ID)
s3Key := fmt.Sprintf("media_sets/%s/audio.raw", mediaSet.ID)
uploader, err := newMultipartUploadWriter(
ctx,
s.s3,
@ -353,6 +355,7 @@ func (s *MediaSetService) getAudioFromYoutube(ctx context.Context, mediaSet stor
}
fetchAudioProgressReader := newGetAudioProgressReader(
binary.LittleEndian,
int64(mediaSet.AudioFramesApprox),
format.AudioChannels,
numBins,

44
frontend/src/Overview.tsx
View File

@ -1,13 +1,11 @@
import { useState, useEffect, useRef, MouseEvent } from 'react';
import {
MediaSetServiceClientImpl,
MediaSet,
GetAudioProgress,
} from './generated/media_set';
import { MediaSetServiceClientImpl, MediaSet } from './generated/media_set';
import { Frames, newRPC } from './App';
import { WaveformCanvas } from './WaveformCanvas';
import { mouseEventToCanvasX } from './Helpers';
import { secsToCanvasX } from './Helpers';
import { from, Observable } from 'rxjs';
import { map } from 'rxjs/operators';
interface Props {
mediaSet: MediaSet;
@ -24,7 +22,7 @@ enum Mode {
Dragging,
}
const CanvasLogicalWidth = 2000;
const CanvasLogicalWidth = 2_000;
const CanvasLogicalHeight = 500;
const emptySelection = { start: 0, end: 0 };
@ -39,7 +37,7 @@ export const Overview: React.FC<Props> = ({
onSelectionChange,
}: Props) => {
const hudCanvasRef = useRef<HTMLCanvasElement>(null);
const [peaks, setPeaks] = useState<number[][]>([[], []]);
const [peaks, setPeaks] = useState<Observable<number[]>>(from([]));
const [mode, setMode] = useState(Mode.Normal);
const [selection, setSelection] = useState({ ...emptySelection });
const [newSelection, setNewSelection] = useState({ ...emptySelection });
@ -54,26 +52,31 @@ export const Overview: React.FC<Props> = ({
return;
}
const canvas = hudCanvasRef.current;
if (canvas == null) {
console.error('no hud canvas ref available');
return;
}
const ctx = canvas.getContext('2d');
if (ctx == null) {
console.error('no hud 2d context available');
return;
}
console.log('fetching audio...');
const service = new MediaSetServiceClientImpl(newRPC());
const observable = service.GetAudio({ id: mediaSet.id, numBins: 2_000 });
console.log('calling forEach...');
await observable.forEach((progress: GetAudioProgress) => {
console.log('got progress', progress.percentCompleted);
const audioProgressStream = service.GetAudio({
id: mediaSet.id,
numBins: CanvasLogicalWidth,
});
console.log('done');
// const resp = await fetch(
// `http://localhost:8888/api/media_sets/${mediaSet.id}/peaks?start=0&end=${mediaSet.audioFrames}&bins=${CanvasLogicalWidth}`
// );
// const peaks = await resp.json();
// setPeaks(peaks);
const peaks = audioProgressStream.pipe(map((progress) => progress.peaks));
setPeaks(peaks);
})();
}, [mediaSet]);
// draw the overview waveform
// draw the overview HUD
useEffect(() => {
(async function () {
const canvas = hudCanvasRef.current;
@ -229,6 +232,7 @@ export const Overview: React.FC<Props> = ({
<div style={containerStyles}>
<WaveformCanvas
peaks={peaks}
channels={mediaSet.audioChannels}
width={CanvasLogicalWidth}
height={CanvasLogicalHeight}
strokeStyle="black"

4
frontend/src/Waveform.tsx
View File

@ -3,6 +3,7 @@ import { Frames } from './App';
import { MediaSet } from './generated/media_set';
import { WaveformCanvas } from './WaveformCanvas';
import { secsToCanvasX } from './Helpers';
import { from, Observable } from 'rxjs';
interface Props {
mediaSet: MediaSet;
@ -20,7 +21,7 @@ export const Waveform: React.FC<Props> = ({
viewport,
offsetPixels,
}: Props) => {
const [peaks, setPeaks] = useState<number[][]>([[], []]);
const [peaks, setPeaks] = useState<Observable<number[]>>(from([]));
const hudCanvasRef = useRef<HTMLCanvasElement>(null);
// effects
@ -98,6 +99,7 @@ export const Waveform: React.FC<Props> = ({
<div style={containerStyles}>
<WaveformCanvas
peaks={peaks}
channels={mediaSet.audioChannels}
width={CanvasLogicalWidth}
height={CanvasLogicalHeight}
strokeStyle="green"

25
frontend/src/WaveformCanvas.tsx
View File

@ -1,11 +1,13 @@
import { useEffect, useRef } from 'react';
import { Observable } from 'rxjs';
const maxPeakValue = 32_768;
interface Props {
width: number;
height: number;
peaks: number[][] | null;
peaks: Observable<number[]>;
channels: number;
strokeStyle: string;
fillStyle: string;
zIndex: number;
@ -44,24 +46,25 @@ export const WaveformCanvas: React.FC<Props> = (props: Props) => {
return;
}
const numChannels = props.peaks.length;
const chanHeight = canvas.height / numChannels;
for (let i = 0; i < numChannels; i++) {
const yOffset = chanHeight * i;
// props.peaks[n].length must equal canvasLogicalWidth:
for (let j = 0; j < props.peaks[i].length; j++) {
const val = props.peaks[i][j];
const chanHeight = canvas.height / props.channels;
let frameIndex = 0;
props.peaks.forEach((peaks) => {
for (let chanIndex = 0; chanIndex < peaks.length; chanIndex++) {
const yOffset = chanHeight * chanIndex;
const val = peaks[chanIndex];
const height = Math.floor((val / maxPeakValue) * chanHeight);
const y1 = (chanHeight - height) / 2 + yOffset;
const y2 = y1 + height;
ctx.beginPath();
ctx.globalAlpha = props.alpha;
ctx.moveTo(j, y1);
ctx.lineTo(j, y2);
ctx.moveTo(frameIndex, y1);
ctx.lineTo(frameIndex, y2);
ctx.stroke();
ctx.globalAlpha = 1;
}
}
frameIndex++;
});
}, [props.peaks]);
const canvasStyles = {