Render WaveformCanvas via peaks
This commit is contained in:
parent
97a55632ef
commit
c1ac075a88
|
@ -25,7 +25,10 @@ func main() {
|
|||
|
||||
// Create a store
|
||||
databaseURL := os.Getenv("DATABASE_URL")
|
||||
log.Printf("DATABASE_URL = %s", databaseURL)
|
||||
if databaseURL == "" {
|
||||
log.Fatal("DATABASE_URL not set")
|
||||
}
|
||||
|
||||
db, err := sql.Open("postgres", databaseURL)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
|
|
|
@ -5,6 +5,7 @@ import (
|
|||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
)
|
||||
|
||||
type GetAudioProgress struct {
|
||||
|
@ -21,6 +22,7 @@ type GetAudioProgressReader interface {
|
|||
// signed int16s and, given a target number of bins, emits a stream of peaks
|
||||
// corresponding to each channel of the audio data.
|
||||
type fetchAudioProgressReader struct {
|
||||
byteOrder binary.ByteOrder
|
||||
framesExpected int64
|
||||
channels int
|
||||
framesPerBin int
|
||||
|
@ -34,11 +36,12 @@ type fetchAudioProgressReader struct {
|
|||
}
|
||||
|
||||
// TODO: validate inputs, debugging is confusing otherwise
|
||||
func newGetAudioProgressReader(framesExpected int64, channels, numBins int) *fetchAudioProgressReader {
|
||||
func newGetAudioProgressReader(byteOrder binary.ByteOrder, framesExpected int64, channels, numBins int) *fetchAudioProgressReader {
|
||||
return &fetchAudioProgressReader{
|
||||
byteOrder: byteOrder,
|
||||
channels: channels,
|
||||
framesExpected: framesExpected,
|
||||
framesPerBin: int(framesExpected / int64(numBins)),
|
||||
framesPerBin: int(math.Ceil(float64(framesExpected) / float64(numBins))),
|
||||
samples: make([]int16, 8_192),
|
||||
currPeaks: make([]int16, channels),
|
||||
progress: make(chan GetAudioProgress),
|
||||
|
@ -55,6 +58,20 @@ func (w *fetchAudioProgressReader) Close() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (w *fetchAudioProgressReader) Read() (GetAudioProgress, error) {
|
||||
for {
|
||||
select {
|
||||
case progress, ok := <-w.progress:
|
||||
if !ok {
|
||||
return GetAudioProgress{}, io.EOF
|
||||
}
|
||||
return progress, nil
|
||||
case err := <-w.errorChan:
|
||||
return GetAudioProgress{}, fmt.Errorf("error waiting for progress: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *fetchAudioProgressReader) Write(p []byte) (int, error) {
|
||||
// expand our target slice if it is of insufficient size:
|
||||
numSamples := len(p) / SizeOfInt16
|
||||
|
@ -64,7 +81,7 @@ func (w *fetchAudioProgressReader) Write(p []byte) (int, error) {
|
|||
|
||||
samples := w.samples[:numSamples]
|
||||
|
||||
if err := binary.Read(bytes.NewReader(p), binary.LittleEndian, samples); err != nil {
|
||||
if err := binary.Read(bytes.NewReader(p), w.byteOrder, samples); err != nil {
|
||||
return 0, fmt.Errorf("error parsing samples: %v", err)
|
||||
}
|
||||
|
||||
|
@ -103,17 +120,3 @@ func (w *fetchAudioProgressReader) nextBin() {
|
|||
}
|
||||
w.framesProcessed++
|
||||
}
|
||||
|
||||
func (w *fetchAudioProgressReader) Read() (GetAudioProgress, error) {
|
||||
for {
|
||||
select {
|
||||
case progress, ok := <-w.progress:
|
||||
if !ok {
|
||||
return GetAudioProgress{}, io.EOF
|
||||
}
|
||||
return progress, nil
|
||||
case err := <-w.errorChan:
|
||||
return GetAudioProgress{}, fmt.Errorf("error waiting for progress: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,6 +4,7 @@ import (
|
|||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
@ -259,6 +260,7 @@ func (s *MediaSetService) getAudioFromS3(ctx context.Context, mediaSet store.Med
|
|||
}
|
||||
|
||||
fetchAudioProgressReader := newGetAudioProgressReader(
|
||||
binary.BigEndian,
|
||||
int64(mediaSet.AudioFrames.Int64),
|
||||
int(mediaSet.AudioChannels),
|
||||
numBins,
|
||||
|
@ -340,7 +342,7 @@ func (s *MediaSetService) getAudioFromYoutube(ctx context.Context, mediaSet stor
|
|||
return nil, fmt.Errorf("error creating ffmpegreader: %v", err)
|
||||
}
|
||||
|
||||
s3Key := fmt.Sprintf("media_sets/%s/audio.webm", mediaSet.ID)
|
||||
s3Key := fmt.Sprintf("media_sets/%s/audio.raw", mediaSet.ID)
|
||||
uploader, err := newMultipartUploadWriter(
|
||||
ctx,
|
||||
s.s3,
|
||||
|
@ -353,6 +355,7 @@ func (s *MediaSetService) getAudioFromYoutube(ctx context.Context, mediaSet stor
|
|||
}
|
||||
|
||||
fetchAudioProgressReader := newGetAudioProgressReader(
|
||||
binary.LittleEndian,
|
||||
int64(mediaSet.AudioFramesApprox),
|
||||
format.AudioChannels,
|
||||
numBins,
|
||||
|
|
|
@ -1,13 +1,11 @@
|
|||
import { useState, useEffect, useRef, MouseEvent } from 'react';
|
||||
import {
|
||||
MediaSetServiceClientImpl,
|
||||
MediaSet,
|
||||
GetAudioProgress,
|
||||
} from './generated/media_set';
|
||||
import { MediaSetServiceClientImpl, MediaSet } from './generated/media_set';
|
||||
import { Frames, newRPC } from './App';
|
||||
import { WaveformCanvas } from './WaveformCanvas';
|
||||
import { mouseEventToCanvasX } from './Helpers';
|
||||
import { secsToCanvasX } from './Helpers';
|
||||
import { from, Observable } from 'rxjs';
|
||||
import { map } from 'rxjs/operators';
|
||||
|
||||
interface Props {
|
||||
mediaSet: MediaSet;
|
||||
|
@ -24,7 +22,7 @@ enum Mode {
|
|||
Dragging,
|
||||
}
|
||||
|
||||
const CanvasLogicalWidth = 2000;
|
||||
const CanvasLogicalWidth = 2_000;
|
||||
const CanvasLogicalHeight = 500;
|
||||
|
||||
const emptySelection = { start: 0, end: 0 };
|
||||
|
@ -39,7 +37,7 @@ export const Overview: React.FC<Props> = ({
|
|||
onSelectionChange,
|
||||
}: Props) => {
|
||||
const hudCanvasRef = useRef<HTMLCanvasElement>(null);
|
||||
const [peaks, setPeaks] = useState<number[][]>([[], []]);
|
||||
const [peaks, setPeaks] = useState<Observable<number[]>>(from([]));
|
||||
const [mode, setMode] = useState(Mode.Normal);
|
||||
const [selection, setSelection] = useState({ ...emptySelection });
|
||||
const [newSelection, setNewSelection] = useState({ ...emptySelection });
|
||||
|
@ -54,26 +52,31 @@ export const Overview: React.FC<Props> = ({
|
|||
return;
|
||||
}
|
||||
|
||||
const canvas = hudCanvasRef.current;
|
||||
if (canvas == null) {
|
||||
console.error('no hud canvas ref available');
|
||||
return;
|
||||
}
|
||||
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (ctx == null) {
|
||||
console.error('no hud 2d context available');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('fetching audio...');
|
||||
const service = new MediaSetServiceClientImpl(newRPC());
|
||||
const observable = service.GetAudio({ id: mediaSet.id, numBins: 2_000 });
|
||||
|
||||
console.log('calling forEach...');
|
||||
await observable.forEach((progress: GetAudioProgress) => {
|
||||
console.log('got progress', progress.percentCompleted);
|
||||
const audioProgressStream = service.GetAudio({
|
||||
id: mediaSet.id,
|
||||
numBins: CanvasLogicalWidth,
|
||||
});
|
||||
|
||||
console.log('done');
|
||||
|
||||
// const resp = await fetch(
|
||||
// `http://localhost:8888/api/media_sets/${mediaSet.id}/peaks?start=0&end=${mediaSet.audioFrames}&bins=${CanvasLogicalWidth}`
|
||||
// );
|
||||
// const peaks = await resp.json();
|
||||
// setPeaks(peaks);
|
||||
const peaks = audioProgressStream.pipe(map((progress) => progress.peaks));
|
||||
setPeaks(peaks);
|
||||
})();
|
||||
}, [mediaSet]);
|
||||
|
||||
// draw the overview waveform
|
||||
// draw the overview HUD
|
||||
useEffect(() => {
|
||||
(async function () {
|
||||
const canvas = hudCanvasRef.current;
|
||||
|
@ -229,6 +232,7 @@ export const Overview: React.FC<Props> = ({
|
|||
<div style={containerStyles}>
|
||||
<WaveformCanvas
|
||||
peaks={peaks}
|
||||
channels={mediaSet.audioChannels}
|
||||
width={CanvasLogicalWidth}
|
||||
height={CanvasLogicalHeight}
|
||||
strokeStyle="black"
|
||||
|
|
|
@ -3,6 +3,7 @@ import { Frames } from './App';
|
|||
import { MediaSet } from './generated/media_set';
|
||||
import { WaveformCanvas } from './WaveformCanvas';
|
||||
import { secsToCanvasX } from './Helpers';
|
||||
import { from, Observable } from 'rxjs';
|
||||
|
||||
interface Props {
|
||||
mediaSet: MediaSet;
|
||||
|
@ -20,7 +21,7 @@ export const Waveform: React.FC<Props> = ({
|
|||
viewport,
|
||||
offsetPixels,
|
||||
}: Props) => {
|
||||
const [peaks, setPeaks] = useState<number[][]>([[], []]);
|
||||
const [peaks, setPeaks] = useState<Observable<number[]>>(from([]));
|
||||
const hudCanvasRef = useRef<HTMLCanvasElement>(null);
|
||||
|
||||
// effects
|
||||
|
@ -98,6 +99,7 @@ export const Waveform: React.FC<Props> = ({
|
|||
<div style={containerStyles}>
|
||||
<WaveformCanvas
|
||||
peaks={peaks}
|
||||
channels={mediaSet.audioChannels}
|
||||
width={CanvasLogicalWidth}
|
||||
height={CanvasLogicalHeight}
|
||||
strokeStyle="green"
|
||||
|
|
|
@ -1,11 +1,13 @@
|
|||
import { useEffect, useRef } from 'react';
|
||||
import { Observable } from 'rxjs';
|
||||
|
||||
const maxPeakValue = 32_768;
|
||||
|
||||
interface Props {
|
||||
width: number;
|
||||
height: number;
|
||||
peaks: number[][] | null;
|
||||
peaks: Observable<number[]>;
|
||||
channels: number;
|
||||
strokeStyle: string;
|
||||
fillStyle: string;
|
||||
zIndex: number;
|
||||
|
@ -44,24 +46,25 @@ export const WaveformCanvas: React.FC<Props> = (props: Props) => {
|
|||
return;
|
||||
}
|
||||
|
||||
const numChannels = props.peaks.length;
|
||||
const chanHeight = canvas.height / numChannels;
|
||||
for (let i = 0; i < numChannels; i++) {
|
||||
const yOffset = chanHeight * i;
|
||||
// props.peaks[n].length must equal canvasLogicalWidth:
|
||||
for (let j = 0; j < props.peaks[i].length; j++) {
|
||||
const val = props.peaks[i][j];
|
||||
const chanHeight = canvas.height / props.channels;
|
||||
|
||||
let frameIndex = 0;
|
||||
props.peaks.forEach((peaks) => {
|
||||
for (let chanIndex = 0; chanIndex < peaks.length; chanIndex++) {
|
||||
const yOffset = chanHeight * chanIndex;
|
||||
const val = peaks[chanIndex];
|
||||
const height = Math.floor((val / maxPeakValue) * chanHeight);
|
||||
const y1 = (chanHeight - height) / 2 + yOffset;
|
||||
const y2 = y1 + height;
|
||||
ctx.beginPath();
|
||||
ctx.globalAlpha = props.alpha;
|
||||
ctx.moveTo(j, y1);
|
||||
ctx.lineTo(j, y2);
|
||||
ctx.moveTo(frameIndex, y1);
|
||||
ctx.lineTo(frameIndex, y2);
|
||||
ctx.stroke();
|
||||
ctx.globalAlpha = 1;
|
||||
}
|
||||
}
|
||||
frameIndex++;
|
||||
});
|
||||
}, [props.peaks]);
|
||||
|
||||
const canvasStyles = {
|
||||
|
|
Loading…
Reference in New Issue