From 3f6ea4238077c675278bdf86352a07e9f2624ecc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 22 May 2023 13:30:46 -0700 Subject: [PATCH] Fixed audio. --- web/src/index.html | 7 +- web/src/index.ts | 2 +- web/src/player/audio.ts | 68 ++++++----- web/src/player/index.ts | 43 ++++--- web/src/player/message.ts | 13 +- web/src/player/renderer.ts | 129 ++------------------ web/src/player/ring.ts | 241 +++++++++++++++++++------------------ web/src/player/video.ts | 47 +++++--- web/src/player/worker.ts | 3 + web/src/player/worklet.ts | 16 ++- 10 files changed, 239 insertions(+), 330 deletions(-) diff --git a/web/src/index.html b/web/src/index.html index aeda194..74179a8 100644 --- a/web/src/index.html +++ b/web/src/index.html @@ -2,7 +2,7 @@ - + WARP @@ -11,7 +11,7 @@
-
click for audio
+
click to play
@@ -31,4 +31,5 @@ - + + \ No newline at end of file diff --git a/web/src/index.ts b/web/src/index.ts index 5c46360..053f819 100644 --- a/web/src/index.ts +++ b/web/src/index.ts @@ -31,7 +31,7 @@ const player = new Player({ const play = document.querySelector("#screen #play")! const playFunc = (e: Event) => { - player.play({}) + player.play() e.preventDefault() play.removeEventListener('click', playFunc) diff --git a/web/src/player/audio.ts b/web/src/player/audio.ts index 361036d..cbfcd3e 100644 --- a/web/src/player/audio.ts +++ b/web/src/player/audio.ts @@ -2,24 +2,17 @@ import * as Message from "./message"; import { Ring } from "./ring" export default class Audio { - ring: Ring; + ring?: Ring; queue: Array; - sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0, in microseconds + render?: number; // non-zero if requestAnimationFrame has been called last?: number; // the timestamp of the last rendered frame, in microseconds - constructor(config: Message.AudioConfig) { - this.ring = new Ring(config.ring); - this.queue = []; + constructor(config: Message.Config) { + this.queue = [] } push(frame: AudioData) { - if (!this.sync) { - // Save the frame as the sync point - // TODO sync with video - this.sync = 1000 * performance.now() - frame.timestamp - } - // Drop any old frames if (this.last && frame.timestamp <= this.last) { frame.close() @@ -27,7 +20,7 @@ export default class Audio { } // Insert the frame into the queue sorted by timestamp. - if (this.queue.length > 0 && this.queue[this.queue.length-1].timestamp <= frame.timestamp) { + if (this.queue.length > 0 && this.queue[this.queue.length - 1].timestamp <= frame.timestamp) { // Fast path because we normally append to the end. this.queue.push(frame) } else { @@ -43,33 +36,44 @@ export default class Audio { this.queue.splice(low, 0, frame) } + + this.emit() } + emit() { + const ring = this.ring + if (!ring) { + return + } - draw() { - // Convert to microseconds - const now = 1000 * performance.now(); - - // Determine the target timestamp. - const target = now - this.sync! - - // Check if we should skip some frames while (this.queue.length) { - const next = this.queue[0] - - if (next.timestamp > target) { - const ok = this.ring.write(next) - if (!ok) { - console.warn("ring buffer is full") - // No more space in the ring - break - } - } else { - console.warn("dropping audio") + let frame = this.queue[0]; + if (ring.size() + frame.numberOfFrames > ring.capacity) { + // Buffer is full + break } - next.close() + const size = ring.write(frame) + if (size < frame.numberOfFrames) { + throw new Error("audio buffer is full") + } + + this.last = frame.timestamp + + frame.close() this.queue.shift() } } + + play(play: Message.Play) { + this.ring = new Ring(play.buffer) + + if (!this.render) { + const sampleRate = 44100 // TODO dynamic + + // Refresh every half buffer + const refresh = play.buffer.capacity / sampleRate * 1000 / 2 + this.render = setInterval(this.emit.bind(this), refresh) + } + } } \ No newline at end of file diff --git a/web/src/player/index.ts b/web/src/player/index.ts index 1ef73b2..a539167 100644 --- a/web/src/player/index.ts +++ b/web/src/player/index.ts @@ -19,26 +19,16 @@ export default class Player { this.transport = config.transport this.transport.callback = this; - const video = { - canvas: config.canvas, - }; - - // Assume 44.1kHz and two audio channels - const audio = { - sampleRate: 44100, - ring: new Ring.Buffer(2, 4410), // 100ms at 44.1khz - } - this.context = new AudioContext({ latencyHint: "interactive", - sampleRate: audio.sampleRate, + sampleRate: 44100, }) - this.worker = this.setupWorker({ audio, video }) - this.worklet = this.setupWorklet(audio) + this.worker = this.setupWorker(config) + this.worklet = this.setupWorklet(config) } - private setupWorker(config: Message.Config): Worker { + private setupWorker(config: Config): Worker { const url = new URL('worker.ts', import.meta.url) const worker = new Worker(url, { @@ -46,12 +36,16 @@ export default class Player { name: "media", }) - worker.postMessage({ config }, [ config.video.canvas ]) + const msg = { + canvas: config.canvas, + } + + worker.postMessage({ config: msg }, [msg.canvas]) return worker } - private async setupWorklet(config: Message.AudioConfig): Promise { + private async setupWorklet(config: Config): Promise { // Load the worklet source code. const url = new URL('worklet.ts', import.meta.url) await this.context.audioWorklet.addModule(url) @@ -65,8 +59,6 @@ export default class Player { console.error("Audio worklet error:", e) }; - worklet.port.postMessage({ config }) - // Connect the worklet to the volume node and then to the speakers worklet.connect(volume) volume.connect(this.context.destination) @@ -75,15 +67,22 @@ export default class Player { } onInit(init: Message.Init) { - this.worker.postMessage({ init }, [ init.buffer.buffer, init.reader ]) + this.worker.postMessage({ init }, [init.buffer.buffer, init.reader]) } onSegment(segment: Message.Segment) { - this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ]) + this.worker.postMessage({ segment }, [segment.buffer.buffer, segment.reader]) } - play(play: Message.Play) { + async play() { this.context.resume() - //this.worker.postMessage({ play }) + + const play = { + buffer: new Ring.Buffer(2, 44100 / 10), // 100ms of audio + } + + const worklet = await this.worklet; + worklet.port.postMessage({ play }) + this.worker.postMessage({ play }) } } \ No newline at end of file diff --git a/web/src/player/message.ts b/web/src/player/message.ts index dcad2e0..0f4b00b 100644 --- a/web/src/player/message.ts +++ b/web/src/player/message.ts @@ -1,20 +1,10 @@ import * as Ring from "./ring" export interface Config { - audio: AudioConfig; - video: VideoConfig; -} - -export interface VideoConfig { + // video stuff canvas: OffscreenCanvas; } -export interface AudioConfig { - // audio stuff - sampleRate: number; - ring: Ring.Buffer; -} - export interface Init { buffer: Uint8Array; // unread buffered data reader: ReadableStream; // unread unbuffered data @@ -27,4 +17,5 @@ export interface Segment { export interface Play { timestamp?: number; + buffer: Ring.Buffer; } \ No newline at end of file diff --git a/web/src/player/renderer.ts b/web/src/player/renderer.ts index d2d6c86..3da2e85 100644 --- a/web/src/player/renderer.ts +++ b/web/src/player/renderer.ts @@ -1,136 +1,29 @@ import * as Message from "./message"; -import { Ring } from "./ring" +import Audio from "./audio" +import Video from "./video" export default class Renderer { - audioRing: Ring; - audioQueue: Array; - - videoCanvas: OffscreenCanvas; - videoQueue: Array; - - render: number; // non-zero if requestAnimationFrame has been called - sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0, in microseconds - last?: number; // the timestamp of the last rendered frame, in microseconds + audio: Audio; + video: Video; constructor(config: Message.Config) { - this.audioRing = new Ring(config.audio.ring); - this.audioQueue = []; - - this.videoCanvas = config.video.canvas; - this.videoQueue = []; - - this.render = 0; + this.audio = new Audio(config); + this.video = new Video(config); } push(frame: AudioData | VideoFrame) { - if (!this.sync) { - // Save the frame as the sync point - this.sync = 1000 * performance.now() - frame.timestamp - } - - // Drop any old frames - if (this.last && frame.timestamp <= this.last) { - frame.close() - return - } - - let queue if (isAudioData(frame)) { - queue = this.audioQueue; + this.audio.push(frame); } else if (isVideoFrame(frame)) { - queue = this.videoQueue; + this.video.push(frame); } else { throw new Error("unknown frame type") } - - // Insert the frame into the queue sorted by timestamp. - if (queue.length > 0 && queue[queue.length-1].timestamp <= frame.timestamp) { - // Fast path because we normally append to the end. - queue.push(frame as any) - } else { - // Do a full binary search - let low = 0 - let high = queue.length; - - while (low < high) { - var mid = (low + high) >>> 1; - if (queue[mid].timestamp < frame.timestamp) low = mid + 1; - else high = mid; - } - - queue.splice(low, 0, frame as any) - } - - // Queue up to render the next frame. - if (!this.render) { - this.render = self.requestAnimationFrame(this.draw.bind(this)) - } } - draw(now: DOMHighResTimeStamp) { - // Convert to microseconds - now *= 1000; - - // Determine the target timestamp. - const target = now - this.sync! - - this.drawAudio(now, target) - this.drawVideo(now, target) - - if (this.audioQueue.length || this.videoQueue.length) { - this.render = self.requestAnimationFrame(this.draw.bind(this)) - } else { - this.render = 0 - } - } - - drawAudio(now: DOMHighResTimeStamp, target: DOMHighResTimeStamp) { - // Check if we should skip some frames - while (this.audioQueue.length) { - const next = this.audioQueue[0] - - if (next.timestamp > target) { - let ok = this.audioRing.write(next) - if (!ok) { - console.warn("ring buffer is full") - // No more space in the ring - break - } - } else { - console.warn("dropping audio") - } - - next.close() - this.audioQueue.shift() - } - } - - drawVideo(now: DOMHighResTimeStamp, target: DOMHighResTimeStamp) { - if (!this.videoQueue.length) return; - - let frame = this.videoQueue[0]; - if (frame.timestamp >= target) { - // nothing to render yet, wait for the next animation frame - this.render = self.requestAnimationFrame(this.draw.bind(this)) - return - } - - this.videoQueue.shift(); - - // Check if we should skip some frames - while (this.videoQueue.length) { - const next = this.videoQueue[0] - if (next.timestamp > target) break - - frame.close() - frame = this.videoQueue.shift()!; - } - - const ctx = this.videoCanvas.getContext("2d"); - ctx!.drawImage(frame, 0, 0, this.videoCanvas.width, this.videoCanvas.height) // TODO aspect ratio - - this.last = frame.timestamp; - frame.close() + play(play: Message.Play) { + this.audio.play(play); + this.video.play(play); } } diff --git a/web/src/player/ring.ts b/web/src/player/ring.ts index ffc1c6b..ccf6f2a 100644 --- a/web/src/player/ring.ts +++ b/web/src/player/ring.ts @@ -1,123 +1,9 @@ // Ring buffer with audio samples. enum STATE { - READ_INDEX = 0, // Index of the current read position (mod capacity) - WRITE_INDEX, // Index of the current write position (mod capacity) - LENGTH // Clever way of saving the total number of enums values. -} - -export class Ring { - state: Int32Array; - channels: Float32Array[]; - capacity: number; - - constructor(buf: Buffer) { - this.state = new Int32Array(buf.state) - - this.channels = [] - for (let channel of buf.channels) { - this.channels.push(new Float32Array(channel)) - } - - this.capacity = buf.capacity - } - - // Add the samples for single audio frame - write(frame: AudioData): boolean { - let count = frame.numberOfFrames; - - let readIndex = Atomics.load(this.state, STATE.READ_INDEX) - let writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX) - let writeIndexNew = writeIndex + count; - - // There's not enough space in the ring buffer - if (writeIndexNew - readIndex > this.capacity) { - return false - } - - let startIndex = writeIndex % this.capacity; - let endIndex = writeIndexNew % this.capacity; - - // Loop over each channel - for (let i = 0; i < this.channels.length; i += 1) { - const channel = this.channels[i] - - if (startIndex < endIndex) { - // One continuous range to copy. - const full = channel.subarray(startIndex, endIndex) - - frame.copyTo(full, { - planeIndex: i, - frameCount: count, - }) - } else { - const first = channel.subarray(startIndex) - const second = channel.subarray(0, endIndex) - - frame.copyTo(first, { - planeIndex: i, - frameCount: first.length, - }) - - frame.copyTo(second, { - planeIndex: i, - frameOffset: first.length, - frameCount: second.length, - }) - } - } - - Atomics.store(this.state, STATE.WRITE_INDEX, writeIndexNew) - - return true - } - - read(dst: Float32Array[]) { - let readIndex = Atomics.load(this.state, STATE.READ_INDEX) - let writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX) - if (readIndex >= writeIndex) { - // nothing to read - return - } - - let readIndexNew = readIndex + dst[0].length - if (readIndexNew > writeIndex) { - // Partial read - readIndexNew = writeIndex - } - - let startIndex = readIndex % this.capacity; - let endIndex = readIndexNew % this.capacity; - - // Loop over each channel - for (let i = 0; i < dst.length; i += 1) { - if (i >= this.channels.length) { - // ignore excess channels - } - - const input = this.channels[i] - const output = dst[i] - - if (startIndex < endIndex) { - const full = input.subarray(startIndex, endIndex) - output.set(full) - } else { - const first = input.subarray(startIndex) - const second = input.subarray(0, endIndex) - - output.set(first) - output.set(second, first.length) - } - } - - Atomics.store(this.state, STATE.READ_INDEX, readIndexNew) - } - - // TODO not thread safe - clear() { - const writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX) - Atomics.store(this.state, STATE.READ_INDEX, writeIndex) - } + READ_POS = 0, // The current read position + WRITE_POS, // The current write position + LENGTH // Clever way of saving the total number of enums values. } // No prototype to make this easier to send via postMessage @@ -140,4 +26,125 @@ export class Buffer { this.capacity = capacity } +} + +export class Ring { + state: Int32Array; + channels: Float32Array[]; + capacity: number; + + constructor(buffer: Buffer) { + this.state = new Int32Array(buffer.state) + + this.channels = [] + for (let channel of buffer.channels) { + this.channels.push(new Float32Array(channel)) + } + + this.capacity = buffer.capacity + } + + // Write samples for single audio frame, returning the total number written. + write(frame: AudioData): number { + let readPos = Atomics.load(this.state, STATE.READ_POS) + let writePos = Atomics.load(this.state, STATE.WRITE_POS) + + const startPos = writePos + let endPos = writePos + frame.numberOfFrames; + + if (endPos > readPos + this.capacity) { + endPos = readPos + this.capacity + if (endPos <= startPos) { + // No space to write + return 0 + } + } + + let startIndex = startPos % this.capacity; + let endIndex = endPos % this.capacity; + + // Loop over each channel + for (let i = 0; i < this.channels.length; i += 1) { + const channel = this.channels[i] + + if (startIndex < endIndex) { + // One continuous range to copy. + const full = channel.subarray(startIndex, endIndex) + + frame.copyTo(full, { + planeIndex: i, + frameCount: endIndex - startIndex, + }) + } else { + const first = channel.subarray(startIndex) + const second = channel.subarray(0, endIndex) + + frame.copyTo(first, { + planeIndex: i, + frameCount: first.length, + }) + + frame.copyTo(second, { + planeIndex: i, + frameOffset: first.length, + frameCount: second.length, + }) + } + } + + Atomics.store(this.state, STATE.WRITE_POS, endPos) + + return endPos - startPos + } + + read(dst: Float32Array[]): number { + let readPos = Atomics.load(this.state, STATE.READ_POS) + let writePos = Atomics.load(this.state, STATE.WRITE_POS) + + let startPos = readPos; + let endPos = startPos + dst[0].length; + + if (endPos > writePos) { + endPos = writePos + if (endPos <= startPos) { + // Nothing to read + return 0 + } + } + + let startIndex = startPos % this.capacity; + let endIndex = endPos % this.capacity; + + // Loop over each channel + for (let i = 0; i < dst.length; i += 1) { + if (i >= this.channels.length) { + // ignore excess channels + } + + const input = this.channels[i] + const output = dst[i] + + if (startIndex < endIndex) { + const full = input.subarray(startIndex, endIndex) + output.set(full) + } else { + const first = input.subarray(startIndex) + const second = input.subarray(0, endIndex) + + output.set(first) + output.set(second, first.length) + } + } + + Atomics.store(this.state, STATE.READ_POS, endPos) + + return endPos - startPos + } + + size() { + let readPos = Atomics.load(this.state, STATE.READ_POS) + let writePos = Atomics.load(this.state, STATE.WRITE_POS) + + return writePos - readPos + } } \ No newline at end of file diff --git a/web/src/player/video.ts b/web/src/player/video.ts index d112150..f725122 100644 --- a/web/src/player/video.ts +++ b/web/src/player/video.ts @@ -5,10 +5,10 @@ export default class Video { queue: Array; render: number; // non-zero if requestAnimationFrame has been called - sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0, in microseconds + sync?: number; // the wall clock value for timestamp 0, in microseconds last?: number; // the timestamp of the last rendered frame, in microseconds - constructor(config: Message.VideoConfig) { + constructor(config: Message.Config) { this.canvas = config.canvas; this.queue = []; @@ -16,11 +16,6 @@ export default class Video { } push(frame: VideoFrame) { - if (!this.sync) { - // Save the frame as the sync point - this.sync = 1000 * performance.now() - frame.timestamp - } - // Drop any old frames if (this.last && frame.timestamp <= this.last) { frame.close() @@ -28,7 +23,7 @@ export default class Video { } // Insert the frame into the queue sorted by timestamp. - if (this.queue.length > 0 && this.queue[this.queue.length-1].timestamp <= frame.timestamp) { + if (this.queue.length > 0 && this.queue[this.queue.length - 1].timestamp <= frame.timestamp) { // Fast path because we normally append to the end. this.queue.push(frame) } else { @@ -44,24 +39,35 @@ export default class Video { this.queue.splice(low, 0, frame) } - - // Queue up to render the next frame. - if (!this.render) { - this.render = self.requestAnimationFrame(this.draw.bind(this)) - } } - draw(now: DOMHighResTimeStamp) { + draw(now: number) { + // Draw and then queue up the next draw call. + this.drawOnce(now); + + // Queue up the new draw frame. + this.render = self.requestAnimationFrame(this.draw.bind(this)) + } + + drawOnce(now: number) { // Convert to microseconds now *= 1000; - // Determine the target timestamp. - const target = now - this.sync! + if (!this.queue.length) { + return + } let frame = this.queue[0]; + + if (!this.sync) { + this.sync = now - frame.timestamp; + } + + // Determine the target timestamp. + const target = now - this.sync + if (frame.timestamp >= target) { // nothing to render yet, wait for the next animation frame - this.render = self.requestAnimationFrame(this.draw.bind(this)) return } @@ -81,11 +87,12 @@ export default class Video { this.last = frame.timestamp; frame.close() + } - if (this.queue.length) { + play(play: Message.Play) { + // Queue up to render the next frame. + if (!this.render) { this.render = self.requestAnimationFrame(this.draw.bind(this)) - } else { - this.render = 0 } } } \ No newline at end of file diff --git a/web/src/player/worker.ts b/web/src/player/worker.ts index 4597c29..5a563f5 100644 --- a/web/src/player/worker.ts +++ b/web/src/player/worker.ts @@ -17,6 +17,9 @@ self.addEventListener('message', async (e: MessageEvent) => { } else if (e.data.segment) { const segment = e.data.segment as Message.Segment await decoder.receiveSegment(segment) + } else if (e.data.play) { + const play = e.data.play as Message.Play + await renderer.play(play) } }) diff --git a/web/src/player/worklet.ts b/web/src/player/worklet.ts index 4946bd8..fe3216e 100644 --- a/web/src/player/worklet.ts +++ b/web/src/player/worklet.ts @@ -19,19 +19,19 @@ class Renderer extends AudioWorkletProcessor { } onMessage(e: MessageEvent) { - if (e.data.config) { - this.onConfig(e.data.config) + if (e.data.play) { + this.onPlay(e.data.play) } } - onConfig(config: Message.AudioConfig) { - this.ring = new Ring(config.ring) + onPlay(play: Message.Play) { + this.ring = new Ring(play.buffer) } // Inputs and outputs in groups of 128 samples. process(inputs: Float32Array[][], outputs: Float32Array[][], parameters: Record): boolean { if (!this.ring) { - // Not initialized yet + // Paused return true } @@ -40,7 +40,11 @@ class Renderer extends AudioWorkletProcessor { } const output = outputs[0] - this.ring.read(output) + + const size = this.ring.read(output) + if (size < output.length) { + // TODO trigger rebuffering event + } return true; }