rob
/
moq-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed audio.

This commit is contained in:
Luke Curley 2023-05-22 13:30:46 -07:00
parent a9fd9186d2
commit 3f6ea42380
10 changed files with 239 additions and 330 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
web/src/index.html
View File

@ -2,7 +2,7 @@
<html> <html>
<head> <head>
<meta charset = "UTF-8"> <meta charset="UTF-8">
<title>WARP</title> <title>WARP</title>
<link rel="stylesheet" href="index.css"> <link rel="stylesheet" href="index.css">
@ -11,7 +11,7 @@
<body> <body>
<div id="player"> <div id="player">
<div id="screen"> <div id="screen">
<div id="play"><span>click for audio</span></div> <div id="play"><span>click to play</span></div>
<canvas id="video" width="1280" height="720"></canvas> <canvas id="video" width="1280" height="720"></canvas>
</div> </div>
@ -31,4 +31,5 @@
<script src="index.ts" type="module"></script> <script src="index.ts" type="module"></script>
</body> </body>
</html> </html>

2
web/src/index.ts
View File

@ -31,7 +31,7 @@ const player = new Player({
const play = document.querySelector<HTMLElement>("#screen #play")! const play = document.querySelector<HTMLElement>("#screen #play")!
const playFunc = (e: Event) => { const playFunc = (e: Event) => {
player.play({}) player.play()
e.preventDefault() e.preventDefault()
play.removeEventListener('click', playFunc) play.removeEventListener('click', playFunc)

68
web/src/player/audio.ts
View File

@ -2,24 +2,17 @@ import * as Message from "./message";
import { Ring } from "./ring" import { Ring } from "./ring"
export default class Audio { export default class Audio {
ring: Ring; ring?: Ring;
queue: Array<AudioData>; queue: Array<AudioData>;
sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0, in microseconds render?: number; // non-zero if requestAnimationFrame has been called
last?: number; // the timestamp of the last rendered frame, in microseconds last?: number; // the timestamp of the last rendered frame, in microseconds
constructor(config: Message.AudioConfig) { constructor(config: Message.Config) {
this.ring = new Ring(config.ring); this.queue = []
this.queue = [];
} }
push(frame: AudioData) { push(frame: AudioData) {
if (!this.sync) {
// Save the frame as the sync point
// TODO sync with video
this.sync = 1000 * performance.now() - frame.timestamp
}
// Drop any old frames // Drop any old frames
if (this.last && frame.timestamp <= this.last) { if (this.last && frame.timestamp <= this.last) {
frame.close() frame.close()
@ -27,7 +20,7 @@ export default class Audio {
} }
// Insert the frame into the queue sorted by timestamp. // Insert the frame into the queue sorted by timestamp.
if (this.queue.length > 0 && this.queue[this.queue.length-1].timestamp <= frame.timestamp) { if (this.queue.length > 0 && this.queue[this.queue.length - 1].timestamp <= frame.timestamp) {
// Fast path because we normally append to the end. // Fast path because we normally append to the end.
this.queue.push(frame) this.queue.push(frame)
} else { } else {
@ -43,33 +36,44 @@ export default class Audio {
this.queue.splice(low, 0, frame) this.queue.splice(low, 0, frame)
} }
this.emit()
} }
emit() {
const ring = this.ring
if (!ring) {
return
}
draw() {
// Convert to microseconds
const now = 1000 * performance.now();
// Determine the target timestamp.
const target = now - this.sync!
// Check if we should skip some frames
while (this.queue.length) { while (this.queue.length) {
const next = this.queue[0] let frame = this.queue[0];
if (ring.size() + frame.numberOfFrames > ring.capacity) {
if (next.timestamp > target) { // Buffer is full
const ok = this.ring.write(next) break
if (!ok) {
console.warn("ring buffer is full")
// No more space in the ring
break
}
} else {
console.warn("dropping audio")
} }
next.close() const size = ring.write(frame)
if (size < frame.numberOfFrames) {
throw new Error("audio buffer is full")
}
this.last = frame.timestamp
frame.close()
this.queue.shift() this.queue.shift()
} }
} }
play(play: Message.Play) {
this.ring = new Ring(play.buffer)
if (!this.render) {
const sampleRate = 44100 // TODO dynamic
// Refresh every half buffer
const refresh = play.buffer.capacity / sampleRate * 1000 / 2
this.render = setInterval(this.emit.bind(this), refresh)
}
}
} }

43
web/src/player/index.ts
View File

@ -19,26 +19,16 @@ export default class Player {
this.transport = config.transport this.transport = config.transport
this.transport.callback = this; this.transport.callback = this;
const video = {
canvas: config.canvas,
};
// Assume 44.1kHz and two audio channels
const audio = {
sampleRate: 44100,
ring: new Ring.Buffer(2, 4410), // 100ms at 44.1khz
}
this.context = new AudioContext({ this.context = new AudioContext({
latencyHint: "interactive", latencyHint: "interactive",
sampleRate: audio.sampleRate, sampleRate: 44100,
}) })
this.worker = this.setupWorker({ audio, video }) this.worker = this.setupWorker(config)
this.worklet = this.setupWorklet(audio) this.worklet = this.setupWorklet(config)
} }
private setupWorker(config: Message.Config): Worker { private setupWorker(config: Config): Worker {
const url = new URL('worker.ts', import.meta.url) const url = new URL('worker.ts', import.meta.url)
const worker = new Worker(url, { const worker = new Worker(url, {
@ -46,12 +36,16 @@ export default class Player {
name: "media", name: "media",
}) })
worker.postMessage({ config }, [ config.video.canvas ]) const msg = {
canvas: config.canvas,
}
worker.postMessage({ config: msg }, [msg.canvas])
return worker return worker
} }
private async setupWorklet(config: Message.AudioConfig): Promise<AudioWorkletNode> { private async setupWorklet(config: Config): Promise<AudioWorkletNode> {
// Load the worklet source code. // Load the worklet source code.
const url = new URL('worklet.ts', import.meta.url) const url = new URL('worklet.ts', import.meta.url)
await this.context.audioWorklet.addModule(url) await this.context.audioWorklet.addModule(url)
@ -65,8 +59,6 @@ export default class Player {
console.error("Audio worklet error:", e) console.error("Audio worklet error:", e)
}; };
worklet.port.postMessage({ config })
// Connect the worklet to the volume node and then to the speakers // Connect the worklet to the volume node and then to the speakers
worklet.connect(volume) worklet.connect(volume)
volume.connect(this.context.destination) volume.connect(this.context.destination)
@ -75,15 +67,22 @@ export default class Player {
} }
onInit(init: Message.Init) { onInit(init: Message.Init) {
this.worker.postMessage({ init }, [ init.buffer.buffer, init.reader ]) this.worker.postMessage({ init }, [init.buffer.buffer, init.reader])
} }
onSegment(segment: Message.Segment) { onSegment(segment: Message.Segment) {
this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ]) this.worker.postMessage({ segment }, [segment.buffer.buffer, segment.reader])
} }
play(play: Message.Play) { async play() {
this.context.resume() this.context.resume()
//this.worker.postMessage({ play })
const play = {
buffer: new Ring.Buffer(2, 44100 / 10), // 100ms of audio
}
const worklet = await this.worklet;
worklet.port.postMessage({ play })
this.worker.postMessage({ play })
} }
} }

13
web/src/player/message.ts
View File

@ -1,20 +1,10 @@
import * as Ring from "./ring" import * as Ring from "./ring"
export interface Config { export interface Config {
audio: AudioConfig; // video stuff
video: VideoConfig;
}
export interface VideoConfig {
canvas: OffscreenCanvas; canvas: OffscreenCanvas;
} }
export interface AudioConfig {
// audio stuff
sampleRate: number;
ring: Ring.Buffer;
}
export interface Init { export interface Init {
buffer: Uint8Array; // unread buffered data buffer: Uint8Array; // unread buffered data
reader: ReadableStream; // unread unbuffered data reader: ReadableStream; // unread unbuffered data
@ -27,4 +17,5 @@ export interface Segment {
export interface Play { export interface Play {
timestamp?: number; timestamp?: number;
buffer: Ring.Buffer;
} }

129
web/src/player/renderer.ts
View File

@ -1,136 +1,29 @@
import * as Message from "./message"; import * as Message from "./message";
import { Ring } from "./ring" import Audio from "./audio"
import Video from "./video"
export default class Renderer { export default class Renderer {
audioRing: Ring; audio: Audio;
audioQueue: Array<AudioData>; video: Video;
videoCanvas: OffscreenCanvas;
videoQueue: Array<VideoFrame>;
render: number; // non-zero if requestAnimationFrame has been called
sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0, in microseconds
last?: number; // the timestamp of the last rendered frame, in microseconds
constructor(config: Message.Config) { constructor(config: Message.Config) {
this.audioRing = new Ring(config.audio.ring); this.audio = new Audio(config);
this.audioQueue = []; this.video = new Video(config);
this.videoCanvas = config.video.canvas;
this.videoQueue = [];
this.render = 0;
} }
push(frame: AudioData | VideoFrame) { push(frame: AudioData | VideoFrame) {
if (!this.sync) {
// Save the frame as the sync point
this.sync = 1000 * performance.now() - frame.timestamp
}
// Drop any old frames
if (this.last && frame.timestamp <= this.last) {
frame.close()
return
}
let queue
if (isAudioData(frame)) { if (isAudioData(frame)) {
queue = this.audioQueue; this.audio.push(frame);
} else if (isVideoFrame(frame)) { } else if (isVideoFrame(frame)) {
queue = this.videoQueue; this.video.push(frame);
} else { } else {
throw new Error("unknown frame type") throw new Error("unknown frame type")
} }
// Insert the frame into the queue sorted by timestamp.
if (queue.length > 0 && queue[queue.length-1].timestamp <= frame.timestamp) {
// Fast path because we normally append to the end.
queue.push(frame as any)
} else {
// Do a full binary search
let low = 0
let high = queue.length;
while (low < high) {
var mid = (low + high) >>> 1;
if (queue[mid].timestamp < frame.timestamp) low = mid + 1;
else high = mid;
}
queue.splice(low, 0, frame as any)
}
// Queue up to render the next frame.
if (!this.render) {
this.render = self.requestAnimationFrame(this.draw.bind(this))
}
} }
draw(now: DOMHighResTimeStamp) { play(play: Message.Play) {
// Convert to microseconds this.audio.play(play);
now *= 1000; this.video.play(play);
// Determine the target timestamp.
const target = now - this.sync!
this.drawAudio(now, target)
this.drawVideo(now, target)
if (this.audioQueue.length || this.videoQueue.length) {
this.render = self.requestAnimationFrame(this.draw.bind(this))
} else {
this.render = 0
}
}
drawAudio(now: DOMHighResTimeStamp, target: DOMHighResTimeStamp) {
// Check if we should skip some frames
while (this.audioQueue.length) {
const next = this.audioQueue[0]
if (next.timestamp > target) {
let ok = this.audioRing.write(next)
if (!ok) {
console.warn("ring buffer is full")
// No more space in the ring
break
}
} else {
console.warn("dropping audio")
}
next.close()
this.audioQueue.shift()
}
}
drawVideo(now: DOMHighResTimeStamp, target: DOMHighResTimeStamp) {
if (!this.videoQueue.length) return;
let frame = this.videoQueue[0];
if (frame.timestamp >= target) {
// nothing to render yet, wait for the next animation frame
this.render = self.requestAnimationFrame(this.draw.bind(this))
return
}
this.videoQueue.shift();
// Check if we should skip some frames
while (this.videoQueue.length) {
const next = this.videoQueue[0]
if (next.timestamp > target) break
frame.close()
frame = this.videoQueue.shift()!;
}
const ctx = this.videoCanvas.getContext("2d");
ctx!.drawImage(frame, 0, 0, this.videoCanvas.width, this.videoCanvas.height) // TODO aspect ratio
this.last = frame.timestamp;
frame.close()
} }
} }

241
web/src/player/ring.ts
View File

@ -1,123 +1,9 @@
// Ring buffer with audio samples. // Ring buffer with audio samples.
enum STATE { enum STATE {
READ_INDEX = 0, // Index of the current read position (mod capacity) READ_POS = 0, // The current read position
WRITE_INDEX, // Index of the current write position (mod capacity) WRITE_POS, // The current write position
LENGTH // Clever way of saving the total number of enums values. LENGTH // Clever way of saving the total number of enums values.
}
export class Ring {
state: Int32Array;
channels: Float32Array[];
capacity: number;
constructor(buf: Buffer) {
this.state = new Int32Array(buf.state)
this.channels = []
for (let channel of buf.channels) {
this.channels.push(new Float32Array(channel))
}
this.capacity = buf.capacity
}
// Add the samples for single audio frame
write(frame: AudioData): boolean {
let count = frame.numberOfFrames;
let readIndex = Atomics.load(this.state, STATE.READ_INDEX)
let writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX)
let writeIndexNew = writeIndex + count;
// There's not enough space in the ring buffer
if (writeIndexNew - readIndex > this.capacity) {
return false
}
let startIndex = writeIndex % this.capacity;
let endIndex = writeIndexNew % this.capacity;
// Loop over each channel
for (let i = 0; i < this.channels.length; i += 1) {
const channel = this.channels[i]
if (startIndex < endIndex) {
// One continuous range to copy.
const full = channel.subarray(startIndex, endIndex)
frame.copyTo(full, {
planeIndex: i,
frameCount: count,
})
} else {
const first = channel.subarray(startIndex)
const second = channel.subarray(0, endIndex)
frame.copyTo(first, {
planeIndex: i,
frameCount: first.length,
})
frame.copyTo(second, {
planeIndex: i,
frameOffset: first.length,
frameCount: second.length,
})
}
}
Atomics.store(this.state, STATE.WRITE_INDEX, writeIndexNew)
return true
}
read(dst: Float32Array[]) {
let readIndex = Atomics.load(this.state, STATE.READ_INDEX)
let writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX)
if (readIndex >= writeIndex) {
// nothing to read
return
}
let readIndexNew = readIndex + dst[0].length
if (readIndexNew > writeIndex) {
// Partial read
readIndexNew = writeIndex
}
let startIndex = readIndex % this.capacity;
let endIndex = readIndexNew % this.capacity;
// Loop over each channel
for (let i = 0; i < dst.length; i += 1) {
if (i >= this.channels.length) {
// ignore excess channels
}
const input = this.channels[i]
const output = dst[i]
if (startIndex < endIndex) {
const full = input.subarray(startIndex, endIndex)
output.set(full)
} else {
const first = input.subarray(startIndex)
const second = input.subarray(0, endIndex)
output.set(first)
output.set(second, first.length)
}
}
Atomics.store(this.state, STATE.READ_INDEX, readIndexNew)
}
// TODO not thread safe
clear() {
const writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX)
Atomics.store(this.state, STATE.READ_INDEX, writeIndex)
}
} }
// No prototype to make this easier to send via postMessage // No prototype to make this easier to send via postMessage
@ -141,3 +27,124 @@ export class Buffer {
this.capacity = capacity this.capacity = capacity
} }
} }
export class Ring {
state: Int32Array;
channels: Float32Array[];
capacity: number;
constructor(buffer: Buffer) {
this.state = new Int32Array(buffer.state)
this.channels = []
for (let channel of buffer.channels) {
this.channels.push(new Float32Array(channel))
}
this.capacity = buffer.capacity
}
// Write samples for single audio frame, returning the total number written.
write(frame: AudioData): number {
let readPos = Atomics.load(this.state, STATE.READ_POS)
let writePos = Atomics.load(this.state, STATE.WRITE_POS)
const startPos = writePos
let endPos = writePos + frame.numberOfFrames;
if (endPos > readPos + this.capacity) {
endPos = readPos + this.capacity
if (endPos <= startPos) {
// No space to write
return 0
}
}
let startIndex = startPos % this.capacity;
let endIndex = endPos % this.capacity;
// Loop over each channel
for (let i = 0; i < this.channels.length; i += 1) {
const channel = this.channels[i]
if (startIndex < endIndex) {
// One continuous range to copy.
const full = channel.subarray(startIndex, endIndex)
frame.copyTo(full, {
planeIndex: i,
frameCount: endIndex - startIndex,
})
} else {
const first = channel.subarray(startIndex)
const second = channel.subarray(0, endIndex)
frame.copyTo(first, {
planeIndex: i,
frameCount: first.length,
})
frame.copyTo(second, {
planeIndex: i,
frameOffset: first.length,
frameCount: second.length,
})
}
}
Atomics.store(this.state, STATE.WRITE_POS, endPos)
return endPos - startPos
}
read(dst: Float32Array[]): number {
let readPos = Atomics.load(this.state, STATE.READ_POS)
let writePos = Atomics.load(this.state, STATE.WRITE_POS)
let startPos = readPos;
let endPos = startPos + dst[0].length;
if (endPos > writePos) {
endPos = writePos
if (endPos <= startPos) {
// Nothing to read
return 0
}
}
let startIndex = startPos % this.capacity;
let endIndex = endPos % this.capacity;
// Loop over each channel
for (let i = 0; i < dst.length; i += 1) {
if (i >= this.channels.length) {
// ignore excess channels
}
const input = this.channels[i]
const output = dst[i]
if (startIndex < endIndex) {
const full = input.subarray(startIndex, endIndex)
output.set(full)
} else {
const first = input.subarray(startIndex)
const second = input.subarray(0, endIndex)
output.set(first)
output.set(second, first.length)
}
}
Atomics.store(this.state, STATE.READ_POS, endPos)
return endPos - startPos
}
size() {
let readPos = Atomics.load(this.state, STATE.READ_POS)
let writePos = Atomics.load(this.state, STATE.WRITE_POS)
return writePos - readPos
}
}

47
web/src/player/video.ts
View File

@ -5,10 +5,10 @@ export default class Video {
queue: Array<VideoFrame>; queue: Array<VideoFrame>;
render: number; // non-zero if requestAnimationFrame has been called render: number; // non-zero if requestAnimationFrame has been called
sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0, in microseconds sync?: number; // the wall clock value for timestamp 0, in microseconds
last?: number; // the timestamp of the last rendered frame, in microseconds last?: number; // the timestamp of the last rendered frame, in microseconds
constructor(config: Message.VideoConfig) { constructor(config: Message.Config) {
this.canvas = config.canvas; this.canvas = config.canvas;
this.queue = []; this.queue = [];
@ -16,11 +16,6 @@ export default class Video {
} }
push(frame: VideoFrame) { push(frame: VideoFrame) {
if (!this.sync) {
// Save the frame as the sync point
this.sync = 1000 * performance.now() - frame.timestamp
}
// Drop any old frames // Drop any old frames
if (this.last && frame.timestamp <= this.last) { if (this.last && frame.timestamp <= this.last) {
frame.close() frame.close()
@ -28,7 +23,7 @@ export default class Video {
} }
// Insert the frame into the queue sorted by timestamp. // Insert the frame into the queue sorted by timestamp.
if (this.queue.length > 0 && this.queue[this.queue.length-1].timestamp <= frame.timestamp) { if (this.queue.length > 0 && this.queue[this.queue.length - 1].timestamp <= frame.timestamp) {
// Fast path because we normally append to the end. // Fast path because we normally append to the end.
this.queue.push(frame) this.queue.push(frame)
} else { } else {
@ -44,24 +39,35 @@ export default class Video {
this.queue.splice(low, 0, frame) this.queue.splice(low, 0, frame)
} }
// Queue up to render the next frame.
if (!this.render) {
this.render = self.requestAnimationFrame(this.draw.bind(this))
}
} }
draw(now: DOMHighResTimeStamp) { draw(now: number) {
// Draw and then queue up the next draw call.
this.drawOnce(now);
// Queue up the new draw frame.
this.render = self.requestAnimationFrame(this.draw.bind(this))
}
drawOnce(now: number) {
// Convert to microseconds // Convert to microseconds
now *= 1000; now *= 1000;
// Determine the target timestamp. if (!this.queue.length) {
const target = now - this.sync! return
}
let frame = this.queue[0]; let frame = this.queue[0];
if (!this.sync) {
this.sync = now - frame.timestamp;
}
// Determine the target timestamp.
const target = now - this.sync
if (frame.timestamp >= target) { if (frame.timestamp >= target) {
// nothing to render yet, wait for the next animation frame // nothing to render yet, wait for the next animation frame
this.render = self.requestAnimationFrame(this.draw.bind(this))
return return
} }
@ -81,11 +87,12 @@ export default class Video {
this.last = frame.timestamp; this.last = frame.timestamp;
frame.close() frame.close()
}
if (this.queue.length) { play(play: Message.Play) {
// Queue up to render the next frame.
if (!this.render) {
this.render = self.requestAnimationFrame(this.draw.bind(this)) this.render = self.requestAnimationFrame(this.draw.bind(this))
} else {
this.render = 0
} }
} }
} }

3
web/src/player/worker.ts
View File

@ -17,6 +17,9 @@ self.addEventListener('message', async (e: MessageEvent) => {
} else if (e.data.segment) { } else if (e.data.segment) {
const segment = e.data.segment as Message.Segment const segment = e.data.segment as Message.Segment
await decoder.receiveSegment(segment) await decoder.receiveSegment(segment)
} else if (e.data.play) {
const play = e.data.play as Message.Play
await renderer.play(play)
} }
}) })

16
web/src/player/worklet.ts
View File

@ -19,19 +19,19 @@ class Renderer extends AudioWorkletProcessor {
} }
onMessage(e: MessageEvent) { onMessage(e: MessageEvent) {
if (e.data.config) { if (e.data.play) {
this.onConfig(e.data.config) this.onPlay(e.data.play)
} }
} }
onConfig(config: Message.AudioConfig) { onPlay(play: Message.Play) {
this.ring = new Ring(config.ring) this.ring = new Ring(play.buffer)
} }
// Inputs and outputs in groups of 128 samples. // Inputs and outputs in groups of 128 samples.
process(inputs: Float32Array[][], outputs: Float32Array[][], parameters: Record<string, Float32Array>): boolean { process(inputs: Float32Array[][], outputs: Float32Array[][], parameters: Record<string, Float32Array>): boolean {
if (!this.ring) { if (!this.ring) {
// Not initialized yet // Paused
return true return true
} }
@ -40,7 +40,11 @@ class Renderer extends AudioWorkletProcessor {
} }
const output = outputs[0] const output = outputs[0]
this.ring.read(output)
const size = this.ring.read(output)
if (size < output.length) {
// TODO trigger rebuffering event
}
return true; return true;
} }