rob
/
moq-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed audio.

This commit is contained in:
Luke Curley 2023-05-22 13:30:46 -07:00
parent a9fd9186d2
commit 3f6ea42380
10 changed files with 239 additions and 330 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
web/src/index.html
View File

@ -2,7 +2,7 @@
<html>
<head>
<meta charset = "UTF-8">
<meta charset="UTF-8">
<title>WARP</title>
<link rel="stylesheet" href="index.css">
@ -11,7 +11,7 @@
<body>
<div id="player">
<div id="screen">
<div id="play"><span>click for audio</span></div>
<div id="play"><span>click to play</span></div>
<canvas id="video" width="1280" height="720"></canvas>
</div>
@ -31,4 +31,5 @@
<script src="index.ts" type="module"></script>
</body>
</html>

2
web/src/index.ts
View File

@ -31,7 +31,7 @@ const player = new Player({
const play = document.querySelector<HTMLElement>("#screen #play")!
const playFunc = (e: Event) => {
player.play({})
player.play()
e.preventDefault()
play.removeEventListener('click', playFunc)

64
web/src/player/audio.ts
View File

@ -2,24 +2,17 @@ import * as Message from "./message";
import { Ring } from "./ring"
export default class Audio {
ring: Ring;
ring?: Ring;
queue: Array<AudioData>;
sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0, in microseconds
render?: number; // non-zero if requestAnimationFrame has been called
last?: number; // the timestamp of the last rendered frame, in microseconds
constructor(config: Message.AudioConfig) {
this.ring = new Ring(config.ring);
this.queue = [];
constructor(config: Message.Config) {
this.queue = []
}
push(frame: AudioData) {
if (!this.sync) {
// Save the frame as the sync point
// TODO sync with video
this.sync = 1000 * performance.now() - frame.timestamp
}
// Drop any old frames
if (this.last && frame.timestamp <= this.last) {
frame.close()
@ -27,7 +20,7 @@ export default class Audio {
}
// Insert the frame into the queue sorted by timestamp.
if (this.queue.length > 0 && this.queue[this.queue.length-1].timestamp <= frame.timestamp) {
if (this.queue.length > 0 && this.queue[this.queue.length - 1].timestamp <= frame.timestamp) {
// Fast path because we normally append to the end.
this.queue.push(frame)
} else {
@ -43,33 +36,44 @@ export default class Audio {
this.queue.splice(low, 0, frame)
}
this.emit()
}
emit() {
const ring = this.ring
if (!ring) {
return
}
draw() {
// Convert to microseconds
const now = 1000 * performance.now();
// Determine the target timestamp.
const target = now - this.sync!
// Check if we should skip some frames
while (this.queue.length) {
const next = this.queue[0]
if (next.timestamp > target) {
const ok = this.ring.write(next)
if (!ok) {
console.warn("ring buffer is full")
// No more space in the ring
let frame = this.queue[0];
if (ring.size() + frame.numberOfFrames > ring.capacity) {
// Buffer is full
break
}
} else {
console.warn("dropping audio")
const size = ring.write(frame)
if (size < frame.numberOfFrames) {
throw new Error("audio buffer is full")
}
next.close()
this.last = frame.timestamp
frame.close()
this.queue.shift()
}
}
play(play: Message.Play) {
this.ring = new Ring(play.buffer)
if (!this.render) {
const sampleRate = 44100 // TODO dynamic
// Refresh every half buffer
const refresh = play.buffer.capacity / sampleRate * 1000 / 2
this.render = setInterval(this.emit.bind(this), refresh)
}
}
}

43
web/src/player/index.ts
View File

@ -19,26 +19,16 @@ export default class Player {
this.transport = config.transport
this.transport.callback = this;
const video = {
canvas: config.canvas,
};
// Assume 44.1kHz and two audio channels
const audio = {
sampleRate: 44100,
ring: new Ring.Buffer(2, 4410), // 100ms at 44.1khz
}
this.context = new AudioContext({
latencyHint: "interactive",
sampleRate: audio.sampleRate,
sampleRate: 44100,
})
this.worker = this.setupWorker({ audio, video })
this.worklet = this.setupWorklet(audio)
this.worker = this.setupWorker(config)
this.worklet = this.setupWorklet(config)
}
private setupWorker(config: Message.Config): Worker {
private setupWorker(config: Config): Worker {
const url = new URL('worker.ts', import.meta.url)
const worker = new Worker(url, {
@ -46,12 +36,16 @@ export default class Player {
name: "media",
})
worker.postMessage({ config }, [ config.video.canvas ])
const msg = {
canvas: config.canvas,
}
worker.postMessage({ config: msg }, [msg.canvas])
return worker
}
private async setupWorklet(config: Message.AudioConfig): Promise<AudioWorkletNode> {
private async setupWorklet(config: Config): Promise<AudioWorkletNode> {
// Load the worklet source code.
const url = new URL('worklet.ts', import.meta.url)
await this.context.audioWorklet.addModule(url)
@ -65,8 +59,6 @@ export default class Player {
console.error("Audio worklet error:", e)
};
worklet.port.postMessage({ config })
// Connect the worklet to the volume node and then to the speakers
worklet.connect(volume)
volume.connect(this.context.destination)
@ -75,15 +67,22 @@ export default class Player {
}
onInit(init: Message.Init) {
this.worker.postMessage({ init }, [ init.buffer.buffer, init.reader ])
this.worker.postMessage({ init }, [init.buffer.buffer, init.reader])
}
onSegment(segment: Message.Segment) {
this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ])
this.worker.postMessage({ segment }, [segment.buffer.buffer, segment.reader])
}
play(play: Message.Play) {
async play() {
this.context.resume()
//this.worker.postMessage({ play })
const play = {
buffer: new Ring.Buffer(2, 44100 / 10), // 100ms of audio
}
const worklet = await this.worklet;
worklet.port.postMessage({ play })
this.worker.postMessage({ play })
}
}

13
web/src/player/message.ts
View File

@ -1,20 +1,10 @@
import * as Ring from "./ring"
export interface Config {
audio: AudioConfig;
video: VideoConfig;
}
export interface VideoConfig {
// video stuff
canvas: OffscreenCanvas;
}
export interface AudioConfig {
// audio stuff
sampleRate: number;
ring: Ring.Buffer;
}
export interface Init {
buffer: Uint8Array; // unread buffered data
reader: ReadableStream; // unread unbuffered data
@ -27,4 +17,5 @@ export interface Segment {
export interface Play {
timestamp?: number;
buffer: Ring.Buffer;
}

129
web/src/player/renderer.ts
View File

@ -1,136 +1,29 @@
import * as Message from "./message";
import { Ring } from "./ring"
import Audio from "./audio"
import Video from "./video"
export default class Renderer {
audioRing: Ring;
audioQueue: Array<AudioData>;
videoCanvas: OffscreenCanvas;
videoQueue: Array<VideoFrame>;
render: number; // non-zero if requestAnimationFrame has been called
sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0, in microseconds
last?: number; // the timestamp of the last rendered frame, in microseconds
audio: Audio;
video: Video;
constructor(config: Message.Config) {
this.audioRing = new Ring(config.audio.ring);
this.audioQueue = [];
this.videoCanvas = config.video.canvas;
this.videoQueue = [];
this.render = 0;
this.audio = new Audio(config);
this.video = new Video(config);
}
push(frame: AudioData | VideoFrame) {
if (!this.sync) {
// Save the frame as the sync point
this.sync = 1000 * performance.now() - frame.timestamp
}
// Drop any old frames
if (this.last && frame.timestamp <= this.last) {
frame.close()
return
}
let queue
if (isAudioData(frame)) {
queue = this.audioQueue;
this.audio.push(frame);
} else if (isVideoFrame(frame)) {
queue = this.videoQueue;
this.video.push(frame);
} else {
throw new Error("unknown frame type")
}
// Insert the frame into the queue sorted by timestamp.
if (queue.length > 0 && queue[queue.length-1].timestamp <= frame.timestamp) {
// Fast path because we normally append to the end.
queue.push(frame as any)
} else {
// Do a full binary search
let low = 0
let high = queue.length;
while (low < high) {
var mid = (low + high) >>> 1;
if (queue[mid].timestamp < frame.timestamp) low = mid + 1;
else high = mid;
}
queue.splice(low, 0, frame as any)
}
// Queue up to render the next frame.
if (!this.render) {
this.render = self.requestAnimationFrame(this.draw.bind(this))
}
}
draw(now: DOMHighResTimeStamp) {
// Convert to microseconds
now *= 1000;
// Determine the target timestamp.
const target = now - this.sync!
this.drawAudio(now, target)
this.drawVideo(now, target)
if (this.audioQueue.length || this.videoQueue.length) {
this.render = self.requestAnimationFrame(this.draw.bind(this))
} else {
this.render = 0
}
}
drawAudio(now: DOMHighResTimeStamp, target: DOMHighResTimeStamp) {
// Check if we should skip some frames
while (this.audioQueue.length) {
const next = this.audioQueue[0]
if (next.timestamp > target) {
let ok = this.audioRing.write(next)
if (!ok) {
console.warn("ring buffer is full")
// No more space in the ring
break
}
} else {
console.warn("dropping audio")
}
next.close()
this.audioQueue.shift()
}
}
drawVideo(now: DOMHighResTimeStamp, target: DOMHighResTimeStamp) {
if (!this.videoQueue.length) return;
let frame = this.videoQueue[0];
if (frame.timestamp >= target) {
// nothing to render yet, wait for the next animation frame
this.render = self.requestAnimationFrame(this.draw.bind(this))
return
}
this.videoQueue.shift();
// Check if we should skip some frames
while (this.videoQueue.length) {
const next = this.videoQueue[0]
if (next.timestamp > target) break
frame.close()
frame = this.videoQueue.shift()!;
}
const ctx = this.videoCanvas.getContext("2d");
ctx!.drawImage(frame, 0, 0, this.videoCanvas.width, this.videoCanvas.height) // TODO aspect ratio
this.last = frame.timestamp;
frame.close()
play(play: Message.Play) {
this.audio.play(play);
this.video.play(play);
}
}

239
web/src/player/ring.ts
View File

@ -1,125 +1,11 @@
// Ring buffer with audio samples.
enum STATE {
READ_INDEX = 0, // Index of the current read position (mod capacity)
WRITE_INDEX, // Index of the current write position (mod capacity)
READ_POS = 0, // The current read position
WRITE_POS, // The current write position
LENGTH // Clever way of saving the total number of enums values.
}
export class Ring {
state: Int32Array;
channels: Float32Array[];
capacity: number;
constructor(buf: Buffer) {
this.state = new Int32Array(buf.state)
this.channels = []
for (let channel of buf.channels) {
this.channels.push(new Float32Array(channel))
}
this.capacity = buf.capacity
}
// Add the samples for single audio frame
write(frame: AudioData): boolean {
let count = frame.numberOfFrames;
let readIndex = Atomics.load(this.state, STATE.READ_INDEX)
let writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX)
let writeIndexNew = writeIndex + count;
// There's not enough space in the ring buffer
if (writeIndexNew - readIndex > this.capacity) {
return false
}
let startIndex = writeIndex % this.capacity;
let endIndex = writeIndexNew % this.capacity;
// Loop over each channel
for (let i = 0; i < this.channels.length; i += 1) {
const channel = this.channels[i]
if (startIndex < endIndex) {
// One continuous range to copy.
const full = channel.subarray(startIndex, endIndex)
frame.copyTo(full, {
planeIndex: i,
frameCount: count,
})
} else {
const first = channel.subarray(startIndex)
const second = channel.subarray(0, endIndex)
frame.copyTo(first, {
planeIndex: i,
frameCount: first.length,
})
frame.copyTo(second, {
planeIndex: i,
frameOffset: first.length,
frameCount: second.length,
})
}
}
Atomics.store(this.state, STATE.WRITE_INDEX, writeIndexNew)
return true
}
read(dst: Float32Array[]) {
let readIndex = Atomics.load(this.state, STATE.READ_INDEX)
let writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX)
if (readIndex >= writeIndex) {
// nothing to read
return
}
let readIndexNew = readIndex + dst[0].length
if (readIndexNew > writeIndex) {
// Partial read
readIndexNew = writeIndex
}
let startIndex = readIndex % this.capacity;
let endIndex = readIndexNew % this.capacity;
// Loop over each channel
for (let i = 0; i < dst.length; i += 1) {
if (i >= this.channels.length) {
// ignore excess channels
}
const input = this.channels[i]
const output = dst[i]
if (startIndex < endIndex) {
const full = input.subarray(startIndex, endIndex)
output.set(full)
} else {
const first = input.subarray(startIndex)
const second = input.subarray(0, endIndex)
output.set(first)
output.set(second, first.length)
}
}
Atomics.store(this.state, STATE.READ_INDEX, readIndexNew)
}
// TODO not thread safe
clear() {
const writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX)
Atomics.store(this.state, STATE.READ_INDEX, writeIndex)
}
}
// No prototype to make this easier to send via postMessage
export class Buffer {
state: SharedArrayBuffer;
@ -141,3 +27,124 @@ export class Buffer {
this.capacity = capacity
}
}
export class Ring {
state: Int32Array;
channels: Float32Array[];
capacity: number;
constructor(buffer: Buffer) {
this.state = new Int32Array(buffer.state)
this.channels = []
for (let channel of buffer.channels) {
this.channels.push(new Float32Array(channel))
}
this.capacity = buffer.capacity
}
// Write samples for single audio frame, returning the total number written.
write(frame: AudioData): number {
let readPos = Atomics.load(this.state, STATE.READ_POS)
let writePos = Atomics.load(this.state, STATE.WRITE_POS)
const startPos = writePos
let endPos = writePos + frame.numberOfFrames;
if (endPos > readPos + this.capacity) {
endPos = readPos + this.capacity
if (endPos <= startPos) {
// No space to write
return 0
}
}
let startIndex = startPos % this.capacity;
let endIndex = endPos % this.capacity;
// Loop over each channel
for (let i = 0; i < this.channels.length; i += 1) {
const channel = this.channels[i]
if (startIndex < endIndex) {
// One continuous range to copy.
const full = channel.subarray(startIndex, endIndex)
frame.copyTo(full, {
planeIndex: i,
frameCount: endIndex - startIndex,
})
} else {
const first = channel.subarray(startIndex)
const second = channel.subarray(0, endIndex)
frame.copyTo(first, {
planeIndex: i,
frameCount: first.length,
})
frame.copyTo(second, {
planeIndex: i,
frameOffset: first.length,
frameCount: second.length,
})
}
}
Atomics.store(this.state, STATE.WRITE_POS, endPos)
return endPos - startPos
}
read(dst: Float32Array[]): number {
let readPos = Atomics.load(this.state, STATE.READ_POS)
let writePos = Atomics.load(this.state, STATE.WRITE_POS)
let startPos = readPos;
let endPos = startPos + dst[0].length;
if (endPos > writePos) {
endPos = writePos
if (endPos <= startPos) {
// Nothing to read
return 0
}
}
let startIndex = startPos % this.capacity;
let endIndex = endPos % this.capacity;
// Loop over each channel
for (let i = 0; i < dst.length; i += 1) {
if (i >= this.channels.length) {
// ignore excess channels
}
const input = this.channels[i]
const output = dst[i]
if (startIndex < endIndex) {
const full = input.subarray(startIndex, endIndex)
output.set(full)
} else {
const first = input.subarray(startIndex)
const second = input.subarray(0, endIndex)
output.set(first)
output.set(second, first.length)
}
}
Atomics.store(this.state, STATE.READ_POS, endPos)
return endPos - startPos
}
size() {
let readPos = Atomics.load(this.state, STATE.READ_POS)
let writePos = Atomics.load(this.state, STATE.WRITE_POS)
return writePos - readPos
}
}

43
web/src/player/video.ts
View File

@ -5,10 +5,10 @@ export default class Video {
queue: Array<VideoFrame>;
render: number; // non-zero if requestAnimationFrame has been called
sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0, in microseconds
sync?: number; // the wall clock value for timestamp 0, in microseconds
last?: number; // the timestamp of the last rendered frame, in microseconds
constructor(config: Message.VideoConfig) {
constructor(config: Message.Config) {
this.canvas = config.canvas;
this.queue = [];
@ -16,11 +16,6 @@ export default class Video {
}
push(frame: VideoFrame) {
if (!this.sync) {
// Save the frame as the sync point
this.sync = 1000 * performance.now() - frame.timestamp
}
// Drop any old frames
if (this.last && frame.timestamp <= this.last) {
frame.close()
@ -28,7 +23,7 @@ export default class Video {
}
// Insert the frame into the queue sorted by timestamp.
if (this.queue.length > 0 && this.queue[this.queue.length-1].timestamp <= frame.timestamp) {
if (this.queue.length > 0 && this.queue[this.queue.length - 1].timestamp <= frame.timestamp) {
// Fast path because we normally append to the end.
this.queue.push(frame)
} else {
@ -44,24 +39,35 @@ export default class Video {
this.queue.splice(low, 0, frame)
}
}
// Queue up to render the next frame.
if (!this.render) {
draw(now: number) {
// Draw and then queue up the next draw call.
this.drawOnce(now);
// Queue up the new draw frame.
this.render = self.requestAnimationFrame(this.draw.bind(this))
}
}
draw(now: DOMHighResTimeStamp) {
drawOnce(now: number) {
// Convert to microseconds
now *= 1000;
// Determine the target timestamp.
const target = now - this.sync!
if (!this.queue.length) {
return
}
let frame = this.queue[0];
if (!this.sync) {
this.sync = now - frame.timestamp;
}
// Determine the target timestamp.
const target = now - this.sync
if (frame.timestamp >= target) {
// nothing to render yet, wait for the next animation frame
this.render = self.requestAnimationFrame(this.draw.bind(this))
return
}
@ -81,11 +87,12 @@ export default class Video {
this.last = frame.timestamp;
frame.close()
}
if (this.queue.length) {
play(play: Message.Play) {
// Queue up to render the next frame.
if (!this.render) {
this.render = self.requestAnimationFrame(this.draw.bind(this))
} else {
this.render = 0
}
}
}

3
web/src/player/worker.ts
View File

@ -17,6 +17,9 @@ self.addEventListener('message', async (e: MessageEvent) => {
} else if (e.data.segment) {
const segment = e.data.segment as Message.Segment
await decoder.receiveSegment(segment)
} else if (e.data.play) {
const play = e.data.play as Message.Play
await renderer.play(play)
}
})

16
web/src/player/worklet.ts
View File

@ -19,19 +19,19 @@ class Renderer extends AudioWorkletProcessor {
}
onMessage(e: MessageEvent) {
if (e.data.config) {
this.onConfig(e.data.config)
if (e.data.play) {
this.onPlay(e.data.play)
}
}
onConfig(config: Message.AudioConfig) {
this.ring = new Ring(config.ring)
onPlay(play: Message.Play) {
this.ring = new Ring(play.buffer)
}
// Inputs and outputs in groups of 128 samples.
process(inputs: Float32Array[][], outputs: Float32Array[][], parameters: Record<string, Float32Array>): boolean {
if (!this.ring) {
// Not initialized yet
// Paused
return true
}
@ -40,7 +40,11 @@ class Renderer extends AudioWorkletProcessor {
}
const output = outputs[0]
this.ring.read(output)
const size = this.ring.read(output)
if (size < output.length) {
// TODO trigger rebuffering event
}
return true;
}