-
click for audio
+
click to play
@@ -31,4 +31,5 @@
-
+
+
\ No newline at end of file
diff --git a/web/src/index.ts b/web/src/index.ts
index 5c46360..053f819 100644
--- a/web/src/index.ts
+++ b/web/src/index.ts
@@ -31,7 +31,7 @@ const player = new Player({
const play = document.querySelector
("#screen #play")!
const playFunc = (e: Event) => {
- player.play({})
+ player.play()
e.preventDefault()
play.removeEventListener('click', playFunc)
diff --git a/web/src/player/audio.ts b/web/src/player/audio.ts
index 361036d..cbfcd3e 100644
--- a/web/src/player/audio.ts
+++ b/web/src/player/audio.ts
@@ -2,24 +2,17 @@ import * as Message from "./message";
import { Ring } from "./ring"
export default class Audio {
- ring: Ring;
+ ring?: Ring;
queue: Array;
- sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0, in microseconds
+ render?: number; // non-zero if requestAnimationFrame has been called
last?: number; // the timestamp of the last rendered frame, in microseconds
- constructor(config: Message.AudioConfig) {
- this.ring = new Ring(config.ring);
- this.queue = [];
+ constructor(config: Message.Config) {
+ this.queue = []
}
push(frame: AudioData) {
- if (!this.sync) {
- // Save the frame as the sync point
- // TODO sync with video
- this.sync = 1000 * performance.now() - frame.timestamp
- }
-
// Drop any old frames
if (this.last && frame.timestamp <= this.last) {
frame.close()
@@ -27,7 +20,7 @@ export default class Audio {
}
// Insert the frame into the queue sorted by timestamp.
- if (this.queue.length > 0 && this.queue[this.queue.length-1].timestamp <= frame.timestamp) {
+ if (this.queue.length > 0 && this.queue[this.queue.length - 1].timestamp <= frame.timestamp) {
// Fast path because we normally append to the end.
this.queue.push(frame)
} else {
@@ -43,33 +36,44 @@ export default class Audio {
this.queue.splice(low, 0, frame)
}
+
+ this.emit()
}
+ emit() {
+ const ring = this.ring
+ if (!ring) {
+ return
+ }
- draw() {
- // Convert to microseconds
- const now = 1000 * performance.now();
-
- // Determine the target timestamp.
- const target = now - this.sync!
-
- // Check if we should skip some frames
while (this.queue.length) {
- const next = this.queue[0]
-
- if (next.timestamp > target) {
- const ok = this.ring.write(next)
- if (!ok) {
- console.warn("ring buffer is full")
- // No more space in the ring
- break
- }
- } else {
- console.warn("dropping audio")
+ let frame = this.queue[0];
+ if (ring.size() + frame.numberOfFrames > ring.capacity) {
+ // Buffer is full
+ break
}
- next.close()
+ const size = ring.write(frame)
+ if (size < frame.numberOfFrames) {
+ throw new Error("audio buffer is full")
+ }
+
+ this.last = frame.timestamp
+
+ frame.close()
this.queue.shift()
}
}
+
+ play(play: Message.Play) {
+ this.ring = new Ring(play.buffer)
+
+ if (!this.render) {
+ const sampleRate = 44100 // TODO dynamic
+
+ // Refresh every half buffer
+ const refresh = play.buffer.capacity / sampleRate * 1000 / 2
+ this.render = setInterval(this.emit.bind(this), refresh)
+ }
+ }
}
\ No newline at end of file
diff --git a/web/src/player/index.ts b/web/src/player/index.ts
index 1ef73b2..a539167 100644
--- a/web/src/player/index.ts
+++ b/web/src/player/index.ts
@@ -19,26 +19,16 @@ export default class Player {
this.transport = config.transport
this.transport.callback = this;
- const video = {
- canvas: config.canvas,
- };
-
- // Assume 44.1kHz and two audio channels
- const audio = {
- sampleRate: 44100,
- ring: new Ring.Buffer(2, 4410), // 100ms at 44.1khz
- }
-
this.context = new AudioContext({
latencyHint: "interactive",
- sampleRate: audio.sampleRate,
+ sampleRate: 44100,
})
- this.worker = this.setupWorker({ audio, video })
- this.worklet = this.setupWorklet(audio)
+ this.worker = this.setupWorker(config)
+ this.worklet = this.setupWorklet(config)
}
- private setupWorker(config: Message.Config): Worker {
+ private setupWorker(config: Config): Worker {
const url = new URL('worker.ts', import.meta.url)
const worker = new Worker(url, {
@@ -46,12 +36,16 @@ export default class Player {
name: "media",
})
- worker.postMessage({ config }, [ config.video.canvas ])
+ const msg = {
+ canvas: config.canvas,
+ }
+
+ worker.postMessage({ config: msg }, [msg.canvas])
return worker
}
- private async setupWorklet(config: Message.AudioConfig): Promise {
+ private async setupWorklet(config: Config): Promise {
// Load the worklet source code.
const url = new URL('worklet.ts', import.meta.url)
await this.context.audioWorklet.addModule(url)
@@ -65,8 +59,6 @@ export default class Player {
console.error("Audio worklet error:", e)
};
- worklet.port.postMessage({ config })
-
// Connect the worklet to the volume node and then to the speakers
worklet.connect(volume)
volume.connect(this.context.destination)
@@ -75,15 +67,22 @@ export default class Player {
}
onInit(init: Message.Init) {
- this.worker.postMessage({ init }, [ init.buffer.buffer, init.reader ])
+ this.worker.postMessage({ init }, [init.buffer.buffer, init.reader])
}
onSegment(segment: Message.Segment) {
- this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ])
+ this.worker.postMessage({ segment }, [segment.buffer.buffer, segment.reader])
}
- play(play: Message.Play) {
+ async play() {
this.context.resume()
- //this.worker.postMessage({ play })
+
+ const play = {
+ buffer: new Ring.Buffer(2, 44100 / 10), // 100ms of audio
+ }
+
+ const worklet = await this.worklet;
+ worklet.port.postMessage({ play })
+ this.worker.postMessage({ play })
}
}
\ No newline at end of file
diff --git a/web/src/player/message.ts b/web/src/player/message.ts
index dcad2e0..0f4b00b 100644
--- a/web/src/player/message.ts
+++ b/web/src/player/message.ts
@@ -1,20 +1,10 @@
import * as Ring from "./ring"
export interface Config {
- audio: AudioConfig;
- video: VideoConfig;
-}
-
-export interface VideoConfig {
+ // video stuff
canvas: OffscreenCanvas;
}
-export interface AudioConfig {
- // audio stuff
- sampleRate: number;
- ring: Ring.Buffer;
-}
-
export interface Init {
buffer: Uint8Array; // unread buffered data
reader: ReadableStream; // unread unbuffered data
@@ -27,4 +17,5 @@ export interface Segment {
export interface Play {
timestamp?: number;
+ buffer: Ring.Buffer;
}
\ No newline at end of file
diff --git a/web/src/player/renderer.ts b/web/src/player/renderer.ts
index d2d6c86..3da2e85 100644
--- a/web/src/player/renderer.ts
+++ b/web/src/player/renderer.ts
@@ -1,136 +1,29 @@
import * as Message from "./message";
-import { Ring } from "./ring"
+import Audio from "./audio"
+import Video from "./video"
export default class Renderer {
- audioRing: Ring;
- audioQueue: Array;
-
- videoCanvas: OffscreenCanvas;
- videoQueue: Array;
-
- render: number; // non-zero if requestAnimationFrame has been called
- sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0, in microseconds
- last?: number; // the timestamp of the last rendered frame, in microseconds
+ audio: Audio;
+ video: Video;
constructor(config: Message.Config) {
- this.audioRing = new Ring(config.audio.ring);
- this.audioQueue = [];
-
- this.videoCanvas = config.video.canvas;
- this.videoQueue = [];
-
- this.render = 0;
+ this.audio = new Audio(config);
+ this.video = new Video(config);
}
push(frame: AudioData | VideoFrame) {
- if (!this.sync) {
- // Save the frame as the sync point
- this.sync = 1000 * performance.now() - frame.timestamp
- }
-
- // Drop any old frames
- if (this.last && frame.timestamp <= this.last) {
- frame.close()
- return
- }
-
- let queue
if (isAudioData(frame)) {
- queue = this.audioQueue;
+ this.audio.push(frame);
} else if (isVideoFrame(frame)) {
- queue = this.videoQueue;
+ this.video.push(frame);
} else {
throw new Error("unknown frame type")
}
-
- // Insert the frame into the queue sorted by timestamp.
- if (queue.length > 0 && queue[queue.length-1].timestamp <= frame.timestamp) {
- // Fast path because we normally append to the end.
- queue.push(frame as any)
- } else {
- // Do a full binary search
- let low = 0
- let high = queue.length;
-
- while (low < high) {
- var mid = (low + high) >>> 1;
- if (queue[mid].timestamp < frame.timestamp) low = mid + 1;
- else high = mid;
- }
-
- queue.splice(low, 0, frame as any)
- }
-
- // Queue up to render the next frame.
- if (!this.render) {
- this.render = self.requestAnimationFrame(this.draw.bind(this))
- }
}
- draw(now: DOMHighResTimeStamp) {
- // Convert to microseconds
- now *= 1000;
-
- // Determine the target timestamp.
- const target = now - this.sync!
-
- this.drawAudio(now, target)
- this.drawVideo(now, target)
-
- if (this.audioQueue.length || this.videoQueue.length) {
- this.render = self.requestAnimationFrame(this.draw.bind(this))
- } else {
- this.render = 0
- }
- }
-
- drawAudio(now: DOMHighResTimeStamp, target: DOMHighResTimeStamp) {
- // Check if we should skip some frames
- while (this.audioQueue.length) {
- const next = this.audioQueue[0]
-
- if (next.timestamp > target) {
- let ok = this.audioRing.write(next)
- if (!ok) {
- console.warn("ring buffer is full")
- // No more space in the ring
- break
- }
- } else {
- console.warn("dropping audio")
- }
-
- next.close()
- this.audioQueue.shift()
- }
- }
-
- drawVideo(now: DOMHighResTimeStamp, target: DOMHighResTimeStamp) {
- if (!this.videoQueue.length) return;
-
- let frame = this.videoQueue[0];
- if (frame.timestamp >= target) {
- // nothing to render yet, wait for the next animation frame
- this.render = self.requestAnimationFrame(this.draw.bind(this))
- return
- }
-
- this.videoQueue.shift();
-
- // Check if we should skip some frames
- while (this.videoQueue.length) {
- const next = this.videoQueue[0]
- if (next.timestamp > target) break
-
- frame.close()
- frame = this.videoQueue.shift()!;
- }
-
- const ctx = this.videoCanvas.getContext("2d");
- ctx!.drawImage(frame, 0, 0, this.videoCanvas.width, this.videoCanvas.height) // TODO aspect ratio
-
- this.last = frame.timestamp;
- frame.close()
+ play(play: Message.Play) {
+ this.audio.play(play);
+ this.video.play(play);
}
}
diff --git a/web/src/player/ring.ts b/web/src/player/ring.ts
index ffc1c6b..ccf6f2a 100644
--- a/web/src/player/ring.ts
+++ b/web/src/player/ring.ts
@@ -1,123 +1,9 @@
// Ring buffer with audio samples.
enum STATE {
- READ_INDEX = 0, // Index of the current read position (mod capacity)
- WRITE_INDEX, // Index of the current write position (mod capacity)
- LENGTH // Clever way of saving the total number of enums values.
-}
-
-export class Ring {
- state: Int32Array;
- channels: Float32Array[];
- capacity: number;
-
- constructor(buf: Buffer) {
- this.state = new Int32Array(buf.state)
-
- this.channels = []
- for (let channel of buf.channels) {
- this.channels.push(new Float32Array(channel))
- }
-
- this.capacity = buf.capacity
- }
-
- // Add the samples for single audio frame
- write(frame: AudioData): boolean {
- let count = frame.numberOfFrames;
-
- let readIndex = Atomics.load(this.state, STATE.READ_INDEX)
- let writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX)
- let writeIndexNew = writeIndex + count;
-
- // There's not enough space in the ring buffer
- if (writeIndexNew - readIndex > this.capacity) {
- return false
- }
-
- let startIndex = writeIndex % this.capacity;
- let endIndex = writeIndexNew % this.capacity;
-
- // Loop over each channel
- for (let i = 0; i < this.channels.length; i += 1) {
- const channel = this.channels[i]
-
- if (startIndex < endIndex) {
- // One continuous range to copy.
- const full = channel.subarray(startIndex, endIndex)
-
- frame.copyTo(full, {
- planeIndex: i,
- frameCount: count,
- })
- } else {
- const first = channel.subarray(startIndex)
- const second = channel.subarray(0, endIndex)
-
- frame.copyTo(first, {
- planeIndex: i,
- frameCount: first.length,
- })
-
- frame.copyTo(second, {
- planeIndex: i,
- frameOffset: first.length,
- frameCount: second.length,
- })
- }
- }
-
- Atomics.store(this.state, STATE.WRITE_INDEX, writeIndexNew)
-
- return true
- }
-
- read(dst: Float32Array[]) {
- let readIndex = Atomics.load(this.state, STATE.READ_INDEX)
- let writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX)
- if (readIndex >= writeIndex) {
- // nothing to read
- return
- }
-
- let readIndexNew = readIndex + dst[0].length
- if (readIndexNew > writeIndex) {
- // Partial read
- readIndexNew = writeIndex
- }
-
- let startIndex = readIndex % this.capacity;
- let endIndex = readIndexNew % this.capacity;
-
- // Loop over each channel
- for (let i = 0; i < dst.length; i += 1) {
- if (i >= this.channels.length) {
- // ignore excess channels
- }
-
- const input = this.channels[i]
- const output = dst[i]
-
- if (startIndex < endIndex) {
- const full = input.subarray(startIndex, endIndex)
- output.set(full)
- } else {
- const first = input.subarray(startIndex)
- const second = input.subarray(0, endIndex)
-
- output.set(first)
- output.set(second, first.length)
- }
- }
-
- Atomics.store(this.state, STATE.READ_INDEX, readIndexNew)
- }
-
- // TODO not thread safe
- clear() {
- const writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX)
- Atomics.store(this.state, STATE.READ_INDEX, writeIndex)
- }
+ READ_POS = 0, // The current read position
+ WRITE_POS, // The current write position
+ LENGTH // Clever way of saving the total number of enums values.
}
// No prototype to make this easier to send via postMessage
@@ -140,4 +26,125 @@ export class Buffer {
this.capacity = capacity
}
+}
+
+export class Ring {
+ state: Int32Array;
+ channels: Float32Array[];
+ capacity: number;
+
+ constructor(buffer: Buffer) {
+ this.state = new Int32Array(buffer.state)
+
+ this.channels = []
+ for (let channel of buffer.channels) {
+ this.channels.push(new Float32Array(channel))
+ }
+
+ this.capacity = buffer.capacity
+ }
+
+ // Write samples for single audio frame, returning the total number written.
+ write(frame: AudioData): number {
+ let readPos = Atomics.load(this.state, STATE.READ_POS)
+ let writePos = Atomics.load(this.state, STATE.WRITE_POS)
+
+ const startPos = writePos
+ let endPos = writePos + frame.numberOfFrames;
+
+ if (endPos > readPos + this.capacity) {
+ endPos = readPos + this.capacity
+ if (endPos <= startPos) {
+ // No space to write
+ return 0
+ }
+ }
+
+ let startIndex = startPos % this.capacity;
+ let endIndex = endPos % this.capacity;
+
+ // Loop over each channel
+ for (let i = 0; i < this.channels.length; i += 1) {
+ const channel = this.channels[i]
+
+ if (startIndex < endIndex) {
+ // One continuous range to copy.
+ const full = channel.subarray(startIndex, endIndex)
+
+ frame.copyTo(full, {
+ planeIndex: i,
+ frameCount: endIndex - startIndex,
+ })
+ } else {
+ const first = channel.subarray(startIndex)
+ const second = channel.subarray(0, endIndex)
+
+ frame.copyTo(first, {
+ planeIndex: i,
+ frameCount: first.length,
+ })
+
+ frame.copyTo(second, {
+ planeIndex: i,
+ frameOffset: first.length,
+ frameCount: second.length,
+ })
+ }
+ }
+
+ Atomics.store(this.state, STATE.WRITE_POS, endPos)
+
+ return endPos - startPos
+ }
+
+ read(dst: Float32Array[]): number {
+ let readPos = Atomics.load(this.state, STATE.READ_POS)
+ let writePos = Atomics.load(this.state, STATE.WRITE_POS)
+
+ let startPos = readPos;
+ let endPos = startPos + dst[0].length;
+
+ if (endPos > writePos) {
+ endPos = writePos
+ if (endPos <= startPos) {
+ // Nothing to read
+ return 0
+ }
+ }
+
+ let startIndex = startPos % this.capacity;
+ let endIndex = endPos % this.capacity;
+
+ // Loop over each channel
+ for (let i = 0; i < dst.length; i += 1) {
+ if (i >= this.channels.length) {
+ // ignore excess channels
+ }
+
+ const input = this.channels[i]
+ const output = dst[i]
+
+ if (startIndex < endIndex) {
+ const full = input.subarray(startIndex, endIndex)
+ output.set(full)
+ } else {
+ const first = input.subarray(startIndex)
+ const second = input.subarray(0, endIndex)
+
+ output.set(first)
+ output.set(second, first.length)
+ }
+ }
+
+ Atomics.store(this.state, STATE.READ_POS, endPos)
+
+ return endPos - startPos
+ }
+
+ size() {
+ let readPos = Atomics.load(this.state, STATE.READ_POS)
+ let writePos = Atomics.load(this.state, STATE.WRITE_POS)
+
+ return writePos - readPos
+ }
}
\ No newline at end of file
diff --git a/web/src/player/video.ts b/web/src/player/video.ts
index d112150..f725122 100644
--- a/web/src/player/video.ts
+++ b/web/src/player/video.ts
@@ -5,10 +5,10 @@ export default class Video {
queue: Array;
render: number; // non-zero if requestAnimationFrame has been called
- sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0, in microseconds
+ sync?: number; // the wall clock value for timestamp 0, in microseconds
last?: number; // the timestamp of the last rendered frame, in microseconds
- constructor(config: Message.VideoConfig) {
+ constructor(config: Message.Config) {
this.canvas = config.canvas;
this.queue = [];
@@ -16,11 +16,6 @@ export default class Video {
}
push(frame: VideoFrame) {
- if (!this.sync) {
- // Save the frame as the sync point
- this.sync = 1000 * performance.now() - frame.timestamp
- }
-
// Drop any old frames
if (this.last && frame.timestamp <= this.last) {
frame.close()
@@ -28,7 +23,7 @@ export default class Video {
}
// Insert the frame into the queue sorted by timestamp.
- if (this.queue.length > 0 && this.queue[this.queue.length-1].timestamp <= frame.timestamp) {
+ if (this.queue.length > 0 && this.queue[this.queue.length - 1].timestamp <= frame.timestamp) {
// Fast path because we normally append to the end.
this.queue.push(frame)
} else {
@@ -44,24 +39,35 @@ export default class Video {
this.queue.splice(low, 0, frame)
}
-
- // Queue up to render the next frame.
- if (!this.render) {
- this.render = self.requestAnimationFrame(this.draw.bind(this))
- }
}
- draw(now: DOMHighResTimeStamp) {
+ draw(now: number) {
+ // Draw and then queue up the next draw call.
+ this.drawOnce(now);
+
+ // Queue up the new draw frame.
+ this.render = self.requestAnimationFrame(this.draw.bind(this))
+ }
+
+ drawOnce(now: number) {
// Convert to microseconds
now *= 1000;
- // Determine the target timestamp.
- const target = now - this.sync!
+ if (!this.queue.length) {
+ return
+ }
let frame = this.queue[0];
+
+ if (!this.sync) {
+ this.sync = now - frame.timestamp;
+ }
+
+ // Determine the target timestamp.
+ const target = now - this.sync
+
if (frame.timestamp >= target) {
// nothing to render yet, wait for the next animation frame
- this.render = self.requestAnimationFrame(this.draw.bind(this))
return
}
@@ -81,11 +87,12 @@ export default class Video {
this.last = frame.timestamp;
frame.close()
+ }
- if (this.queue.length) {
+ play(play: Message.Play) {
+ // Queue up to render the next frame.
+ if (!this.render) {
this.render = self.requestAnimationFrame(this.draw.bind(this))
- } else {
- this.render = 0
}
}
}
\ No newline at end of file
diff --git a/web/src/player/worker.ts b/web/src/player/worker.ts
index 4597c29..5a563f5 100644
--- a/web/src/player/worker.ts
+++ b/web/src/player/worker.ts
@@ -17,6 +17,9 @@ self.addEventListener('message', async (e: MessageEvent) => {
} else if (e.data.segment) {
const segment = e.data.segment as Message.Segment
await decoder.receiveSegment(segment)
+ } else if (e.data.play) {
+ const play = e.data.play as Message.Play
+ await renderer.play(play)
}
})
diff --git a/web/src/player/worklet.ts b/web/src/player/worklet.ts
index 4946bd8..fe3216e 100644
--- a/web/src/player/worklet.ts
+++ b/web/src/player/worklet.ts
@@ -19,19 +19,19 @@ class Renderer extends AudioWorkletProcessor {
}
onMessage(e: MessageEvent) {
- if (e.data.config) {
- this.onConfig(e.data.config)
+ if (e.data.play) {
+ this.onPlay(e.data.play)
}
}
- onConfig(config: Message.AudioConfig) {
- this.ring = new Ring(config.ring)
+ onPlay(play: Message.Play) {
+ this.ring = new Ring(play.buffer)
}
// Inputs and outputs in groups of 128 samples.
process(inputs: Float32Array[][], outputs: Float32Array[][], parameters: Record): boolean {
if (!this.ring) {
- // Not initialized yet
+ // Paused
return true
}
@@ -40,7 +40,11 @@ class Renderer extends AudioWorkletProcessor {
}
const output = outputs[0]
- this.ring.read(output)
+
+ const size = this.ring.read(output)
+ if (size < output.length) {
+ // TODO trigger rebuffering event
+ }
return true;
}