Merge pull request #10 from kixelated/webcodecs

WebCodecs

cert/generate

@@ -17,4 +17,4 @@ go run filippo.io/mkcert -ecdsa -install
 go run filippo.io/mkcert -ecdsa -days 10 -cert-file "$CRT" -key-file "$KEY" localhost 127.0.0.1 ::1
 
 # Compute the sha256 fingerprint of the certificate for WebTransport
-openssl x509 -in "$CRT" -outform der | openssl dgst -sha256
+openssl x509 -in "$CRT" -outform der | openssl dgst -sha256 > ../player/src/transport/fingerprint.hex

player/.proxyrc.js

@@ -0,0 +1,7 @@
+module.exports = function (app) {
+    app.use((req, res, next) => {
+        res.setHeader('Cross-Origin-Opener-Policy', 'same-origin');
+        res.setHeader('Cross-Origin-Embedder-Policy', 'require-corp');
+        next();
+    });
+};

player/package.json

@@ -6,8 +6,14 @@
     "check": "tsc --noEmit"
   },
   "devDependencies": {
+    "@parcel/transformer-inline-string": "2.8.3",
     "@parcel/validator-typescript": "^2.6.0",
-    "parcel": "^2.6.0",
+    "@types/audioworklet": "^0.0.41",
+    "@types/dom-webcodecs": "^0.1.6",
+    "parcel": "^2.8.0",
     "typescript": ">=3.0.0"
+  },
+  "dependencies": {
+    "mp4box": "^0.5.2"
   }
 }

player/src/audio/decoder.ts

@@ -0,0 +1,121 @@
+import * as Message from "./message";
+import * as MP4 from "../mp4"
+import * as Stream from "../stream"
+import * as Util from "../util"
+
+import Renderer from "./renderer"
+
+export default class Decoder {
+    // Store the init message for each track
+    tracks: Map<string, Util.Deferred<Message.Init>>;
+    decoder: AudioDecoder; // TODO one per track
+    sync: Message.Sync;
+
+    constructor(config: Message.Config, renderer: Renderer) {
+        this.tracks = new Map();
+
+        this.decoder = new AudioDecoder({
+            output: renderer.emit.bind(renderer),
+            error: console.warn,
+        });
+    }
+
+    init(msg: Message.Init) {
+		let defer = this.tracks.get(msg.track);
+		if (!defer) {
+            defer = new Util.Deferred()
+			this.tracks.set(msg.track, defer)
+		}
+
+        if (msg.info.audioTracks.length != 1 || msg.info.videoTracks.length != 0) {
+            throw new Error("Expected a single audio track")
+        }
+
+        const track = msg.info.audioTracks[0]
+        const audio = track.audio
+
+        defer.resolve(msg)
+    }
+
+    async decode(msg: Message.Segment) {
+		let track = this.tracks.get(msg.track);
+		if (!track) {
+            track = new Util.Deferred()
+			this.tracks.set(msg.track, track)
+		}
+
+		// Wait for the init segment to be fully received and parsed
+		const init = await track.promise;
+        const audio = init.info.audioTracks[0]
+
+        if (this.decoder.state == "unconfigured") {
+            this.decoder.configure({
+                codec: audio.codec,
+                numberOfChannels: audio.audio.channel_count,
+                sampleRate: audio.audio.sample_rate,
+            })
+        }
+
+		const input = MP4.New();
+
+        input.onSamples = (id: number, user: any, samples: MP4.Sample[]) => {
+            for (let sample of samples) {
+                // Convert to microseconds
+                const timestamp = 1000 * 1000 * sample.dts / sample.timescale
+                const duration = 1000 * 1000 * sample.duration / sample.timescale
+
+                // This assumes that timescale == sample rate
+                this.decoder.decode(new EncodedAudioChunk({
+                    type: sample.is_sync ? "key" : "delta",
+                    data: sample.data,
+                    duration: duration,
+                    timestamp: timestamp,
+                }))
+            }
+        }
+
+		input.onReady = (info: any) => {
+			input.setExtractionOptions(info.tracks[0].id, {}, { nbSamples: 1 });
+			input.start();
+		}
+
+        // MP4box requires us to reparse the init segment unfortunately
+        let offset = 0;
+
+        for (let raw of init.raw) {
+            raw.fileStart = offset
+            input.appendBuffer(raw)
+        }
+
+        const stream = new Stream.Reader(msg.reader, msg.buffer)
+
+		/* TODO I'm not actually sure why this code doesn't work; something trips up the MP4 parser
+			while (1) {
+				const data = await stream.read()
+				if (!data) break
+
+				input.appendBuffer(data)
+				input.flush()
+			}
+		*/
+
+		// One day I'll figure it out; until then read one top-level atom at a time
+		while (!await stream.done()) {
+			const raw = await stream.peek(4)
+			const size = new DataView(raw.buffer, raw.byteOffset, raw.byteLength).getUint32(0)
+			const atom = await stream.bytes(size)
+
+            // Make a copy of the atom because mp4box only accepts an ArrayBuffer unfortunately
+            let box = new Uint8Array(atom.byteLength);
+            box.set(atom)
+
+            // and for some reason we need to modify the underlying ArrayBuffer with offset
+            let buffer = box.buffer as MP4.ArrayBuffer
+            buffer.fileStart = offset
+
+            // Parse the data
+            offset = input.appendBuffer(buffer)
+            input.flush()
+		}
+    }
+}

player/src/audio/index.ts

@@ -0,0 +1,77 @@
+import * as Message from "./message"
+import Renderer from "./renderer"
+import Decoder from "./decoder"
+
+import { RingInit } from "./ring"
+
+// Abstracts the Worker and Worklet into a simpler API
+// This class must be created on the main thread due to AudioContext.
+export default class Audio {
+    context: AudioContext;
+    worker: Worker;
+    worklet: Promise<AudioWorkletNode>;
+
+    constructor() {
+        // Assume 44.1kHz and two audio channels
+        const config = {
+            sampleRate: 44100,
+            ring: new RingInit(2, 4410), // 100ms at 44.1khz
+        }
+
+        this.context = new AudioContext({
+            latencyHint: "interactive",
+            sampleRate: config.sampleRate,
+        })
+
+        this.worker = this.setupWorker(config)
+        this.worklet = this.setupWorklet(config)
+    }
+
+    private setupWorker(config: Message.Config): Worker {
+        const url = new URL('worker.ts', import.meta.url)
+        const worker = new Worker(url, {
+            name: "audio",
+            type: "module",
+        })
+
+        worker.postMessage({ config })
+
+        return worker
+    }
+
+    private async setupWorklet(config: Message.Config): Promise<AudioWorkletNode> {
+        // Load the worklet source code.
+        const url = new URL('worklet.ts', import.meta.url)
+        await this.context.audioWorklet.addModule(url)
+
+        const volume = this.context.createGain()
+        volume.gain.value = 2.0;
+
+        // Create a worklet
+        const worklet = new AudioWorkletNode(this.context, 'renderer');
+        worklet.onprocessorerror = (e: Event) => {
+            console.error("Audio worklet error:", e)
+        };
+
+        worklet.port.postMessage({ config })
+
+        // Connect the worklet to the volume node and then to the speakers
+        worklet.connect(volume)
+        volume.connect(this.context.destination)
+
+        return worklet
+    }
+
+    init(init: Message.Init) {
+        this.worker.postMessage({ init })
+    }
+
+    segment(segment: Message.Segment) {
+        this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ])
+    }
+
+    play(play: Message.Play) {
+        this.context.resume()
+        //this.worker.postMessage({ play })
+    }
+}

player/src/audio/message.ts

@@ -0,0 +1,30 @@
+import * as MP4 from "../mp4"
+import { RingInit } from "./ring"
+
+export interface Config {
+    sampleRate: number;
+    ring: RingInit;
+}
+
+export interface Init {
+    track: string;
+    info: MP4.Info;
+    raw: MP4.ArrayBuffer[];
+}
+
+export interface Segment {
+    track: string;
+    buffer: Uint8Array;     // unread buffered data
+    reader: ReadableStream; // unread unbuffered data
+}
+
+// Audio tells video when the given timestamp should be rendered.
+export interface Sync {
+    origin: number;
+    clock: DOMHighResTimeStamp;
+    timestamp: number;
+}
+
+export interface Play {
+    timestamp?: number;
+}

player/src/audio/renderer.ts

@@ -0,0 +1,85 @@
+import * as Message from "./message"
+import { Ring } from "./ring"
+
+export default class Renderer {
+    ring: Ring;
+    queue: Array<AudioData>;
+    sync?: DOMHighResTimeStamp
+    running: number;
+
+    constructor(config: Message.Config) {
+        this.ring = new Ring(config.ring)
+        this.queue = [];
+        this.running = 0
+    }
+
+    emit(frame: AudioData) {
+        if (!this.sync) {
+            // Save the frame as the sync point
+            this.sync = 1000 * performance.now() - frame.timestamp
+        }
+
+        // Insert the frame into the queue sorted by timestamp.
+        if (this.queue.length > 0 && this.queue[this.queue.length-1].timestamp <= frame.timestamp) {
+            // Fast path because we normally append to the end.
+            this.queue.push(frame)
+        } else {
+            // Do a full binary search
+            let low = 0
+            let high = this.queue.length;
+
+            while (low < high) {
+                var mid = (low + high) >>> 1;
+                if (this.queue[mid].timestamp < frame.timestamp) low = mid + 1;
+                else high = mid;
+            }
+
+            this.queue.splice(low, 0, frame)
+        }
+
+        if (!this.running) {
+            // Wait for the next animation frame
+            this.running = self.requestAnimationFrame(this.render.bind(this))
+        }
+    }
+
+    render() {
+        // Determine the target timestamp.
+        const target = 1000 * performance.now() - this.sync!
+
+        // Check if we should skip some frames
+        while (this.queue.length) {
+            const next = this.queue[0]
+            if (next.timestamp >= target) {
+                break
+            }
+
+            console.warn("dropping audio")
+
+            this.queue.shift()
+            next.close()
+        }
+
+        // Push as many as we can to the ring buffer.
+        while (this.queue.length) {
+            let frame = this.queue[0]
+            let ok = this.ring.write(frame)
+            if (!ok) {
+                break
+            }
+
+            frame.close()
+            this.queue.shift()
+        }
+
+        if (this.queue.length) {
+            this.running = self.requestAnimationFrame(this.render.bind(this))
+        } else {
+            this.running = 0
+        }
+    }
+
+    play(play: Message.Play) {
+        this.ring.reset()
+    }
+}

player/src/audio/ring.ts

@@ -0,0 +1,143 @@
+// Ring buffer with audio samples.
+
+enum STATE {
+    READ_INDEX = 0, // Index of the current read position (mod capacity)
+    WRITE_INDEX,    // Index of the current write position (mod capacity)
+    LENGTH          // Clever way of saving the total number of enums values.
+}
+
+export class Ring {
+    state: Int32Array;
+    channels: Float32Array[];
+    capacity: number;
+
+    constructor(init: RingInit) {
+        this.state = new Int32Array(init.state)
+
+        this.channels = []
+        for (let channel of init.channels) {
+            this.channels.push(new Float32Array(channel))
+        }
+
+        this.capacity = init.capacity
+    }
+
+    // Add the samples for single audio frame
+    write(frame: AudioData): boolean {
+        let count = frame.numberOfFrames;
+
+        let readIndex = Atomics.load(this.state, STATE.READ_INDEX)
+        let writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX)
+        let writeIndexNew = writeIndex + count;
+
+        // There's not enough space in the ring buffer
+        if (writeIndexNew - readIndex > this.capacity) {
+            return false
+        }
+
+        let startIndex = writeIndex % this.capacity;
+        let endIndex = writeIndexNew % this.capacity;
+
+        // Loop over each channel
+        for (let i = 0; i < this.channels.length; i += 1) {
+            const channel = this.channels[i]
+
+            if (startIndex < endIndex) {
+                // One continuous range to copy.
+                const full = channel.subarray(startIndex, endIndex)
+
+                frame.copyTo(full, {
+                    planeIndex: i,
+                    frameCount: count,
+                })
+            } else {
+                const first = channel.subarray(startIndex)
+                const second = channel.subarray(0, endIndex)
+
+                frame.copyTo(first, {
+                    planeIndex: i,
+                    frameCount: first.length,
+                })
+
+                frame.copyTo(second, {
+                    planeIndex: i,
+                    frameOffset: first.length,
+                    frameCount: second.length,
+                })
+            }
+        }
+
+        Atomics.store(this.state, STATE.WRITE_INDEX, writeIndexNew)
+
+        return true
+    }
+
+    read(dst: Float32Array[]) {
+        let readIndex = Atomics.load(this.state, STATE.READ_INDEX)
+        let writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX)
+        if (readIndex >= writeIndex) {
+            // nothing to read
+            return
+        }
+
+        let readIndexNew = readIndex + dst[0].length
+        if (readIndexNew > writeIndex) {
+            // Partial read
+            readIndexNew = writeIndex
+        }
+
+        let startIndex = readIndex % this.capacity;
+        let endIndex = readIndexNew % this.capacity;
+
+        // Loop over each channel
+        for (let i = 0; i < dst.length; i += 1) {
+            if (i >= this.channels.length) {
+                // ignore excess channels
+            }
+
+            const input = this.channels[i]
+            const output = dst[i]
+
+            if (startIndex < endIndex) {
+                const full = input.subarray(startIndex, endIndex)
+                output.set(full)
+            } else {
+                const first = input.subarray(startIndex)
+                const second = input.subarray(0, endIndex)
+
+                output.set(first)
+                output.set(second, first.length)
+            }
+        }
+
+        Atomics.store(this.state, STATE.READ_INDEX, readIndexNew)
+    }
+
+    // TODO not thread safe
+    clear() {
+        const writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX)
+        Atomics.store(this.state, STATE.READ_INDEX, writeIndex)
+    }
+}
+
+// No prototype to make this easier to send via postMessage
+export class RingInit {
+    state: SharedArrayBuffer;
+
+    channels: SharedArrayBuffer[];
+    capacity: number;
+
+    constructor(channels: number, capacity: number) {
+        // Store the current state in a separate ring buffer.
+        this.state = new SharedArrayBuffer(STATE.LENGTH * Int32Array.BYTES_PER_ELEMENT)
+
+        // Create a buffer for each audio channel
+        this.channels = []
+        for (let i = 0; i < channels; i += 1) {
+            const buffer = new SharedArrayBuffer(capacity * Float32Array.BYTES_PER_ELEMENT)
+            this.channels.push(buffer)
+        }
+
+        this.capacity = capacity
+    }
+}

player/src/audio/worker.ts

@@ -0,0 +1,26 @@
+import Decoder from "./decoder"
+import Renderer from "./renderer"
+
+import * as Message from "./message"
+
+let decoder: Decoder
+let renderer: Renderer;
+
+self.addEventListener('message', (e: MessageEvent) => {
+    if (e.data.config) {
+        renderer = new Renderer(e.data.config)
+        decoder = new Decoder(e.data.config, renderer)
+    }
+
+    if (e.data.init) {
+        decoder.init(e.data.init)
+    }
+
+    if (e.data.segment) {
+        decoder.decode(e.data.segment)
+    }
+
+    if (e.data.play) {
+        renderer.play(e.data.play)
+    }
+})

player/src/audio/worklet.ts

@@ -0,0 +1,49 @@
+// This is an AudioWorklet that acts as a media source.
+// The renderer copies audio samples to a ring buffer read by this worklet.
+// The worklet then outputs those samples to emit audio.
+
+import * as Message from "./message"
+
+import { Ring } from "./ring"
+
+class Renderer extends AudioWorkletProcessor {
+    ring?: Ring;
+    base: number;
+
+    constructor(params: AudioWorkletNodeOptions) {
+        // The super constructor call is required.
+        super();
+
+        this.base = 0
+        this.port.onmessage = this.onMessage.bind(this)
+    }
+
+    onMessage(e: MessageEvent) {
+        if (e.data.config) {
+            this.config(e.data.config)
+        }
+    }
+
+    config(config: Message.Config) {
+        this.ring = new Ring(config.ring)
+    }
+
+    // Inputs and outputs in groups of 128 samples.
+    process(inputs: Float32Array[][], outputs: Float32Array[][], parameters: Record<string, Float32Array>): boolean {
+        if (!this.ring) {
+            // Not initialized yet
+            return true
+        }
+
+        if (inputs.length != 1 && outputs.length != 1) {
+            throw new Error("only a single track is supported")
+        }
+
+        const output = outputs[0]
+        this.ring.read(output)
+
+        return true;
+    }
+}
+
+registerProcessor("renderer", Renderer);

player/src/index.css

@@ -16,7 +16,7 @@ body {
 	position: relative;
 }
 
-#play {
+#screen #play {
 	position: absolute;
 	width: 100%;
 	height: 100%;
@@ -29,12 +29,6 @@ body {
 	z-index: 1;
 }
 
-#vid {
-	width: 100%;
-	height: 100%;
-	max-height: 100vh;
-}
-
 #controls {
 	display: flex;
 	flex-wrap: wrap;

player/src/index.html

@@ -11,8 +11,8 @@
 <body>
 	<div id="player">
 		<div id="screen">
-			<div id="play"><span>click to play</span></div>
-			<video id="vid" controls></video>
+			<div id="play"><span>click for audio</span></div>
+			<canvas id="video" width="1280" height="720"></canvas>
 		</div>
 
 		<div id="controls">

player/src/index.ts

@@ -1,48 +1,23 @@
-import { Player } from "./player"
-
-// This is so ghetto but I'm too lazy to improve it right now
-const videoRef = document.querySelector<HTMLVideoElement>("video#vid")!;
-const liveRef = document.querySelector<HTMLElement>("#live")!;
-const throttleRef = document.querySelector<HTMLElement>("#throttle")!;
-const statsRef = document.querySelector<HTMLElement>("#stats")!;
-const playRef = document.querySelector<HTMLElement>("#play")!;
+import Player from "./player"
 
 const params = new URLSearchParams(window.location.search)
 
 const url = params.get("url") || "https://localhost:4443/watch"
+const canvas = document.querySelector<HTMLCanvasElement>("canvas#video")!
 
 const player = new Player({
     url: url,
-    videoRef: videoRef,
-    statsRef: statsRef,
-    throttleRef: throttleRef,
+    canvas: canvas,
 })
 
-liveRef.addEventListener("click", (e) => {
+const play = document.querySelector<HTMLElement>("#screen #play")!
+
+let playFunc = (e: Event) => {
+    player.play()
     e.preventDefault()
-    player.goLive()
-})
 
-throttleRef.addEventListener("click", (e) => {
-    e.preventDefault()
-    player.throttle()
-})
-
-playRef.addEventListener('click', (e) => {
-    videoRef.play()
-    e.preventDefault()
-})
-
-function playFunc(e: Event) {
-    playRef.style.display = "none"
-    //player.goLive()
-
-    // Only fire once to restore pause/play functionality
-    videoRef.removeEventListener('play', playFunc)
+    play.removeEventListener('click', playFunc)
+    play.style.display = "none"
 }
 
-videoRef.addEventListener('play', playFunc)
-videoRef.volume = 0.5
-
-// Try to autoplay but ignore errors on mobile; they need to click
-//vidRef.play().catch((e) => console.warn(e))
+play.addEventListener('click', playFunc)

player/src/mp4.ts

@@ -1,85 +0,0 @@
-// Wrapper around MP4Box to play nicely with MP4Box.
-// I tried getting a mp4box.all.d.ts file to work but just couldn't figure it out
-import { createFile, ISOFile, DataStream, BoxParser } from "./mp4box.all"
-
-// Rename some stuff so it's on brand.
-export { createFile as MP4New, ISOFile as MP4File, DataStream as MP4Stream, BoxParser as MP4Parser }
-
-export type MP4ArrayBuffer = ArrayBuffer & {fileStart: number};
-
-export interface MP4MediaTrack {
-	id: number;
-	created: Date;
-	modified: Date;
-	movie_duration: number;
-	layer: number;
-	alternate_group: number;
-	volume: number;
-	track_width: number;
-	track_height: number;
-	timescale: number;
-	duration: number;
-	bitrate: number;
-	codec: string;
-	language: string;
-	nb_samples: number;
-}
-
-export interface MP4VideoData {
-	width: number;
-	height: number;
-}
-
-export interface MP4VideoTrack extends MP4MediaTrack {
-	video: MP4VideoData;
-}
-
-export interface MP4AudioData {
-	sample_rate: number;
-	channel_count: number;
-	sample_size: number;
-}
-
-export interface MP4AudioTrack extends MP4MediaTrack {
-	audio: MP4AudioData;
-}
-
-export type MP4Track = MP4VideoTrack | MP4AudioTrack;
-
-export interface MP4Info {
-	duration: number;
-	timescale: number;
-	fragment_duration: number;
-	isFragmented: boolean;
-	isProgressive: boolean;
-	hasIOD: boolean;
-	brands: string[];
-	created: Date;
-	modified: Date;
-	tracks: MP4Track[];
-	mime: string;
-	videoTracks: MP4Track[];
-	audioTracks: MP4Track[];
-}
-
-export interface MP4Sample {
-	number: number;
-	track_id: number;
-	timescale: number;
-	description_index: number;
-	description: any;
-	data: ArrayBuffer;
-	size: number;
-	alreadyRead: number;
-	duration: number;
-	cts: number;
-	dts: number;
-	is_sync: boolean;
-	is_leading: number;
-	depends_on: number;
-	is_depended_on: number;
-	has_redundancy: number;
-	degration_priority: number;
-	offset: number;
-	subsamples: any;
-}

player/src/mp4/index.ts

@@ -0,0 +1,11 @@
+// Rename some stuff so it's on brand.
+export {
+	createFile as New,
+	MP4File as File,
+	MP4ArrayBuffer as ArrayBuffer,
+	MP4Info as Info,
+	DataStream as Stream,
+	Sample,
+} from "mp4box"
+
+export { Init, InitParser } from "./init"

player/src/mp4/init.ts

@@ -1,24 +1,28 @@
-import { MP4New, MP4File, MP4ArrayBuffer, MP4Info } from "./mp4"
+import * as MP4 from "./index"
+
+export interface Init {
+	raw: MP4.ArrayBuffer;
+	info: MP4.Info;
+}
 
 export class InitParser {
-	mp4box: MP4File;
+	mp4box: MP4.File;
 	offset: number;
 
-	raw: MP4ArrayBuffer[];
-	ready: Promise<Init>;
+	raw: MP4.ArrayBuffer[];
+	info: Promise<MP4.Info>;
 
 	constructor() {
-		this.mp4box = MP4New()
-
+		this.mp4box = MP4.New()
 		this.raw = []
 		this.offset = 0
 
 		// Create a promise that gets resolved once the init segment has been parsed.
-		this.ready = new Promise((resolve, reject) => {
+		this.info = new Promise((resolve, reject) => {
 			this.mp4box.onError = reject
 
 			// https://github.com/gpac/mp4box.js#onreadyinfo
-			this.mp4box.onReady = (info: MP4Info) => {
+			this.mp4box.onReady = (info: MP4.Info) => {
 				if (!info.isFragmented) {
 					reject("expected a fragmented mp4")
 				}
@@ -27,10 +31,7 @@ export class InitParser {
 					reject("expected a single track")
 				}
 
-				resolve({
-					info: info,
-					raw: this.raw,
-				})
+				resolve(info)
 			}
 		})
 	}
@@ -41,7 +42,7 @@ export class InitParser {
 		box.set(data)
 
 		// and for some reason we need to modify the underlying ArrayBuffer with fileStart
-		let buffer = box.buffer as MP4ArrayBuffer
+		let buffer = box.buffer as MP4.ArrayBuffer
 		buffer.fileStart = this.offset
 
 		// Parse the data
@@ -52,8 +53,3 @@ export class InitParser {
 		this.raw.push(buffer)
 	}
 }
-
-export interface Init {
-	raw: MP4ArrayBuffer[];
-	info: MP4Info;
-}

player/src/mp4/mp4box.d.ts

@@ -0,0 +1,148 @@
+// https://github.com/gpac/mp4box.js/issues/233
+
+declare module "mp4box" {
+    interface MP4MediaTrack {
+        id: number;
+        created: Date;
+        modified: Date;
+        movie_duration: number;
+        layer: number;
+        alternate_group: number;
+        volume: number;
+        track_width: number;
+        track_height: number;
+        timescale: number;
+        duration: number;
+        bitrate: number;
+        codec: string;
+        language: string;
+        nb_samples: number;
+    }
+
+    interface MP4VideoData {
+        width: number;
+        height: number;
+    }
+
+    interface MP4VideoTrack extends MP4MediaTrack {
+        video: MP4VideoData;
+    }
+
+    interface MP4AudioData {
+        sample_rate: number;
+        channel_count: number;
+        sample_size: number;
+    }
+
+    interface MP4AudioTrack extends MP4MediaTrack {
+        audio: MP4AudioData;
+    }
+
+    type MP4Track = MP4VideoTrack | MP4AudioTrack;
+
+    export interface MP4Info {
+        duration: number;
+        timescale: number;
+        fragment_duration: number;
+        isFragmented: boolean;
+        isProgressive: boolean;
+        hasIOD: boolean;
+        brands: string[];
+        created: Date;
+        modified: Date;
+        tracks: MP4Track[];
+        mime: string;
+        audioTracks: MP4AudioTrack[];
+        videoTracks: MP4VideoTrack[];
+    }
+
+    export type MP4ArrayBuffer = ArrayBuffer & {fileStart: number};
+
+    export interface MP4File {
+        onMoovStart?: () => void;
+        onReady?: (info: MP4Info) => void;
+        onError?: (e: string) => void;
+        onSamples?: (id: number, user: any, samples: Sample[]) => void;
+
+        appendBuffer(data: MP4ArrayBuffer): number;
+        start(): void;
+        stop(): void;
+        flush(): void;
+
+        setExtractionOptions(id: number, user: any, options: ExtractionOptions): void;
+    }
+
+    export function createFile(): MP4File;
+
+    export interface Sample {
+        number: number;
+        track_id: number;
+        timescale: number;
+        description_index: number;
+        description: any;
+        data: ArrayBuffer;
+        size: number;
+        alreadyRead: number;
+        duration: number;
+        cts: number;
+        dts: number;
+        is_sync: boolean;
+        is_leading: number;
+        depends_on: number;
+        is_depended_on: number;
+        has_redundancy: number;
+        degration_priority: number;
+        offset: number;
+        subsamples: any;
+    }
+
+    export interface ExtractionOptions {
+        nbSamples: number;
+    }
+
+    const BIG_ENDIAN: boolean;
+    const LITTLE_ENDIAN: boolean;
+
+    export class DataStream {
+        constructor(buffer: ArrayBuffer, byteOffset?: number, littleEndian?: boolean);
+        getPosition(): number;
+
+        get byteLength(): number;
+        get buffer(): ArrayBuffer;
+        set buffer(v: ArrayBuffer);
+        get byteOffset(): number;
+        set byteOffset(v: number);
+        get dataView(): DataView;
+        set dataView(v: DataView);
+
+        seek(pos: number): void;
+        isEof(): boolean;
+
+        mapUint8Array(length: number): Uint8Array;
+        readInt32Array(length: number, littleEndian: boolean): Int32Array;
+        readInt16Array(length: number, littleEndian: boolean): Int16Array;
+        readInt8(length: number): Int8Array;
+        readUint32Array(length: number, littleEndian: boolean): Uint32Array;
+        readUint16Array(length: number, littleEndian: boolean): Uint16Array;
+        readUint8(length: number): Uint8Array;
+        readFloat64Array(length: number, littleEndian: boolean): Float64Array;
+        readFloat32Array(length: number, littleEndian: boolean): Float32Array;
+
+        readInt32(littleEndian: boolean): number;
+        readInt16(littleEndian: boolean): number;
+        readInt8(): number;
+        readUint32(littleEndian: boolean): number;
+        readUint16(littleEndian: boolean): number;
+        readUint8(): number;
+        readFloat32(littleEndian: boolean): number;
+        readFloat64(littleEndian: boolean): number;
+
+        endianness: boolean;
+
+        memcpy(dst: ArrayBufferLike, dstOffset: number, src: ArrayBufferLike, srcOffset: number, byteLength: number): void;
+
+        // TODO I got bored porting the remaining functions
+    }
+
+    export { };
+}

player/src/player.ts

@@ -1,374 +0,0 @@
-import { Source } from "./source"
-import { StreamReader, StreamWriter } from "./stream"
-import { InitParser } from "./init"
-import { Segment } from "./segment"
-import { Track } from "./track"
-import { Message, MessageInit, MessageSegment } from "./message"
-
-///<reference path="./types/webtransport.d.ts"/>
-
-export interface PlayerInit {
-	url: string;
-
-	videoRef: HTMLVideoElement;
-	statsRef: HTMLElement;
-	throttleRef: HTMLElement;
-}
-
-/*
-*/
-
-
-export class Player {
-	mediaSource: MediaSource;
-
-	init: Map<string, InitParser>;
-	audio: Track;
-	video: Track;
-
-	quic: Promise<WebTransport>;
-	api: Promise<WritableStream>;
-
-	// References to elements in the DOM
-	vidRef: HTMLVideoElement; // The video element itself
-	statsRef: HTMLElement; // The stats div
-	throttleRef: HTMLElement; // The throttle button
-	throttleCount: number; // number of times we've clicked the button in a row
-
-	interval: number;
-
-	timeRef?: DOMHighResTimeStamp;
-
-	constructor(props: PlayerInit) {
-		this.vidRef = props.videoRef
-		this.statsRef = props.statsRef
-		this.throttleRef = props.throttleRef
-		this.throttleCount = 0
-
-		this.mediaSource = new MediaSource()
-		this.vidRef.src = URL.createObjectURL(this.mediaSource)
-
-		this.init = new Map()
-		this.audio = new Track(new Source(this.mediaSource));
-		this.video = new Track(new Source(this.mediaSource));
-
-		this.interval = setInterval(this.tick.bind(this), 100)
-		this.vidRef.addEventListener("waiting", this.tick.bind(this))
-
-		this.quic = this.connect(props.url)
-
-		// Create a unidirectional stream for all of our messages
-		this.api = this.quic.then((q) => {
-			return q.createUnidirectionalStream()
-		})
-
-		// async functions
-		this.receiveStreams()
-
-		// Limit to 4Mb/s
-		this.sendThrottle()
-	}
-
-	async close() {
-		clearInterval(this.interval);
-		(await this.quic).close()
-	}
-
-	async connect(url: string): Promise<WebTransport> {
-		// TODO remove this when WebTransport supports the system CA pool
-		const fingerprintURL = new URL(url);
-		fingerprintURL.pathname = "/fingerprint"
-
-		const response = await fetch(fingerprintURL)
-		if (!response.ok) {
-			throw new Error('failed to get server fingerprint');
-		}
-
-		const hex = await response.text()
-
-		// Convert the hex to binary.
-		let fingerprint = [];
-		for (let c = 0; c < hex.length; c += 2) {
-			fingerprint.push(parseInt(hex.substring(c, c+2), 16));
-		}
-
-		//const fingerprint = Uint8Array.from(atob(hex), c => c.charCodeAt(0))
-
-		const quic = new WebTransport(url, { 
-			"serverCertificateHashes": [{
-				"algorithm": "sha-256",
-				"value": new Uint8Array(fingerprint),
-			}]
-		})
-
-		await quic.ready
-
-		return quic
-	}
-
-	async sendMessage(msg: any) {
-		const payload = JSON.stringify(msg)
-		const size = payload.length + 8
-
-		const stream = await this.api
-
-		const writer = new StreamWriter(stream)
-		await writer.uint32(size)
-		await writer.string("warp")
-		await writer.string(payload)
-		writer.release()
-	}
-
-	throttle() {
-		// Throttle is incremented each time we click the throttle button
-		this.throttleCount += 1
-		this.sendThrottle()
-
-		// After 5 seconds disable the throttling
-		setTimeout(() => {
-			this.throttleCount -= 1
-			this.sendThrottle()
-		}, 5000)
-	}
-
-	sendThrottle() {
-		let rate = 0;
-
-		if (this.throttleCount > 0) {
-			// TODO detect the incoming bitrate instead of hard-coding
-			// Right shift by throttle to divide by 2,4,8,16,etc each time
-			const bitrate = 4 * 1024 * 1024 // 4Mb/s
-
-			rate = bitrate >> (this.throttleCount-1)
-
-			const str = formatBits(rate) + "/s"
-			this.throttleRef.textContent = `Throttle: ${ str }`;
-		} else {
-			this.throttleRef.textContent = "Throttle: none";
-		}
-
-		// Send the server a message to fake network congestion.
-		this.sendMessage({
-			"debug": {
-				max_bitrate: rate,
-			},
-		})
-	}
-
-	tick() {
-		// Try skipping ahead if there's no data in the current buffer.
-		this.trySeek()
-
-		// Try skipping video if it would fix any desync.
-		this.trySkip()
-
-		// Update the stats at the end
-		this.updateStats()
-	}
-
-	goLive() {
-		const ranges = this.vidRef.buffered
-		if (!ranges.length) {
-			return
-		}
-
-		this.vidRef.currentTime = ranges.end(ranges.length-1);
-		this.vidRef.play();
-	}
-
-	// Try seeking ahead to the next buffered range if there's a gap
-	trySeek() {
-		if (this.vidRef.readyState > 2) { // HAVE_CURRENT_DATA
-			// No need to seek
-			return
-		}
-
-		const ranges = this.vidRef.buffered
-		if (!ranges.length) {
-			// Video has not started yet
-			return
-		}
-
-		for (let i = 0; i < ranges.length; i += 1) {
-			const pos = ranges.start(i)
-
-			if (this.vidRef.currentTime >= pos) {
-				// This would involve seeking backwards
-				continue
-			}
-
-			console.warn("seeking forward", pos - this.vidRef.currentTime)
-
-			this.vidRef.currentTime = pos
-			return
-		}
-	}
-
-	// Try dropping video frames if there is future data available.
-	trySkip() {
-		let playhead: number | undefined
-
-		if (this.vidRef.readyState > 2) {
-			// If we're not buffering, only skip video if it's before the current playhead
-			playhead = this.vidRef.currentTime
-		}
-
-		this.video.advance(playhead)
-	}
-
-	async receiveStreams() {
-		const q = await this.quic
-		const streams = q.incomingUnidirectionalStreams.getReader()
-
-		while (true) {
-			const result = await streams.read()
-			if (result.done) break
-
-			const stream = result.value
-			this.handleStream(stream) // don't await
-		}
-	}
-
-	async handleStream(stream: ReadableStream) {
-		let r = new StreamReader(stream.getReader())
-
-		while (!await r.done()) {
-			const size = await r.uint32();
-			const typ = new TextDecoder('utf-8').decode(await r.bytes(4));
-
-			if (typ != "warp") throw "expected warp atom"
-			if (size < 8) throw "atom too small"
-
-			const payload = new TextDecoder('utf-8').decode(await r.bytes(size - 8));
-			const msg = JSON.parse(payload) as Message
-
-			if (msg.init) {
-				return this.handleInit(r, msg.init)
-			} else if (msg.segment) {
-				return this.handleSegment(r, msg.segment)
-			}
-		}
-	}
-
-	async handleInit(stream: StreamReader, msg: MessageInit) {
-		let init = this.init.get(msg.id);
-		if (!init) {
-			init = new InitParser()
-			this.init.set(msg.id, init)
-		}
-
-		while (1) {
-			const data = await stream.read()
-			if (!data) break
-
-			init.push(data)
-		}
-	}
-
-	async handleSegment(stream: StreamReader, msg: MessageSegment) {
-		let pending = this.init.get(msg.init);
-		if (!pending) {
-			pending = new InitParser()
-			this.init.set(msg.init, pending)
-		}
-
-		// Wait for the init segment to be fully received and parsed
-		const init = await pending.ready;
-
-		let track: Track;
-		if (init.info.videoTracks.length) {
-			track = this.video
-		} else {
-			track = this.audio
-		}
-
-		const segment = new Segment(track.source, init, msg.timestamp)
-
-		// The track is responsible for flushing the segments in order
-		track.add(segment)
-
-		/* TODO I'm not actually sure why this code doesn't work; something trips up the MP4 parser
-			while (1) {
-				const data = await stream.read()
-				if (!data) break
-
-				segment.push(data)
-				track.flush() // Flushes if the active segment has samples
-			}
-		*/
-
-		// One day I'll figure it out; until then read one top-level atom at a time
-		while (!await stream.done()) {
-			const raw = await stream.peek(4)
-			const size = new DataView(raw.buffer, raw.byteOffset, raw.byteLength).getUint32(0)
-			const atom = await stream.bytes(size)
-
-			segment.push(atom)
-			track.flush() // Flushes if the active segment has new samples
-		}
-
-		segment.finish()
-	}
-
-	updateStats() {
-		for (const child of this.statsRef.children) {
-			if (child.className == "audio buffer") {
-				const ranges: any = (this.audio) ? this.audio.buffered() : { length: 0 }
-				this.visualizeBuffer(child as HTMLElement, ranges)
-			} else if (child.className == "video buffer") {
-				const ranges: any = (this.video) ? this.video.buffered() : { length: 0 }
-				this.visualizeBuffer(child as HTMLElement, ranges)
-			}
-		}
-	}
-
-	visualizeBuffer(element: HTMLElement, ranges: TimeRanges) {
-		const children = element.children
-		const max = 5
-
-		let index = 0
-		let prev = 0
-
-		for (let i = 0; i < ranges.length; i += 1) {
-			let start = ranges.start(i) - this.vidRef.currentTime
-			let end = ranges.end(i) - this.vidRef.currentTime
-
-			if (end < 0 || start > max) {
-				continue
-			}
-
-			let fill: HTMLElement;
-
-			if (index < children.length) {
-				fill = children[index] as HTMLElement;
-			} else {
-				fill = document.createElement("div")
-				element.appendChild(fill)
-			}
-
-			fill.className = "fill"
-			fill.innerHTML = end.toFixed(2)
-			fill.setAttribute('style', "left: " + (100 * Math.max(start, 0) / max) + "%; right: " + (100 - 100 * Math.min(end, max) / max) + "%")
-			index += 1
-
-			prev = end
-		}
-
-		for (let i = index; i < children.length; i += 1) {
-			element.removeChild(children[i])
-		}
-	}
-}
-
-// https://stackoverflow.com/questions/15900485/correct-way-to-convert-size-in-bytes-to-kb-mb-gb-in-javascript
-function formatBits(bits: number, decimals: number = 1) {
-	if (bits === 0) return '0 bits';
-
-	const k = 1024;
-	const dm = decimals < 0 ? 0 : decimals;
-	const sizes = ['b', 'Kb', 'Mb', 'Gb', 'Tb', 'Pb', 'Eb', 'Zb', 'Yb'];
-
-	const i = Math.floor(Math.log(bits) / Math.log(k));
-
-	return parseFloat((bits / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
-}

player/src/player/index.ts

@@ -0,0 +1,46 @@
+import Audio from "../audio"
+import Transport from "../transport"
+import Video from "../video"
+
+export interface PlayerInit {
+	url: string;
+	canvas: HTMLCanvasElement;
+}
+
+export default class Player {
+	audio: Audio;
+	video: Video;
+    transport: Transport;
+
+	constructor(props: PlayerInit) {
+		this.audio = new Audio()
+		this.video = new Video({
+			canvas: props.canvas.transferControlToOffscreen(),
+		})
+
+        this.transport = new Transport({
+            url: props.url,
+            audio: this.audio,
+            video: this.video,
+        })
+	}
+
+	async close() {
+        this.transport.close()
+	}
+
+	async connect(url: string) {
+        await this.transport.connect(url)
+	}
+
+	play() {
+		this.audio.play({})
+		//this.video.play()
+	}
+
+    onMessage(msg: any) {
+        if (msg.sync) {
+            msg.sync
+        }
+    }
+}

player/src/segment.ts

@@ -1,150 +0,0 @@
-import { Source } from "./source"
-import { Init } from "./init"
-import { MP4New, MP4File, MP4Sample, MP4Stream, MP4Parser, MP4ArrayBuffer } from "./mp4"
-
-// Manage a segment download, keeping a buffer of a single sample to potentially rewrite the duration.
-export class Segment {
-	source: Source; // The SourceBuffer used to decode media.
-	offset: number; // The byte offset in the received file so far
-	samples: MP4Sample[]; // The samples ready to be flushed to the source.
-	timestamp: number; // The expected timestamp of the first sample in milliseconds
-	init: Init;
-
-	dts?: number;       // The parsed DTS of the first sample
-	timescale?: number; // The parsed timescale of the segment
-
-	input: MP4File; // MP4Box file used to parse the incoming atoms.
-	output: MP4File; // MP4Box file used to write the outgoing atoms after modification.
-
-	done: boolean; // The segment has been completed
-
-	constructor(source: Source, init: Init, timestamp: number) {
-		this.source = source
-		this.offset = 0
-		this.done = false
-		this.timestamp = timestamp
-		this.init = init
-
-		this.input = MP4New();
-		this.output = MP4New();
-		this.samples = [];
-
-		this.input.onReady = (info: any) => {
-			this.input.setExtractionOptions(info.tracks[0].id, {}, { nbSamples: 1 });
-
-			this.input.onSamples = this.onSamples.bind(this)
-			this.input.start();
-		}
-
-		// We have to reparse the init segment to work with mp4box
-		for (let i = 0; i < init.raw.length; i += 1) {
-			this.offset = this.input.appendBuffer(init.raw[i])
-
-			// Also populate the output with our init segment so it knows about tracks
-			this.output.appendBuffer(init.raw[i])
-		}
-
-		this.input.flush()
-		this.output.flush()
-	}
-
-	push(data: Uint8Array) {
-		if (this.done) return; // ignore new data after marked done
-
-		// Make a copy of the atom because mp4box only accepts an ArrayBuffer unfortunately
-		let box = new Uint8Array(data.byteLength);
-		box.set(data)
-
-		// and for some reason we need to modify the underlying ArrayBuffer with offset
-		let buffer = box.buffer as MP4ArrayBuffer
-		buffer.fileStart = this.offset
-
-		// Parse the data
-		this.offset = this.input.appendBuffer(buffer)
-		this.input.flush()
-	}
-
-	onSamples(id: number, user: any, samples: MP4Sample[]) {
-		if (!samples.length) return;
-
-		if (this.dts === undefined) {
-			this.dts = samples[0].dts;
-			this.timescale = samples[0].timescale;
-		}
-
-		// Add the samples to a queue
-		this.samples.push(...samples)
-	}
-
-	// Flushes any pending samples, returning true if the stream has finished.
-	flush(): boolean {
-		let stream = new MP4Stream(new ArrayBuffer(0), 0, false); // big-endian
-
-		while (this.samples.length) {
-			// Keep a single sample if we're not done yet
-			if (!this.done && this.samples.length < 2) break;
-
-			const sample = this.samples.shift()
-			if (!sample) break;
-
-			let moof = this.output.createSingleSampleMoof(sample);
-			moof.write(stream);
-
-			// adjusting the data_offset now that the moof size is known
-			moof.trafs[0].truns[0].data_offset = moof.size+8; //8 is mdat header
-			stream.adjustUint32(moof.trafs[0].truns[0].data_offset_position, moof.trafs[0].truns[0].data_offset);
-
-			// @ts-ignore
-			var mdat = new MP4Parser.mdatBox();
-			mdat.data = sample.data;
-			mdat.write(stream);
-		}
-
-		this.source.initialize(this.init)
-		this.source.append(stream.buffer as ArrayBuffer)
-
-		return this.done
-	}
-
-	// The segment has completed
-	finish() {
-		this.done = true
-		this.flush()
-
-		// Trim the buffer to 30s long after each segment.
-		this.source.trim(30)
-	}
-
-	// Extend the last sample so it reaches the provided timestamp
-	skipTo(pts: number) {
-		if (this.samples.length == 0) return
-		let last = this.samples[this.samples.length-1]
-
-		const skip = pts - (last.dts + last.duration);
-
-		if (skip == 0) return;
-		if (skip < 0) throw "can't skip backwards"
-
-		last.duration += skip
-
-		if (this.timescale) {
-			console.warn("skipping video", skip / this.timescale)
-		}
-	}
-
-	buffered() {
-		// Ignore if we have a single sample
-		if (this.samples.length <= 1) return undefined;
-		if (!this.timescale) return undefined;
-
-		const first = this.samples[0];
-		const last = this.samples[this.samples.length-1]
-
-
-		return {
-			length: 1,
-			start: first.dts / this.timescale,
-			end: (last.dts + last.duration) / this.timescale,
-		}
-	}
-}

player/src/source.ts

@@ -1,147 +0,0 @@
-import { Init } from "./init"
-
-// Create a SourceBuffer with convenience methods
-export class Source {
-	sourceBuffer?: SourceBuffer;
-	mediaSource: MediaSource;
-	queue: Array<SourceInit | SourceData | SourceTrim>;
-	init?: Init;
-
-	constructor(mediaSource: MediaSource) {
-		this.mediaSource = mediaSource;
-		this.queue = [];
-	}
-
-	// (re)initialize the source using the provided init segment.
-	initialize(init: Init) {
-		// Check if the init segment is already in the queue.
-		for (let i = this.queue.length - 1; i >= 0; i--) {
-			if ((this.queue[i] as SourceInit).init == init) {
-				// Already queued up.
-				return
-			}
-		}
-
-		// Check if the init segment has already been applied.
-		if (this.init == init) {
-			return
-		}
-
-		// Add the init segment to the queue so we call addSourceBuffer or changeType
-		this.queue.push({
-			kind: "init",
-			init: init,
-		})
-
-		for (let i = 0; i < init.raw.length; i += 1) {
-			this.queue.push({
-				kind: "data",
-				data: init.raw[i],
-			})
-		}
-
-		this.flush()
-	}
-
-	// Append the segment data to the buffer.
-	append(data: Uint8Array | ArrayBuffer) {
-		this.queue.push({
-			kind: "data",
-			data: data,
-		})
-
-		this.flush()
-	}
-
-	// Return the buffered range.
-	buffered() {
-		if (!this.sourceBuffer) {
-			return { length: 0 }
-		}
-
-		return this.sourceBuffer.buffered
-	}
-
-	// Delete any media older than x seconds from the buffer.
-	trim(duration: number) {
-		this.queue.push({
-			kind: "trim",
-			trim: duration,
-		})
-
-		this.flush()
-	}
-
-	// Flush any queued instructions
-	flush() {
-		while (1) {
-			// Check if the buffer is currently busy.
-			if (this.sourceBuffer && this.sourceBuffer.updating) {
-				break;
-			}
-
-			// Process the next item in the queue.
-			const next = this.queue.shift()
-			if (!next) {
-				break;
-			}
-
-			switch (next.kind) {
-			case "init":
-				this.init = next.init;
-
-				if (!this.sourceBuffer) {
-					// Create a new source buffer.
-					this.sourceBuffer = this.mediaSource.addSourceBuffer(this.init.info.mime)
-
-					// Call flush automatically after each update finishes.
-					this.sourceBuffer.addEventListener('updateend', this.flush.bind(this))
-				} else {
-					this.sourceBuffer.changeType(next.init.info.mime)
-				}
-
-				break;
-			case "data":
-				if (!this.sourceBuffer) {
-					throw "failed to call initailize before append"
-				}
-
-				this.sourceBuffer.appendBuffer(next.data)
-
-				break;
-			case "trim":
-				if (!this.sourceBuffer) {
-					throw "failed to call initailize before trim"
-				}
-
-				const end = this.sourceBuffer.buffered.end(this.sourceBuffer.buffered.length - 1) - next.trim;
-				const start = this.sourceBuffer.buffered.start(0)
-
-				if (end > start) {
-					this.sourceBuffer.remove(start, end)
-				}
-
-				break;
-			default:
-				throw "impossible; unknown SourceItem"
-			}
-		}
-	}
-}
-
-interface SourceItem {}
-
-class SourceInit implements SourceItem {
-  kind!: "init";
-  init!: Init;
-}
-
-class SourceData implements SourceItem {
-  kind!: "data";
-  data!: Uint8Array | ArrayBuffer;
-}
-
-class SourceTrim implements SourceItem {
-	kind!: "trim";
-	trim!: number;
-}

player/src/stream/index.ts

@@ -0,0 +1,2 @@
+export { default as Reader } from "./reader"
+export { default as Writer } from "./writer"

player/src/stream/reader.ts

@@ -1,33 +1,64 @@
 // Reader wraps a stream and provides convience methods for reading pieces from a stream
-export class StreamReader {
-	reader: ReadableStreamDefaultReader; // TODO make a separate class without promises when null
+export default class Reader {
+	reader: ReadableStream;
 	buffer: Uint8Array;
 
-	constructor(reader: ReadableStreamDefaultReader, buffer: Uint8Array = new Uint8Array(0)) {
+	constructor(reader: ReadableStream, buffer: Uint8Array = new Uint8Array(0)) {
 		this.reader = reader
 		this.buffer = buffer
 	}
 
-	// TODO implementing pipeTo seems more reasonable than releasing the lock
-	release() {
-		this.reader.releaseLock()
-	}
-
 	// Returns any number of bytes
 	async read(): Promise<Uint8Array | undefined> {
+
 		if (this.buffer.byteLength) {
 			const buffer = this.buffer;
 			this.buffer = new Uint8Array()
 			return buffer
 		}
 
-		const result = await this.reader.read()
+		const r = this.reader.getReader()
+		const result = await r.read()
+
+		r.releaseLock()
+
 		return result.value
 	}
 
+	async readAll(): Promise<Uint8Array> {
+		const r = this.reader.getReader()
+
+		while (1) {
+			const result = await r.read()
+			if (result.done) {
+				break
+			}
+
+			const buffer = new Uint8Array(result.value)
+
+			if (this.buffer.byteLength == 0) {
+				this.buffer = buffer
+			} else {
+				const temp = new Uint8Array(this.buffer.byteLength + buffer.byteLength)
+				temp.set(this.buffer)
+				temp.set(buffer, this.buffer.byteLength)
+				this.buffer = temp
+			}
+		}
+
+		const result = this.buffer
+		this.buffer = new Uint8Array()
+
+		r.releaseLock()
+
+		return result
+	}
+
 	async bytes(size: number): Promise<Uint8Array> {
+		const r = this.reader.getReader()
+
 		while (this.buffer.byteLength < size) {
-			const result = await this.reader.read()
+			const result = await r.read()
 			if (result.done) {
 				throw "short buffer"
 			}
@@ -47,12 +78,16 @@ export class StreamReader {
 		const result = new Uint8Array(this.buffer.buffer, this.buffer.byteOffset, size)
 		this.buffer = new Uint8Array(this.buffer.buffer, this.buffer.byteOffset + size)
 
+		r.releaseLock()
+
 		return result
 	}
 
 	async peek(size: number): Promise<Uint8Array> {
+		const r = this.reader.getReader()
+
 		while (this.buffer.byteLength < size) {
-			const result = await this.reader.read()
+			const result = await r.read()
 			if (result.done) {
 				throw "short buffer"
 			}
@@ -69,7 +104,11 @@ export class StreamReader {
 			}
 		}
 
-		return new Uint8Array(this.buffer.buffer, this.buffer.byteOffset, size)
+		const result = new Uint8Array(this.buffer.buffer, this.buffer.byteOffset, size)
+
+		r.releaseLock()
+
+		return result
 	}
 
 	async view(size: number): Promise<DataView> {
@@ -151,104 +190,3 @@ export class StreamReader {
 		}
 	}
 }
-
-// StreamWriter wraps a stream and writes chunks of data
-export class StreamWriter {
-	buffer: ArrayBuffer;
-	writer: WritableStreamDefaultWriter;
-
-	constructor(stream: WritableStream) {
-		this.buffer = new ArrayBuffer(8)
-		this.writer = stream.getWriter()
-	}
-
-	release() {
-		this.writer.releaseLock()
-	}
-
-	async close() {
-		return this.writer.close()
-	}
-
-	async uint8(v: number) {
-		const view = new DataView(this.buffer, 0, 1)
-		view.setUint8(0, v)
-		return this.writer.write(view)
-	}
-
-	async uint16(v: number) {
-		const view = new DataView(this.buffer, 0, 2)
-		view.setUint16(0, v)
-		return this.writer.write(view)
-	}
-
-	async uint24(v: number) {
-		const v1 = (v >> 16) & 0xff
-		const v2 = (v >> 8) & 0xff
-		const v3 = (v) & 0xff
-
-		const view = new DataView(this.buffer, 0, 3)
-		view.setUint8(0, v1)
-		view.setUint8(1, v2)
-		view.setUint8(2, v3)
-
-		return this.writer.write(view)
-	}
-
-	async uint32(v: number) {
-		const view = new DataView(this.buffer, 0, 4)
-		view.setUint32(0, v)
-		return this.writer.write(view)
-	}
-
-	async uint52(v: number) {
-		if (v > Number.MAX_SAFE_INTEGER) {
-			throw "value too large"
-		}
-
-		this.uint64(BigInt(v))
-	}
-
-	async vint52(v: number) {
-		if (v > Number.MAX_SAFE_INTEGER) {
-			throw "value too large"
-		}
-
-		if (v < (1 << 6)) {
-			return this.uint8(v)
-		} else if (v < (1 << 14)) {
-			return this.uint16(v|0x4000)
-		} else if (v < (1 << 30)) {
-			return this.uint32(v|0x80000000)
-		} else {
-			return this.uint64(BigInt(v) | 0xc000000000000000n)
-		}
-	}
-
-	async uint64(v: bigint) {
-		const view = new DataView(this.buffer, 0, 8)
-		view.setBigUint64(0, v)
-		return this.writer.write(view)
-	}
-
-	async vint64(v: bigint) {
-		if (v < (1 << 6)) {
-			return this.uint8(Number(v))
-		} else if (v < (1 << 14)) {
-			return this.uint16(Number(v)|0x4000)
-		} else if (v < (1 << 30)) {
-			return this.uint32(Number(v)|0x80000000)
-		} else {
-			return this.uint64(v | 0xc000000000000000n)
-		}
-	}
-
-	async bytes(buffer: ArrayBuffer) {
-		return this.writer.write(buffer)
-	}
-
-	async string(str: string) {
-		const data = new TextEncoder().encode(str)
-		return this.writer.write(data)
-	}
-}

player/src/stream/writer.ts

@@ -0,0 +1,100 @@
+// Writer wraps a stream and writes chunks of data
+export default class Writer {
+	buffer: ArrayBuffer;
+	writer: WritableStreamDefaultWriter;
+
+	constructor(stream: WritableStream) {
+		this.buffer = new ArrayBuffer(8)
+		this.writer = stream.getWriter()
+	}
+
+	release() {
+		this.writer.releaseLock()
+	}
+
+	async close() {
+		return this.writer.close()
+	}
+
+	async uint8(v: number) {
+		const view = new DataView(this.buffer, 0, 1)
+		view.setUint8(0, v)
+		return this.writer.write(view)
+	}
+
+	async uint16(v: number) {
+		const view = new DataView(this.buffer, 0, 2)
+		view.setUint16(0, v)
+		return this.writer.write(view)
+	}
+
+	async uint24(v: number) {
+		const v1 = (v >> 16) & 0xff
+		const v2 = (v >> 8) & 0xff
+		const v3 = (v) & 0xff
+
+		const view = new DataView(this.buffer, 0, 3)
+		view.setUint8(0, v1)
+		view.setUint8(1, v2)
+		view.setUint8(2, v3)
+
+		return this.writer.write(view)
+	}
+
+	async uint32(v: number) {
+		const view = new DataView(this.buffer, 0, 4)
+		view.setUint32(0, v)
+		return this.writer.write(view)
+	}
+
+	async uint52(v: number) {
+		if (v > Number.MAX_SAFE_INTEGER) {
+			throw "value too large"
+		}
+
+		this.uint64(BigInt(v))
+	}
+
+	async vint52(v: number) {
+		if (v > Number.MAX_SAFE_INTEGER) {
+			throw "value too large"
+		}
+
+		if (v < (1 << 6)) {
+			return this.uint8(v)
+		} else if (v < (1 << 14)) {
+			return this.uint16(v|0x4000)
+		} else if (v < (1 << 30)) {
+			return this.uint32(v|0x80000000)
+		} else {
+			return this.uint64(BigInt(v) | 0xc000000000000000n)
+		}
+	}
+
+	async uint64(v: bigint) {
+		const view = new DataView(this.buffer, 0, 8)
+		view.setBigUint64(0, v)
+		return this.writer.write(view)
+	}
+
+	async vint64(v: bigint) {
+		if (v < (1 << 6)) {
+			return this.uint8(Number(v))
+		} else if (v < (1 << 14)) {
+			return this.uint16(Number(v)|0x4000)
+		} else if (v < (1 << 30)) {
+			return this.uint32(Number(v)|0x80000000)
+		} else {
+			return this.uint64(v | 0xc000000000000000n)
+		}
+	}
+
+	async bytes(buffer: ArrayBuffer) {
+		return this.writer.write(buffer)
+	}
+
+	async string(str: string) {
+		const data = new TextEncoder().encode(str)
+		return this.writer.write(data)
+	}
+}

player/src/track.ts

@@ -1,124 +0,0 @@
-import { Source } from "./source"
-import { Segment } from "./segment"
-import { TimeRange } from "./util"
-
-// An audio or video track that consists of multiple sequential segments.
-//
-// Instead of buffering, we want to drop video while audio plays uninterupted.
-// Chrome actually plays up to 3s of audio without video before buffering when in low latency mode.
-// Unforuntately, this does not recover correctly when there are gaps (pls fix).
-// Our solution is to flush segments in decode order, buffering a single additional frame.
-// We extend the duration of the buffered frame and flush it to cover any gaps.
-export class Track {
-	source: Source;
-	segments: Segment[];
-
-	constructor(source: Source) {
-		this.source = source;
-		this.segments = [];
-	}
-
-	add(segment: Segment) {
-		// TODO don't add if the segment is out of date already
-		this.segments.push(segment)
-
-		// Sort by timestamp ascending
-		// NOTE: The timestamp is in milliseconds, and we need to parse the media to get the accurate PTS/DTS.
-		this.segments.sort((a: Segment, b: Segment): number => {
-			return a.timestamp - b.timestamp
-		})
-	}
-
-	buffered(): TimeRanges {
-		let ranges: TimeRange[] = []
-
-		const buffered = this.source.buffered() as TimeRanges
-		for (let i = 0; i < buffered.length; i += 1) {
-			// Convert the TimeRanges into an oject we can modify
-			ranges.push({
-				start: buffered.start(i),
-				end: buffered.end(i)
-			})
-		}
-
-		// Loop over segments and add in their ranges, merging if possible.
-		for (let segment of this.segments) {
-			const buffered = segment.buffered()
-			if (!buffered) continue;
-
-			if (ranges.length) {
-				// Try to merge with an existing range
-				const last = ranges[ranges.length-1];
-				if (buffered.start < last.start) {
-					// Network buffer is old; ignore it
-					continue
-				}
-
-				// Extend the end of the last range instead of pushing
-				if (buffered.start <= last.end && buffered.end > last.end) {
-					last.end = buffered.end
-					continue
-				}
-			}
-
-			ranges.push(buffered)
-		}
-
-		// TODO typescript
-		return {
-			length: ranges.length,
-			start: (x) => { return ranges[x].start },
-			end: (x) => { return ranges[x].end },
-		}
-	}
-
-	flush() {
-		while (1) {
-			if (!this.segments.length) break
-
-			const first = this.segments[0]
-			const done = first.flush()
-			if (!done) break
-
-			this.segments.shift()
-		}
-	}
-
-	// Given the current playhead, determine if we should drop any segments
-	// If playhead is undefined, it means we're buffering so skip to anything now.
-	advance(playhead: number | undefined) {
-		if (this.segments.length < 2) return
-
-		while (this.segments.length > 1) {
-			const current = this.segments[0];
-			const next = this.segments[1];
-
-			if (next.dts === undefined || next.timescale == undefined) {
-				// No samples have been parsed for the next segment yet.
-				break
-			}
-
-			if (current.dts === undefined) {
-				// No samples have been parsed for the current segment yet.
-				// We can't cover the gap by extending the sample so we have to seek.
-				// TODO I don't think this can happen, but I guess we have to seek past the gap.
-				break
-			}
-
-			if (playhead !== undefined) {
-				// Check if the next segment has playable media now.
-				// Otherwise give the current segment more time to catch up.
-				if ((next.dts / next.timescale) > playhead) {
-					return
-				}
-			}
-
-			current.skipTo(next.dts || 0) // tell typescript that it's not undefined; we already checked
-			current.finish()
-
-			// TODO cancel the QUIC stream to save bandwidth
-
-			this.segments.shift()
-		}
-	}
-}

player/src/transport/.gitignore

@@ -0,0 +1 @@
+fingerprint.hex

player/src/transport/index.ts

@@ -0,0 +1,175 @@
+import * as Message from "./message"
+import * as Stream from "../stream"
+import * as MP4 from "../mp4"
+
+import Audio from "../audio"
+import Video from "../video"
+
+// @ts-ignore bundler embeds data
+import fingerprint from 'bundle-text:./fingerprint.hex';
+
+export interface TransportInit {
+	url: string;
+	audio: Audio;
+	video: Video;
+}
+
+export default class Transport {
+	quic: Promise<WebTransport>;
+	api: Promise<WritableStream>;
+    tracks: Map<string, MP4.InitParser>
+
+	audio: Audio;
+	video: Video;
+
+	constructor(props: TransportInit) {
+		this.tracks = new Map();
+
+		this.audio = props.audio;
+		this.video = props.video;
+
+		this.quic = this.connect(props.url)
+
+		// Create a unidirectional stream for all of our messages
+		this.api = this.quic.then((q) => {
+			return q.createUnidirectionalStream()
+		})
+
+		// async functions
+		this.receiveStreams()
+	}
+
+	async close() {
+		(await this.quic).close()
+	}
+
+	async connect(url: string): Promise<WebTransport> {
+		// Convert the hex to binary.
+		let hash = [];
+		for (let c = 0; c < fingerprint.length-1; c += 2) {
+			hash.push(parseInt(fingerprint.substring(c, c+2), 16));
+		}
+
+		const quic = new WebTransport(url, {
+			"serverCertificateHashes": [{
+				"algorithm": "sha-256",
+				"value": new Uint8Array(hash),
+			}]
+		})
+
+		await quic.ready
+
+		return quic
+	}
+
+	async sendMessage(msg: any) {
+		const payload = JSON.stringify(msg)
+		const size = payload.length + 8
+
+		const stream = await this.api
+
+		const writer = new Stream.Writer(stream)
+		await writer.uint32(size)
+		await writer.string("warp")
+		await writer.string(payload)
+		writer.release()
+	}
+
+	async receiveStreams() {
+		const q = await this.quic
+		const streams = q.incomingUnidirectionalStreams.getReader()
+
+		while (true) {
+			const result = await streams.read()
+			if (result.done) break
+
+			const stream = result.value
+			this.handleStream(stream) // don't await
+		}
+	}
+
+	async handleStream(stream: ReadableStream) {
+		let r = new Stream.Reader(stream)
+
+		while (!await r.done()) {
+			const size = await r.uint32();
+			const typ = new TextDecoder('utf-8').decode(await r.bytes(4));
+
+			if (typ != "warp") throw "expected warp atom"
+			if (size < 8) throw "atom too small"
+
+			const payload = new TextDecoder('utf-8').decode(await r.bytes(size - 8));
+			const msg = JSON.parse(payload)
+
+			if (msg.init) {
+				return this.handleInit(r, msg.init as Message.Init)
+			} else if (msg.segment) {
+				return this.handleSegment(r, msg.segment as Message.Segment)
+			}
+		}
+	}
+
+	async handleInit(stream: Stream.Reader, msg: Message.Init) {
+        let track = this.tracks.get(msg.id);
+        if (!track) {
+            track = new MP4.InitParser()
+            this.tracks.set(msg.id, track)
+        }
+
+        while (1) {
+            const data = await stream.read()
+            if (!data) break
+
+            track.push(data)
+        }
+
+		const info = await track.info
+
+        if (info.audioTracks.length + info.videoTracks.length != 1) {
+            throw new Error("expected a single track")
+        }
+
+		if (info.audioTracks.length) {
+			this.audio.init({
+				track: msg.id,
+				info: info,
+				raw: track.raw,
+			})
+		} else if (info.videoTracks.length) {
+			this.video.init({
+				track: msg.id,
+				info: info,
+				raw: track.raw,
+			})
+		} else {
+			throw new Error("init is neither audio nor video")
+		}
+	}
+
+	async handleSegment(stream: Stream.Reader, msg: Message.Segment) {
+        let track = this.tracks.get(msg.init);
+        if (!track) {
+            track = new MP4.InitParser()
+            this.tracks.set(msg.init, track)
+        }
+
+		// Wait until we learn if this is an audio or video track
+		const info = await track.info
+
+		if (info.audioTracks.length) {
+			this.audio.segment({
+				track: msg.init,
+				buffer: stream.buffer,
+				reader: stream.reader,
+			})
+		} else if (info.videoTracks.length) {
+			this.video.segment({
+				track: msg.init,
+				buffer: stream.buffer,
+				reader: stream.reader,
+			})
+		} else {
+			throw new Error("segment is neither audio nor video")
+		}
+	}
+}

player/src/transport/message.ts

@@ -1,13 +1,8 @@
-export interface Message {
-	init?: MessageInit
-	segment?: MessageSegment
-}
-
-export interface MessageInit {
+export interface Init {
 	id: string
 }
 
-export interface MessageSegment {
+export interface Segment {
 	init: string // id of the init segment
 	timestamp: number // presentation timestamp in milliseconds of the first sample
 	// TODO track would be nice

player/src/util.ts

@@ -1,4 +0,0 @@
-export interface TimeRange {
-	start: number;
-	end: number;
-}

player/src/util/deferred.ts

@@ -0,0 +1,16 @@
+export default class Deferred<T> {
+    promise: Promise<T>
+    resolve: (value: T | PromiseLike<T>) => void
+    reject: (value: T | PromiseLike<T>) => void
+  
+    constructor() {
+        // Set initial values so TS stops being annoying.
+        this.resolve = (value: T | PromiseLike<T>) => {};
+        this.reject = (value: T | PromiseLike<T>) => {};
+
+        this.promise = new Promise((resolve, reject) => {
+            this.resolve = resolve
+            this.reject = reject
+        })
+    }
+  }

player/src/util/index.ts

@@ -0,0 +1 @@
+export { default as Deferred } from "./deferred"

player/src/video/decoder.ts

@@ -0,0 +1,127 @@
+import * as Message from "./message";
+import * as MP4 from "../mp4"
+import * as Stream from "../stream"
+import * as Util from "../util"
+
+import Renderer from "./renderer"
+
+export default class Decoder {
+    // Store the init message for each track
+    tracks: Map<string, Util.Deferred<Message.Init>>
+    renderer: Renderer;
+
+    constructor(renderer: Renderer) {
+        this.tracks = new Map();
+        this.renderer = renderer;
+    }
+
+    async init(msg: Message.Init) {
+		let track = this.tracks.get(msg.track);
+		if (!track) {
+            track = new Util.Deferred()
+			this.tracks.set(msg.track, track)
+		}
+
+        if (msg.info.videoTracks.length != 1 || msg.info.audioTracks.length != 0) {
+            throw new Error("Expected a single video track")
+        }
+
+        track.resolve(msg)
+    }
+
+    async decode(msg: Message.Segment) {
+		let track = this.tracks.get(msg.track);
+		if (!track) {
+            track = new Util.Deferred()
+			this.tracks.set(msg.track, track)
+		}
+
+		// Wait for the init segment to be fully received and parsed
+		const init = await track.promise;
+        const info = init.info;
+        const video = info.videoTracks[0]
+
+        const decoder = new VideoDecoder({
+            output: (frame: VideoFrame) => {
+                this.renderer.emit(frame)
+            },
+            error: (err: Error) => {
+                console.warn(err)
+            }
+        });
+
+		const input = MP4.New();
+
+        input.onSamples = (id: number, user: any, samples: MP4.Sample[]) => {
+            for (let sample of samples) {
+                const timestamp = 1000 * sample.dts / sample.timescale // milliseconds
+
+                if (sample.is_sync) {
+                    // Configure the decoder using the AVC box for H.264
+                    const avcc = sample.description.avcC;
+                    const description = new MP4.Stream(new Uint8Array(avcc.size), 0, false)
+                    avcc.write(description)
+
+                    decoder.configure({
+                        codec: video.codec,
+                        codedHeight: video.track_height,
+                        codedWidth: video.track_width,
+                        description: description.buffer?.slice(8),
+                        // optimizeForLatency: true
+                    })
+                }
+
+                decoder.decode(new EncodedVideoChunk({
+                    data: sample.data,
+                    duration: sample.duration,
+                    timestamp: timestamp,
+                    type: sample.is_sync ? "key" : "delta",
+                }))
+            }
+        }
+
+		input.onReady = (info: any) => {
+			input.setExtractionOptions(info.tracks[0].id, {}, { nbSamples: 1 });
+			input.start();
+		}
+
+        // MP4box requires us to reparse the init segment unfortunately
+        let offset = 0;
+
+        for (let raw of init.raw) {
+            raw.fileStart = offset
+            input.appendBuffer(raw)
+        }
+
+        const stream = new Stream.Reader(msg.reader, msg.buffer)
+
+		/* TODO I'm not actually sure why this code doesn't work; something trips up the MP4 parser
+			while (1) {
+				const data = await stream.read()
+				if (!data) break
+
+				input.appendBuffer(data)
+				input.flush()
+			}
+		*/
+
+		// One day I'll figure it out; until then read one top-level atom at a time
+		while (!await stream.done()) {
+			const raw = await stream.peek(4)
+			const size = new DataView(raw.buffer, raw.byteOffset, raw.byteLength).getUint32(0)
+			const atom = await stream.bytes(size)
+
+            // Make a copy of the atom because mp4box only accepts an ArrayBuffer unfortunately
+            let box = new Uint8Array(atom.byteLength);
+            box.set(atom)
+
+            // and for some reason we need to modify the underlying ArrayBuffer with offset
+            let buffer = box.buffer as MP4.ArrayBuffer
+            buffer.fileStart = offset
+
+            // Parse the data
+            offset = input.appendBuffer(buffer)
+            input.flush()
+		}
+    }
+}

player/src/video/index.ts

@@ -0,0 +1,27 @@
+import * as Message from "./message"
+
+// Wrapper around the WebWorker API
+export default class Video {
+    worker: Worker;
+
+    constructor(config: Message.Config) {
+        const url = new URL('worker.ts', import.meta.url)
+        this.worker = new Worker(url, {
+            type: "module",
+            name: "video",
+        })
+        this.worker.postMessage({ config }, [ config.canvas ])
+    }
+
+    init(init: Message.Init) {
+        this.worker.postMessage({ init }) // note: we copy the raw init bytes each time
+    }
+
+    segment(segment: Message.Segment) {
+        this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ])
+    }
+
+    play() {
+        // TODO
+    }
+}

player/src/video/message.ts

@@ -0,0 +1,17 @@
+import * as MP4 from "../mp4"
+
+export interface Config {
+    canvas: OffscreenCanvas;
+}
+
+export interface Init {
+    track: string;
+    info: MP4.Info;
+    raw: MP4.ArrayBuffer[];
+}
+
+export interface Segment {
+    track: string;
+    buffer: Uint8Array;     // unread buffered data
+    reader: ReadableStream; // unread unbuffered data
+}

player/src/video/renderer.ts

@@ -0,0 +1,91 @@
+import * as Message from "./message";
+
+export default class Renderer {
+    canvas: OffscreenCanvas;
+    queue: Array<VideoFrame>;
+    render: number; // non-zero if requestAnimationFrame has been called
+    sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0
+    last?: number; // the timestamp of the last rendered frame
+
+    constructor(config: Message.Config) {
+        this.canvas = config.canvas;
+        this.queue = [];
+        this.render = 0;
+    }
+
+    emit(frame: VideoFrame) {
+        if (!this.sync) {
+            // Save the frame as the sync point
+            this.sync = performance.now() - frame.timestamp
+        }
+
+        // Drop any old frames
+        if (this.last && frame.timestamp <= this.last) {
+            frame.close()
+            return
+        }
+
+        // Insert the frame into the queue sorted by timestamp.
+        if (this.queue.length > 0 && this.queue[this.queue.length-1].timestamp <= frame.timestamp) {
+            // Fast path because we normally append to the end.
+            this.queue.push(frame)
+        } else {
+            // Do a full binary search
+            let low = 0
+            let high = this.queue.length;
+
+            while (low < high) {
+                var mid = (low + high) >>> 1;
+                if (this.queue[mid].timestamp < frame.timestamp) low = mid + 1;
+                else high = mid;
+            }
+
+            this.queue.splice(low, 0, frame)
+        }
+
+        // Queue up to render the next frame.
+        if (!this.render) {
+            this.render = self.requestAnimationFrame(this.draw.bind(this))
+        }
+    }
+
+    draw(now: DOMHighResTimeStamp) {
+        // Determine the target timestamp.
+        const target = now - this.sync!
+
+        let frame = this.queue[0]
+        if (frame.timestamp >= target) {
+            // nothing to render yet, wait for the next animation frame
+            this.render = self.requestAnimationFrame(this.draw.bind(this))
+            return
+        }
+
+        this.queue.shift()
+
+        // Check if we should skip some frames
+        while (this.queue.length) {
+            const next = this.queue[0]
+            if (next.timestamp > target) {
+                break
+            }
+
+            frame.close()
+
+            this.queue.shift()
+            frame = next
+        }
+
+        const ctx = this.canvas.getContext("2d");
+        ctx!.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height) // TODO aspect ratio
+
+        this.last = frame.timestamp;
+        frame.close()
+
+        if (this.queue.length > 0) {
+            this.render = self.requestAnimationFrame(this.draw.bind(this))
+        } else {
+            // Break the loop for now
+            this.render = 0
+        }
+    }
+}

player/src/video/worker.ts

@@ -0,0 +1,22 @@
+import Renderer from "./renderer"
+import Decoder from "./decoder"
+import * as Message from "./message"
+
+let decoder: Decoder;
+let renderer: Renderer;
+
+self.addEventListener('message', async (e: MessageEvent) => {
+    if (e.data.config) {
+        const config = e.data.config as Message.Config
+
+        renderer = new Renderer(config)
+        decoder = new Decoder(renderer)
+    } else if (e.data.init) {
+        const init = e.data.init as Message.Init
+        await decoder.init(init)
+    } else if (e.data.segment) {
+        const segment = e.data.segment as Message.Segment
+        await decoder.decode(segment)
+    }
+})
+

player/tsconfig.json

@@ -3,11 +3,9 @@
     "src/**/*"
   ],
   "compilerOptions": {
-    "target": "es2021",
+    "target": "es2022",
+    "module": "es2022",
+    "moduleResolution": "node",
     "strict": true,
-    "typeRoots": [
-      "src/types"
-    ],
-    "allowJs": true
   }
 }

server/go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/marten-seemann/qtls-go1-18 v0.1.3 // indirect
 	github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect
 	github.com/onsi/ginkgo/v2 v2.2.0 // indirect
-	golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 // indirect
+	golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect
 	golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect

server/go.sum

@@ -77,7 +77,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/lucas-clemente/quic-go v0.31.0 h1:MfNp3fk0wjWRajw6quMFA3ap1AVtlU+2mtwmbVogB2M=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/marten-seemann/qpack v0.3.0 h1:UiWstOgT8+znlkDPOg2+3rIuYXJ2CnGDkGUXN6ki6hE=
@@ -153,8 +152,8 @@ golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 h1:/pEO3GD/ABYAjuakUS6xSEmmlyVS4kxBNkeA9tLJiTI=
-golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 h1:tkVvjkPTB7pnW3jnid7kNyAMPVWllTNOf/qKDze4p9o=
+golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA=
 golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA=

server/internal/warp/session.go

@@ -176,6 +176,11 @@ func (s *Session) writeInit(ctx context.Context, init *MediaInit) (err error) {
 		return fmt.Errorf("failed to create stream: %w", err)
 	}
 
+	if temp == nil {
+		// Not sure when this happens, perhaps when closing a connection?
+		return fmt.Errorf("received a nil stream from quic-go")
+	}
+
 	// Wrap the stream in an object that buffers writes instead of blocking.
 	stream := NewStream(temp)
 	s.streams.Add(stream.Run)
@@ -200,6 +205,11 @@ func (s *Session) writeInit(ctx context.Context, init *MediaInit) (err error) {
 		return fmt.Errorf("failed to write init data: %w", err)
 	}
 
+	err = stream.Close()
+	if err != nil {
+		return fmt.Errorf("failed to close init stream: %w", err)
+	}
+
 	return nil
 }
 
@@ -210,6 +220,11 @@ func (s *Session) writeSegment(ctx context.Context, segment *MediaSegment) (err
 		return fmt.Errorf("failed to create stream: %w", err)
 	}
 
+	if temp == nil {
+		// Not sure when this happens, perhaps when closing a connection?
+		return fmt.Errorf("received a nil stream from quic-go")
+	}
+
 	// Wrap the stream in an object that buffers writes instead of blocking.
 	stream := NewStream(temp)
 	s.streams.Add(stream.Run)