WIP

player/.proxyrc.js

@@ -0,0 +1,7 @@
+module.exports = function (app) {
+    app.use((req, res, next) => {
+        res.setHeader('Cross-Origin-Opener-Policy', 'same-origin');
+        res.setHeader('Cross-Origin-Embedder-Policy', 'require-corp');
+        next();
+    });
+};

player/package.json

@@ -8,6 +8,7 @@
   "devDependencies": {
     "@parcel/transformer-inline-string": "2.8.3",
     "@parcel/validator-typescript": "^2.6.0",
+    "@types/audioworklet": "^0.0.41",
     "@types/dom-webcodecs": "^0.1.6",
     "parcel": "^2.8.0",
     "typescript": ">=3.0.0"

player/src/audio/decoder.ts

@@ -10,7 +10,7 @@ export class Decoder {
     tracks: Map<string, Util.Deferred<Message.Init>>
     renderer: Renderer;
 
-    constructor(renderer: Renderer) {
+    constructor(config: Message.Config, renderer: Renderer) {
         this.tracks = new Map();
         this.renderer = renderer;
     }
@@ -22,6 +22,7 @@ export class Decoder {
 			this.tracks.set(msg.track, track)
 		}
 
+        console.log(msg.info)
         if (msg.info.audioTracks.length != 1 || msg.info.videoTracks.length != 0) {
             throw new Error("Expected a single audio track")
         }
@@ -42,16 +43,19 @@ export class Decoder {
         const audio = info.audioTracks[0]
 
         const decoder = new AudioDecoder({
-            output: (frame: AudioFrame) => {
+            output: (frame: AudioData) => {
                 this.renderer.emit(frame)
             },
             error: (err: Error) => {
                 console.warn(err)
             }
         });
+        console.log(audio)
 
         decoder.configure({
             codec: audio.codec,
+            numberOfChannels: audio.audio.channel_count,
+            sampleRate: audio.audio.sample_rate,
             // optimizeForLatency: true
         })
 
@@ -61,6 +65,7 @@ export class Decoder {
             for (let sample of samples) {
                 // TODO this assumes that timescale == sample rate
                 decoder.decode(new EncodedAudioChunk({
+                    type: sample.is_sync ? "key" : "delta",
                     data: sample.data,
                     duration: sample.duration,
                     timestamp: sample.dts,

player/src/audio/index.ts

@@ -1,19 +1,21 @@
 import * as Message from "./message"
+import { Renderer } from "./renderer"
+import { Decoder } from "./decoder"
 
-// Wrapper around the WebWorker API
 export default class Audio {
-    worker: Worker;
+    renderer: Renderer;
+    decoder: Decoder;
 
     constructor(config: Message.Config) {
-        this.worker = new Worker(new URL('worker.ts', import.meta.url), { type: "module" })
+        this.renderer = new Renderer(config)
-        this.worker.postMessage({ config }, [])
+        this.decoder = new Decoder(config, this.renderer)
     }
 
-    init(init: Message.Init) {
+    async init(init: Message.Init) {
-        this.worker.postMessage({ init }) // note: we copy the raw init bytes each time
+        await this.decoder.init(init)
     }
 
-    segment(segment: Message.Segment) {
+    async segment(segment: Message.Segment) {
-        this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ])
+        await this.decoder.decode(segment)
     }
 }

player/src/audio/message.ts

@@ -1,7 +1,7 @@
 import * as MP4 from "../mp4"
 
 export interface Config {
-    // temporarily empty
+    ctx: AudioContext;
 }
 
 export interface Init {

player/src/audio/renderer.ts

@@ -3,7 +3,6 @@ import * as Message from "./message";
 import Source from "./source";
 
 export class Renderer {
-    ctx: AudioContext;
     source: Source;
 
     render: number; // non-zero if requestAnimationFrame has been called
@@ -11,16 +10,15 @@ export class Renderer {
 
     maxDuration: number; // the maximum duration allowed in the buffer
 
-    constructor() {
+    constructor(config: Message.Config) {
         this.render = 0;
         this.maxDuration = 10 * 1000
 
         // TODO evaluate { latencyHint: "interactive" }
-        this.ctx = new AudioContext()
+        this.source = new Source(config.ctx)
-        this.source = new Source(this.ctx)
     }
 
-    emit(frame: AudioFrame) {
+    emit(frame: AudioData) {
         this.source.emit(frame)
     }
 }

player/src/audio/source/index.ts

@@ -55,7 +55,7 @@ export default class Source {
         }
     }
 
-    emit(frame: AudioFrame) {
+    emit(frame: AudioData) {
         for (let i = 0; i < frame.channels; i += 1) {
             const ring = this.channels[i]
             ring.set(frame, i)

player/src/audio/worker.ts

@@ -9,8 +9,8 @@ self.addEventListener('message', async (e: MessageEvent) => {
     if (e.data.config) {
         const config = e.data.config as Message.Config;
 
-        renderer = new Renderer()
+        renderer = new Renderer(config)
-        decoder = new Decoder(renderer)
+        decoder = new Decoder(config, renderer)
     }
 
     if (e.data.init) {

player/src/index.css

@@ -29,12 +29,6 @@ body {
 	z-index: 1;
 }
 
-#vid {
-	width: 100%;
-	height: 100%;
-	max-height: 100vh;
-}
-
 #controls {
 	display: flex;
 	flex-wrap: wrap;

player/src/transport/index.ts

@@ -27,7 +27,9 @@ export class Player {
 		this.tracks = new Map();
 
 		// TODO move these to another class so this only deals with the transport.
-		this.audio = new Audio({})
+		this.audio = new Audio({
+			ctx: new AudioContext(),
+		})
 		this.video = new Video({
 			canvas: props.canvas.transferControlToOffscreen(),
 		})
@@ -133,13 +135,13 @@ export class Player {
             throw new Error("expected a single track")
         }
 
-		if (info.audioTracks) {
+		if (info.audioTracks.length) {
 			this.audio.init({
 				track: msg.id,
 				info: info,
 				raw: track.raw,
 			})
-		} else if (info.videoTracks) {
+		} else if (info.videoTracks.length) {
 			this.video.init({
 				track: msg.id,
 				info: info,
@@ -160,13 +162,13 @@ export class Player {
 		// Wait until we learn if this is an audio or video track
 		const info = await track.info
 
-		if (info.audioTracks) {
+		if (info.audioTracks.length) {
 			this.audio.segment({
 				track: msg.init,
 				buffer: stream.buffer,
 				reader: stream.reader,
 			})
-		} else if (info.videoTracks) {
+		} else if (info.videoTracks.length) {
 			this.video.segment({
 				track: msg.init,
 				buffer: stream.buffer,

player/yarn.lock

@@ -743,6 +743,11 @@
   resolved "https://registry.yarnpkg.com/@trysound/sax/-/sax-0.2.0.tgz#cccaab758af56761eb7bf37af6f03f326dd798ad"
   integrity sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==
 
+"@types/audioworklet@^0.0.41":
+  version "0.0.41"
+  resolved "https://registry.yarnpkg.com/@types/audioworklet/-/audioworklet-0.0.41.tgz#ed2e67c7dcbe207974c54812e21819984c74a4f0"
+  integrity sha512-8BWffzGoSRz436IviQVPye75YYWfac4OKdcLgkZxb3APZxSmAOp2SMtsH1yuM1x57/z/J7bsm05Yq98Hzk1t/w==
+
 "@types/dom-webcodecs@^0.1.6":
   version "0.1.6"
   resolved "https://registry.yarnpkg.com/@types/dom-webcodecs/-/dom-webcodecs-0.1.6.tgz#204f2c8c2e6286114e1ec2997630bc0fd2081ffa"