From c8c856d6b8ed7694a4e7699f1b8fc2d5056eab7e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 1 Apr 2023 18:34:10 -0700 Subject: [PATCH] WIP --- player/.proxyrc.js | 7 +++++++ player/package.json | 1 + player/src/audio/decoder.ts | 9 +++++++-- player/src/audio/index.ts | 18 ++++++++++-------- player/src/audio/message.ts | 2 +- player/src/audio/renderer.ts | 8 +++----- player/src/audio/source/index.ts | 2 +- player/src/audio/worker.ts | 4 ++-- player/src/index.css | 6 ------ player/src/transport/index.ts | 12 +++++++----- player/yarn.lock | 5 +++++ 11 files changed, 44 insertions(+), 30 deletions(-) create mode 100644 player/.proxyrc.js diff --git a/player/.proxyrc.js b/player/.proxyrc.js new file mode 100644 index 0000000..4d53c59 --- /dev/null +++ b/player/.proxyrc.js @@ -0,0 +1,7 @@ +module.exports = function (app) { + app.use((req, res, next) => { + res.setHeader('Cross-Origin-Opener-Policy', 'same-origin'); + res.setHeader('Cross-Origin-Embedder-Policy', 'require-corp'); + next(); + }); +}; diff --git a/player/package.json b/player/package.json index 0c4f4be..8238f94 100644 --- a/player/package.json +++ b/player/package.json @@ -8,6 +8,7 @@ "devDependencies": { "@parcel/transformer-inline-string": "2.8.3", "@parcel/validator-typescript": "^2.6.0", + "@types/audioworklet": "^0.0.41", "@types/dom-webcodecs": "^0.1.6", "parcel": "^2.8.0", "typescript": ">=3.0.0" diff --git a/player/src/audio/decoder.ts b/player/src/audio/decoder.ts index 5c1d654..baf76dd 100644 --- a/player/src/audio/decoder.ts +++ b/player/src/audio/decoder.ts @@ -10,7 +10,7 @@ export class Decoder { tracks: Map> renderer: Renderer; - constructor(renderer: Renderer) { + constructor(config: Message.Config, renderer: Renderer) { this.tracks = new Map(); this.renderer = renderer; } @@ -22,6 +22,7 @@ export class Decoder { this.tracks.set(msg.track, track) } + console.log(msg.info) if (msg.info.audioTracks.length != 1 || msg.info.videoTracks.length != 0) { throw new Error("Expected a single audio track") } @@ -42,16 +43,19 @@ export class Decoder { const audio = info.audioTracks[0] const decoder = new AudioDecoder({ - output: (frame: AudioFrame) => { + output: (frame: AudioData) => { this.renderer.emit(frame) }, error: (err: Error) => { console.warn(err) } }); + console.log(audio) decoder.configure({ codec: audio.codec, + numberOfChannels: audio.audio.channel_count, + sampleRate: audio.audio.sample_rate, // optimizeForLatency: true }) @@ -61,6 +65,7 @@ export class Decoder { for (let sample of samples) { // TODO this assumes that timescale == sample rate decoder.decode(new EncodedAudioChunk({ + type: sample.is_sync ? "key" : "delta", data: sample.data, duration: sample.duration, timestamp: sample.dts, diff --git a/player/src/audio/index.ts b/player/src/audio/index.ts index 3c06976..670e874 100644 --- a/player/src/audio/index.ts +++ b/player/src/audio/index.ts @@ -1,19 +1,21 @@ import * as Message from "./message" +import { Renderer } from "./renderer" +import { Decoder } from "./decoder" -// Wrapper around the WebWorker API export default class Audio { - worker: Worker; + renderer: Renderer; + decoder: Decoder; constructor(config: Message.Config) { - this.worker = new Worker(new URL('worker.ts', import.meta.url), { type: "module" }) - this.worker.postMessage({ config }, []) + this.renderer = new Renderer(config) + this.decoder = new Decoder(config, this.renderer) } - init(init: Message.Init) { - this.worker.postMessage({ init }) // note: we copy the raw init bytes each time + async init(init: Message.Init) { + await this.decoder.init(init) } - segment(segment: Message.Segment) { - this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ]) + async segment(segment: Message.Segment) { + await this.decoder.decode(segment) } } \ No newline at end of file diff --git a/player/src/audio/message.ts b/player/src/audio/message.ts index f9e39e5..39bab14 100644 --- a/player/src/audio/message.ts +++ b/player/src/audio/message.ts @@ -1,7 +1,7 @@ import * as MP4 from "../mp4" export interface Config { - // temporarily empty + ctx: AudioContext; } export interface Init { diff --git a/player/src/audio/renderer.ts b/player/src/audio/renderer.ts index 5a704cd..a379685 100644 --- a/player/src/audio/renderer.ts +++ b/player/src/audio/renderer.ts @@ -3,7 +3,6 @@ import * as Message from "./message"; import Source from "./source"; export class Renderer { - ctx: AudioContext; source: Source; render: number; // non-zero if requestAnimationFrame has been called @@ -11,16 +10,15 @@ export class Renderer { maxDuration: number; // the maximum duration allowed in the buffer - constructor() { + constructor(config: Message.Config) { this.render = 0; this.maxDuration = 10 * 1000 // TODO evaluate { latencyHint: "interactive" } - this.ctx = new AudioContext() - this.source = new Source(this.ctx) + this.source = new Source(config.ctx) } - emit(frame: AudioFrame) { + emit(frame: AudioData) { this.source.emit(frame) } } \ No newline at end of file diff --git a/player/src/audio/source/index.ts b/player/src/audio/source/index.ts index c0f8421..1d70385 100644 --- a/player/src/audio/source/index.ts +++ b/player/src/audio/source/index.ts @@ -55,7 +55,7 @@ export default class Source { } } - emit(frame: AudioFrame) { + emit(frame: AudioData) { for (let i = 0; i < frame.channels; i += 1) { const ring = this.channels[i] ring.set(frame, i) diff --git a/player/src/audio/worker.ts b/player/src/audio/worker.ts index 028fc65..bd5ec76 100644 --- a/player/src/audio/worker.ts +++ b/player/src/audio/worker.ts @@ -9,8 +9,8 @@ self.addEventListener('message', async (e: MessageEvent) => { if (e.data.config) { const config = e.data.config as Message.Config; - renderer = new Renderer() - decoder = new Decoder(renderer) + renderer = new Renderer(config) + decoder = new Decoder(config, renderer) } if (e.data.init) { diff --git a/player/src/index.css b/player/src/index.css index 844f581..c406cf4 100644 --- a/player/src/index.css +++ b/player/src/index.css @@ -29,12 +29,6 @@ body { z-index: 1; } -#vid { - width: 100%; - height: 100%; - max-height: 100vh; -} - #controls { display: flex; flex-wrap: wrap; diff --git a/player/src/transport/index.ts b/player/src/transport/index.ts index f9df8ee..39243c0 100644 --- a/player/src/transport/index.ts +++ b/player/src/transport/index.ts @@ -27,7 +27,9 @@ export class Player { this.tracks = new Map(); // TODO move these to another class so this only deals with the transport. - this.audio = new Audio({}) + this.audio = new Audio({ + ctx: new AudioContext(), + }) this.video = new Video({ canvas: props.canvas.transferControlToOffscreen(), }) @@ -133,13 +135,13 @@ export class Player { throw new Error("expected a single track") } - if (info.audioTracks) { + if (info.audioTracks.length) { this.audio.init({ track: msg.id, info: info, raw: track.raw, }) - } else if (info.videoTracks) { + } else if (info.videoTracks.length) { this.video.init({ track: msg.id, info: info, @@ -160,13 +162,13 @@ export class Player { // Wait until we learn if this is an audio or video track const info = await track.info - if (info.audioTracks) { + if (info.audioTracks.length) { this.audio.segment({ track: msg.init, buffer: stream.buffer, reader: stream.reader, }) - } else if (info.videoTracks) { + } else if (info.videoTracks.length) { this.video.segment({ track: msg.init, buffer: stream.buffer, diff --git a/player/yarn.lock b/player/yarn.lock index 24f0b4e..c4a13b7 100644 --- a/player/yarn.lock +++ b/player/yarn.lock @@ -743,6 +743,11 @@ resolved "https://registry.yarnpkg.com/@trysound/sax/-/sax-0.2.0.tgz#cccaab758af56761eb7bf37af6f03f326dd798ad" integrity sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA== +"@types/audioworklet@^0.0.41": + version "0.0.41" + resolved "https://registry.yarnpkg.com/@types/audioworklet/-/audioworklet-0.0.41.tgz#ed2e67c7dcbe207974c54812e21819984c74a4f0" + integrity sha512-8BWffzGoSRz436IviQVPye75YYWfac4OKdcLgkZxb3APZxSmAOp2SMtsH1yuM1x57/z/J7bsm05Yq98Hzk1t/w== + "@types/dom-webcodecs@^0.1.6": version "0.1.6" resolved "https://registry.yarnpkg.com/@types/dom-webcodecs/-/dom-webcodecs-0.1.6.tgz#204f2c8c2e6286114e1ec2997630bc0fd2081ffa"