diff --git a/media/generate b/media/generate index 922d2bc..5105c06 100755 --- a/media/generate +++ b/media/generate @@ -6,8 +6,7 @@ cd "$(dirname "$0")" # separate_moof: Splits audio and video into separate moof flags. # omit_tfhd_offset: Removes absolute byte offsets so we can fragment. -ffmpeg -i source.mp4 \ +ffmpeg -i source.mp4 -y \ + -c copy \ -movflags empty_moov+frag_every_frame+separate_moof+omit_tfhd_offset \ - -c:v copy \ - -an \ - fragmented.mp4 + fragmented.mp4 2>&1 diff --git a/player/src/index.ts b/player/src/index.ts index 89e2e22..5882480 100644 --- a/player/src/index.ts +++ b/player/src/index.ts @@ -1,4 +1,5 @@ import Player from "./player" +import Transport from "./transport" // @ts-ignore embed the certificate fingerprint using bundler import fingerprintHex from 'bundle-text:../fingerprint.hex'; @@ -14,19 +15,23 @@ const params = new URLSearchParams(window.location.search) const url = params.get("url") || "https://127.0.0.1:4443/watch" const canvas = document.querySelector("canvas#video")! -const player = new Player({ +const transport = new Transport({ url: url, fingerprint: { // TODO remove when Chrome accepts the system CA "algorithm": "sha-256", "value": new Uint8Array(fingerprint), }, - canvas: canvas, +}) + +const player = new Player({ + transport, + canvas: canvas.transferControlToOffscreen(), }) const play = document.querySelector("#screen #play")! let playFunc = (e: Event) => { - player.play() + player.play({}) e.preventDefault() play.removeEventListener('click', playFunc) diff --git a/player/src/media/index.ts b/player/src/media/index.ts deleted file mode 100644 index 376052a..0000000 --- a/player/src/media/index.ts +++ /dev/null @@ -1,81 +0,0 @@ -import * as Message from "./message" -import { RingInit } from "./ring" - -// Abstracts the Worker and Worklet into a simpler API -// This class must be created on the main thread due to AudioContext. -export default class Media { - context: AudioContext; - worker: Worker; - worklet: Promise; - - constructor(videoConfig: Message.VideoConfig) { - // Assume 44.1kHz and two audio channels - const audioConfig = { - sampleRate: 44100, - ring: new RingInit(2, 4410), // 100ms at 44.1khz - } - - const config = { - audio: audioConfig, - video: videoConfig, - } - - this.context = new AudioContext({ - latencyHint: "interactive", - sampleRate: config.audio.sampleRate, - }) - - - this.worker = this.setupWorker(config) - this.worklet = this.setupWorklet(config) - } - - private setupWorker(config: Message.Config): Worker { - const url = new URL('worker.ts', import.meta.url) - - const worker = new Worker(url, { - type: "module", - name: "media", - }) - - worker.postMessage({ config }, [ config.video.canvas ]) - - return worker - } - - private async setupWorklet(config: Message.Config): Promise { - // Load the worklet source code. - const url = new URL('worklet.ts', import.meta.url) - await this.context.audioWorklet.addModule(url) - - const volume = this.context.createGain() - volume.gain.value = 2.0; - - // Create a worklet - const worklet = new AudioWorkletNode(this.context, 'renderer'); - worklet.onprocessorerror = (e: Event) => { - console.error("Audio worklet error:", e) - }; - - worklet.port.postMessage({ config }) - - // Connect the worklet to the volume node and then to the speakers - worklet.connect(volume) - volume.connect(this.context.destination) - - return worklet - } - - init(init: Message.Init) { - this.worker.postMessage({ init }, [ init.buffer.buffer, init.reader ]) - } - - segment(segment: Message.Segment) { - this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ]) - } - - play(play: Message.Play) { - this.context.resume() - //this.worker.postMessage({ play }) - } -} \ No newline at end of file diff --git a/player/src/mp4/index.ts b/player/src/mp4/index.ts index ca1a0bd..b375434 100644 --- a/player/src/mp4/index.ts +++ b/player/src/mp4/index.ts @@ -1,12 +1,16 @@ -import * as MP4 from "./rename" -export * from "./rename" +// Rename some stuff so it's on brand. +export { + createFile as New, + MP4File as File, + MP4ArrayBuffer as ArrayBuffer, + MP4Info as Info, + MP4Track as Track, + MP4AudioTrack as AudioTrack, + MP4VideoTrack as VideoTrack, + DataStream as Stream, + Box, + ISOFile, + Sample, +} from "mp4box" -export { Init, InitParser } from "./init" - -export function isAudioTrack(track: MP4.Track): track is MP4.AudioTrack { - return (track as MP4.AudioTrack).audio !== undefined; -} - -export function isVideoTrack(track: MP4.Track): track is MP4.VideoTrack { - return (track as MP4.VideoTrack).video !== undefined; -} \ No newline at end of file +export { Init, InitParser } from "./init" \ No newline at end of file diff --git a/player/src/mp4/mp4box.d.ts b/player/src/mp4/mp4box.d.ts index 018f185..2f95f94 100644 --- a/player/src/mp4/mp4box.d.ts +++ b/player/src/mp4/mp4box.d.ts @@ -82,7 +82,7 @@ declare module "mp4box" { description: any; data: ArrayBuffer; size: number; - alreadyRead: number; + alreadyRead?: number; duration: number; cts: number; dts: number; @@ -104,7 +104,7 @@ declare module "mp4box" { const LITTLE_ENDIAN: boolean; export class DataStream { - constructor(buffer: ArrayBuffer, byteOffset?: number, littleEndian?: boolean); + constructor(buffer?: ArrayBuffer, byteOffset?: number, littleEndian?: boolean); getPosition(): number; get byteLength(): number; @@ -144,5 +144,82 @@ declare module "mp4box" { // TODO I got bored porting the remaining functions } + export class Box { + write(stream: DataStream): void; + } + + export interface TrackOptions { + id?: number; + type?: string; + width?: number; + height?: number; + duration?: number; + layer?: number; + timescale?: number; + media_duration?: number; + language?: string; + hdlr?: string; + + // video + avcDecoderConfigRecord?: any; + + // audio + balance?: number; + channel_count?: number; + samplesize?: number; + samplerate?: number; + + //captions + namespace?: string; + schema_location?: string; + auxiliary_mime_types?: string; + + description?: any; + description_boxes?: Box[]; + + default_sample_description_index_id?: number; + default_sample_duration?: number; + default_sample_size?: number; + default_sample_flags?: number; + } + + export interface FileOptions { + brands?: string[]; + timescale?: number; + rate?: number; + duration?: number; + width?: number; + } + + export interface SampleOptions { + sample_description_index?: number; + duration?: number; + cts?: number; + dts?: number; + is_sync?: boolean; + is_leading?: number; + depends_on?: number; + is_depended_on?: number; + has_redundancy?: number; + degradation_priority?: number; + subsamples?: any; + } + + // TODO add the remaining functions + // TODO move to another module + export class ISOFile { + constructor(stream?: DataStream); + + init(options?: FileOptions): ISOFile; + addTrack(options?: TrackOptions): number; + addSample(track: number, data: ArrayBuffer, options?: SampleOptions): Sample; + + createSingleSampleMoof(sample: Sample): Box; + + // helpers + getTrackById(id: number): Box | undefined; + getTrexById(id: number): Box | undefined; + } + export { }; } \ No newline at end of file diff --git a/player/src/mp4/rename.ts b/player/src/mp4/rename.ts deleted file mode 100644 index d45682b..0000000 --- a/player/src/mp4/rename.ts +++ /dev/null @@ -1,12 +0,0 @@ -// Rename some stuff so it's on brand. -export { - createFile as New, - MP4File as File, - MP4ArrayBuffer as ArrayBuffer, - MP4Info as Info, - MP4Track as Track, - MP4AudioTrack as AudioTrack, - MP4VideoTrack as VideoTrack, - DataStream as Stream, - Sample, -} from "mp4box" \ No newline at end of file diff --git a/player/src/media/decoder.ts b/player/src/player/decoder.ts similarity index 94% rename from player/src/media/decoder.ts rename to player/src/player/decoder.ts index ca2f639..64c5683 100644 --- a/player/src/media/decoder.ts +++ b/player/src/player/decoder.ts @@ -1,7 +1,6 @@ import * as Message from "./message"; import * as MP4 from "../mp4" import * as Stream from "../stream" -import * as Util from "../util" import Renderer from "./renderer" @@ -82,7 +81,7 @@ export default class Decoder { // We need a sample to initalize the video decoder, because of mp4box limitations. let sample = samples[0]; - if (MP4.isVideoTrack(track)) { + if (isVideoTrack(track)) { // Configure the decoder using the AVC box for H.264 // TODO it should be easy to support other codecs, just need to know the right boxes. const avcc = sample.description.avcC; @@ -105,7 +104,7 @@ export default class Decoder { }) decoder = videoDecoder - } else if (MP4.isAudioTrack(track)) { + } else if (isAudioTrack(track)) { const audioDecoder = new AudioDecoder({ output: this.renderer.push.bind(this.renderer), error: console.warn, @@ -157,4 +156,12 @@ function isAudioDecoder(decoder: AudioDecoder | VideoDecoder): decoder is AudioD function isVideoDecoder(decoder: AudioDecoder | VideoDecoder): decoder is VideoDecoder { return decoder instanceof VideoDecoder +} + +function isAudioTrack(track: MP4.Track): track is MP4.AudioTrack { + return (track as MP4.AudioTrack).audio !== undefined; +} + +function isVideoTrack(track: MP4.Track): track is MP4.VideoTrack { + return (track as MP4.VideoTrack).video !== undefined; } \ No newline at end of file diff --git a/player/src/player/index.ts b/player/src/player/index.ts index 9e9c2a6..1ef73b2 100644 --- a/player/src/player/index.ts +++ b/player/src/player/index.ts @@ -1,33 +1,89 @@ +import * as Message from "./message" +import * as Ring from "./ring" import Transport from "../transport" -import Media from "../media" -export interface PlayerInit { - url: string; - fingerprint?: WebTransportHash; // the certificate fingerprint, temporarily needed for local development - canvas: HTMLCanvasElement; +export interface Config { + transport: Transport + canvas: OffscreenCanvas; } +// This class must be created on the main thread due to AudioContext. export default class Player { - media: Media; - transport: Transport; + context: AudioContext; + worker: Worker; + worklet: Promise; - constructor(props: PlayerInit) { - this.media = new Media({ - canvas: props.canvas.transferControlToOffscreen(), - }) + transport: Transport - this.transport = new Transport({ - url: props.url, - fingerprint: props.fingerprint, - media: this.media, + constructor(config: Config) { + this.transport = config.transport + this.transport.callback = this; + + const video = { + canvas: config.canvas, + }; + + // Assume 44.1kHz and two audio channels + const audio = { + sampleRate: 44100, + ring: new Ring.Buffer(2, 4410), // 100ms at 44.1khz + } + + this.context = new AudioContext({ + latencyHint: "interactive", + sampleRate: audio.sampleRate, }) - } - async close() { - this.transport.close() - } + this.worker = this.setupWorker({ audio, video }) + this.worklet = this.setupWorklet(audio) + } - play() { - this.media.play({}) - } + private setupWorker(config: Message.Config): Worker { + const url = new URL('worker.ts', import.meta.url) + + const worker = new Worker(url, { + type: "module", + name: "media", + }) + + worker.postMessage({ config }, [ config.video.canvas ]) + + return worker + } + + private async setupWorklet(config: Message.AudioConfig): Promise { + // Load the worklet source code. + const url = new URL('worklet.ts', import.meta.url) + await this.context.audioWorklet.addModule(url) + + const volume = this.context.createGain() + volume.gain.value = 2.0; + + // Create a worklet + const worklet = new AudioWorkletNode(this.context, 'renderer'); + worklet.onprocessorerror = (e: Event) => { + console.error("Audio worklet error:", e) + }; + + worklet.port.postMessage({ config }) + + // Connect the worklet to the volume node and then to the speakers + worklet.connect(volume) + volume.connect(this.context.destination) + + return worklet + } + + onInit(init: Message.Init) { + this.worker.postMessage({ init }, [ init.buffer.buffer, init.reader ]) + } + + onSegment(segment: Message.Segment) { + this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ]) + } + + play(play: Message.Play) { + this.context.resume() + //this.worker.postMessage({ play }) + } } \ No newline at end of file diff --git a/player/src/media/message.ts b/player/src/player/message.ts similarity index 85% rename from player/src/media/message.ts rename to player/src/player/message.ts index a457200..dcad2e0 100644 --- a/player/src/media/message.ts +++ b/player/src/player/message.ts @@ -1,5 +1,4 @@ -import * as MP4 from "../mp4" -import { RingInit } from "../media/ring" +import * as Ring from "./ring" export interface Config { audio: AudioConfig; @@ -13,7 +12,7 @@ export interface VideoConfig { export interface AudioConfig { // audio stuff sampleRate: number; - ring: RingInit; + ring: Ring.Buffer; } export interface Init { diff --git a/player/src/media/renderer.ts b/player/src/player/renderer.ts similarity index 94% rename from player/src/media/renderer.ts rename to player/src/player/renderer.ts index af56e2a..d2d6c86 100644 --- a/player/src/media/renderer.ts +++ b/player/src/player/renderer.ts @@ -68,8 +68,11 @@ export default class Renderer { } draw(now: DOMHighResTimeStamp) { + // Convert to microseconds + now *= 1000; + // Determine the target timestamp. - const target = 1000 * now - this.sync! + const target = now - this.sync! this.drawAudio(now, target) this.drawVideo(now, target) @@ -85,9 +88,11 @@ export default class Renderer { // Check if we should skip some frames while (this.audioQueue.length) { const next = this.audioQueue[0] - if (next.timestamp >= target) { + + if (next.timestamp > target) { let ok = this.audioRing.write(next) if (!ok) { + console.warn("ring buffer is full") // No more space in the ring break } @@ -101,7 +106,7 @@ export default class Renderer { } drawVideo(now: DOMHighResTimeStamp, target: DOMHighResTimeStamp) { - if (this.videoQueue.length == 0) return; + if (!this.videoQueue.length) return; let frame = this.videoQueue[0]; if (frame.timestamp >= target) { diff --git a/player/src/media/ring.ts b/player/src/player/ring.ts similarity index 95% rename from player/src/media/ring.ts rename to player/src/player/ring.ts index cd796ee..ffc1c6b 100644 --- a/player/src/media/ring.ts +++ b/player/src/player/ring.ts @@ -11,15 +11,15 @@ export class Ring { channels: Float32Array[]; capacity: number; - constructor(init: RingInit) { - this.state = new Int32Array(init.state) + constructor(buf: Buffer) { + this.state = new Int32Array(buf.state) this.channels = [] - for (let channel of init.channels) { + for (let channel of buf.channels) { this.channels.push(new Float32Array(channel)) } - this.capacity = init.capacity + this.capacity = buf.capacity } // Add the samples for single audio frame @@ -121,7 +121,7 @@ export class Ring { } // No prototype to make this easier to send via postMessage -export class RingInit { +export class Buffer { state: SharedArrayBuffer; channels: SharedArrayBuffer[]; diff --git a/player/src/media/worker.ts b/player/src/player/worker.ts similarity index 100% rename from player/src/media/worker.ts rename to player/src/player/worker.ts diff --git a/player/src/media/worklet.ts b/player/src/player/worklet.ts similarity index 87% rename from player/src/media/worklet.ts rename to player/src/player/worklet.ts index 5961c32..4946bd8 100644 --- a/player/src/media/worklet.ts +++ b/player/src/player/worklet.ts @@ -20,12 +20,12 @@ class Renderer extends AudioWorkletProcessor { onMessage(e: MessageEvent) { if (e.data.config) { - this.config(e.data.config) + this.onConfig(e.data.config) } } - config(config: Message.Config) { - this.ring = new Ring(config.audio.ring) + onConfig(config: Message.AudioConfig) { + this.ring = new Ring(config.ring) } // Inputs and outputs in groups of 128 samples. diff --git a/player/src/transport/index.ts b/player/src/transport/index.ts index e588a8f..72eb4fd 100644 --- a/player/src/transport/index.ts +++ b/player/src/transport/index.ts @@ -1,25 +1,18 @@ -import * as Message from "./message" import * as Stream from "../stream" -import * as MP4 from "../mp4" +import * as Interface from "./interface" -import Media from "../media" - -export interface TransportInit { +export interface Config { url: string; fingerprint?: WebTransportHash; // the certificate fingerprint, temporarily needed for local development - media: Media; } export default class Transport { quic: Promise; api: Promise; + callback?: Interface.Callback; - media: Media; - - constructor(props: TransportInit) { - this.media = props.media; - - this.quic = this.connect(props) + constructor(config: Config) { + this.quic = this.connect(config) // Create a unidirectional stream for all of our messages this.api = this.quic.then((q) => { @@ -35,13 +28,13 @@ export default class Transport { } // Helper function to make creating a promise easier - private async connect(props: TransportInit): Promise { + private async connect(config: Config): Promise { let options: WebTransportOptions = {}; - if (props.fingerprint) { - options.serverCertificateHashes = [ props.fingerprint ] + if (config.fingerprint) { + options.serverCertificateHashes = [ config.fingerprint ] } - const quic = new WebTransport(props.url, options) + const quic = new WebTransport(config.url, options) await quic.ready return quic } @@ -86,12 +79,12 @@ export default class Transport { const msg = JSON.parse(payload) if (msg.init) { - return this.media.init({ + return this.callback?.onInit({ buffer: r.buffer, reader: r.reader, }) } else if (msg.segment) { - return this.media.segment({ + return this.callback?.onSegment({ buffer: r.buffer, reader: r.reader, }) diff --git a/player/src/transport/interface.ts b/player/src/transport/interface.ts new file mode 100644 index 0000000..cc5e3aa --- /dev/null +++ b/player/src/transport/interface.ts @@ -0,0 +1,14 @@ +export interface Callback { + onInit(init: Init): any + onSegment(segment: Segment): any +} + +export interface Init { + buffer: Uint8Array; // unread buffered data + reader: ReadableStream; // unread unbuffered data +} + +export interface Segment { + buffer: Uint8Array; // unread buffered data + reader: ReadableStream; // unread unbuffered data +} \ No newline at end of file diff --git a/player/src/transport/message.ts b/player/src/transport/message.ts index d151538..7cb511f 100644 --- a/player/src/transport/message.ts +++ b/player/src/transport/message.ts @@ -1,12 +1,5 @@ -export interface Init { - id: string -} - -export interface Segment { - init: string // id of the init segment - timestamp: number // presentation timestamp in milliseconds of the first sample - // TODO track would be nice -} +export interface Init {} +export interface Segment {} export interface Debug { max_bitrate: number