rob
/
moq-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor the TS code a bit.

This commit is contained in:
Luke Curley 2023-05-08 10:30:32 -06:00
parent e6791b872d
commit 9f0c24b552
16 changed files with 238 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
media/generate
View File

@ -6,8 +6,7 @@ cd "$(dirname "$0")"
# separate_moof: Splits audio and video into separate moof flags.
# omit_tfhd_offset: Removes absolute byte offsets so we can fragment.
ffmpeg -i source.mp4 \
ffmpeg -i source.mp4 -y \
-c copy \
-movflags empty_moov+frag_every_frame+separate_moof+omit_tfhd_offset \
-c:v copy \
-an \
fragmented.mp4
fragmented.mp4 2>&1

11
player/src/index.ts
View File

@ -1,4 +1,5 @@
import Player from "./player"
import Transport from "./transport"
// @ts-ignore embed the certificate fingerprint using bundler
import fingerprintHex from 'bundle-text:../fingerprint.hex';
@ -14,19 +15,23 @@ const params = new URLSearchParams(window.location.search)
const url = params.get("url") || "https://127.0.0.1:4443/watch"
const canvas = document.querySelector<HTMLCanvasElement>("canvas#video")!
const player = new Player({
const transport = new Transport({
url: url,
fingerprint: { // TODO remove when Chrome accepts the system CA
"algorithm": "sha-256",
"value": new Uint8Array(fingerprint),
},
canvas: canvas,
})
const player = new Player({
transport,
canvas: canvas.transferControlToOffscreen(),
})
const play = document.querySelector<HTMLElement>("#screen #play")!
let playFunc = (e: Event) => {
player.play()
player.play({})
e.preventDefault()
play.removeEventListener('click', playFunc)

81
player/src/media/index.ts
View File

@ -1,81 +0,0 @@
import * as Message from "./message"
import { RingInit } from "./ring"
// Abstracts the Worker and Worklet into a simpler API
// This class must be created on the main thread due to AudioContext.
export default class Media {
context: AudioContext;
worker: Worker;
worklet: Promise<AudioWorkletNode>;
constructor(videoConfig: Message.VideoConfig) {
// Assume 44.1kHz and two audio channels
const audioConfig = {
sampleRate: 44100,
ring: new RingInit(2, 4410), // 100ms at 44.1khz
}
const config = {
audio: audioConfig,
video: videoConfig,
}
this.context = new AudioContext({
latencyHint: "interactive",
sampleRate: config.audio.sampleRate,
})
this.worker = this.setupWorker(config)
this.worklet = this.setupWorklet(config)
}
private setupWorker(config: Message.Config): Worker {
const url = new URL('worker.ts', import.meta.url)
const worker = new Worker(url, {
type: "module",
name: "media",
})
worker.postMessage({ config }, [ config.video.canvas ])
return worker
}
private async setupWorklet(config: Message.Config): Promise<AudioWorkletNode> {
// Load the worklet source code.
const url = new URL('worklet.ts', import.meta.url)
await this.context.audioWorklet.addModule(url)
const volume = this.context.createGain()
volume.gain.value = 2.0;
// Create a worklet
const worklet = new AudioWorkletNode(this.context, 'renderer');
worklet.onprocessorerror = (e: Event) => {
console.error("Audio worklet error:", e)
};
worklet.port.postMessage({ config })
// Connect the worklet to the volume node and then to the speakers
worklet.connect(volume)
volume.connect(this.context.destination)
return worklet
}
init(init: Message.Init) {
this.worker.postMessage({ init }, [ init.buffer.buffer, init.reader ])
}
segment(segment: Message.Segment) {
this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ])
}
play(play: Message.Play) {
this.context.resume()
//this.worker.postMessage({ play })
}
}

24
player/src/mp4/index.ts
View File

@ -1,12 +1,16 @@
import * as MP4 from "./rename"
export * from "./rename"
// Rename some stuff so it's on brand.
export {
createFile as New,
MP4File as File,
MP4ArrayBuffer as ArrayBuffer,
MP4Info as Info,
MP4Track as Track,
MP4AudioTrack as AudioTrack,
MP4VideoTrack as VideoTrack,
DataStream as Stream,
Box,
ISOFile,
Sample,
} from "mp4box"
export { Init, InitParser } from "./init"
export function isAudioTrack(track: MP4.Track): track is MP4.AudioTrack {
return (track as MP4.AudioTrack).audio !== undefined;
}
export function isVideoTrack(track: MP4.Track): track is MP4.VideoTrack {
return (track as MP4.VideoTrack).video !== undefined;
}

81
player/src/mp4/mp4box.d.ts vendored
View File

@ -82,7 +82,7 @@ declare module "mp4box" {
description: any;
data: ArrayBuffer;
size: number;
alreadyRead: number;
alreadyRead?: number;
duration: number;
cts: number;
dts: number;
@ -104,7 +104,7 @@ declare module "mp4box" {
const LITTLE_ENDIAN: boolean;
export class DataStream {
constructor(buffer: ArrayBuffer, byteOffset?: number, littleEndian?: boolean);
constructor(buffer?: ArrayBuffer, byteOffset?: number, littleEndian?: boolean);
getPosition(): number;
get byteLength(): number;
@ -144,5 +144,82 @@ declare module "mp4box" {
// TODO I got bored porting the remaining functions
}
export class Box {
write(stream: DataStream): void;
}
export interface TrackOptions {
id?: number;
type?: string;
width?: number;
height?: number;
duration?: number;
layer?: number;
timescale?: number;
media_duration?: number;
language?: string;
hdlr?: string;
// video
avcDecoderConfigRecord?: any;
// audio
balance?: number;
channel_count?: number;
samplesize?: number;
samplerate?: number;
//captions
namespace?: string;
schema_location?: string;
auxiliary_mime_types?: string;
description?: any;
description_boxes?: Box[];
default_sample_description_index_id?: number;
default_sample_duration?: number;
default_sample_size?: number;
default_sample_flags?: number;
}
export interface FileOptions {
brands?: string[];
timescale?: number;
rate?: number;
duration?: number;
width?: number;
}
export interface SampleOptions {
sample_description_index?: number;
duration?: number;
cts?: number;
dts?: number;
is_sync?: boolean;
is_leading?: number;
depends_on?: number;
is_depended_on?: number;
has_redundancy?: number;
degradation_priority?: number;
subsamples?: any;
}
// TODO add the remaining functions
// TODO move to another module
export class ISOFile {
constructor(stream?: DataStream);
init(options?: FileOptions): ISOFile;
addTrack(options?: TrackOptions): number;
addSample(track: number, data: ArrayBuffer, options?: SampleOptions): Sample;
createSingleSampleMoof(sample: Sample): Box;
// helpers
getTrackById(id: number): Box | undefined;
getTrexById(id: number): Box | undefined;
}
export { };
}

12
player/src/mp4/rename.ts
View File

@ -1,12 +0,0 @@
// Rename some stuff so it's on brand.
export {
createFile as New,
MP4File as File,
MP4ArrayBuffer as ArrayBuffer,
MP4Info as Info,
MP4Track as Track,
MP4AudioTrack as AudioTrack,
MP4VideoTrack as VideoTrack,
DataStream as Stream,
Sample,
} from "mp4box"

13
player/src/media/decoder.ts → player/src/player/decoder.ts
View File

@ -1,7 +1,6 @@
import * as Message from "./message";
import * as MP4 from "../mp4"
import * as Stream from "../stream"
import * as Util from "../util"
import Renderer from "./renderer"
@ -82,7 +81,7 @@ export default class Decoder {
// We need a sample to initalize the video decoder, because of mp4box limitations.
let sample = samples[0];
if (MP4.isVideoTrack(track)) {
if (isVideoTrack(track)) {
// Configure the decoder using the AVC box for H.264
// TODO it should be easy to support other codecs, just need to know the right boxes.
const avcc = sample.description.avcC;
@ -105,7 +104,7 @@ export default class Decoder {
})
decoder = videoDecoder
} else if (MP4.isAudioTrack(track)) {
} else if (isAudioTrack(track)) {
const audioDecoder = new AudioDecoder({
output: this.renderer.push.bind(this.renderer),
error: console.warn,
@ -158,3 +157,11 @@ function isAudioDecoder(decoder: AudioDecoder | VideoDecoder): decoder is AudioD
function isVideoDecoder(decoder: AudioDecoder | VideoDecoder): decoder is VideoDecoder {
return decoder instanceof VideoDecoder
}
function isAudioTrack(track: MP4.Track): track is MP4.AudioTrack {
return (track as MP4.AudioTrack).audio !== undefined;
}
function isVideoTrack(track: MP4.Track): track is MP4.VideoTrack {
return (track as MP4.VideoTrack).video !== undefined;
}

100
player/src/player/index.ts
View File

@ -1,33 +1,89 @@
import * as Message from "./message"
import * as Ring from "./ring"
import Transport from "../transport"
import Media from "../media"
export interface PlayerInit {
url: string;
fingerprint?: WebTransportHash; // the certificate fingerprint, temporarily needed for local development
canvas: HTMLCanvasElement;
export interface Config {
transport: Transport
canvas: OffscreenCanvas;
}
// This class must be created on the main thread due to AudioContext.
export default class Player {
media: Media;
transport: Transport;
context: AudioContext;
worker: Worker;
worklet: Promise<AudioWorkletNode>;
constructor(props: PlayerInit) {
this.media = new Media({
canvas: props.canvas.transferControlToOffscreen(),
})
transport: Transport
this.transport = new Transport({
url: props.url,
fingerprint: props.fingerprint,
media: this.media,
constructor(config: Config) {
this.transport = config.transport
this.transport.callback = this;
const video = {
canvas: config.canvas,
};
// Assume 44.1kHz and two audio channels
const audio = {
sampleRate: 44100,
ring: new Ring.Buffer(2, 4410), // 100ms at 44.1khz
}
this.context = new AudioContext({
latencyHint: "interactive",
sampleRate: audio.sampleRate,
})
}
async close() {
this.transport.close()
}
this.worker = this.setupWorker({ audio, video })
this.worklet = this.setupWorklet(audio)
}
play() {
this.media.play({})
}
private setupWorker(config: Message.Config): Worker {
const url = new URL('worker.ts', import.meta.url)
const worker = new Worker(url, {
type: "module",
name: "media",
})
worker.postMessage({ config }, [ config.video.canvas ])
return worker
}
private async setupWorklet(config: Message.AudioConfig): Promise<AudioWorkletNode> {
// Load the worklet source code.
const url = new URL('worklet.ts', import.meta.url)
await this.context.audioWorklet.addModule(url)
const volume = this.context.createGain()
volume.gain.value = 2.0;
// Create a worklet
const worklet = new AudioWorkletNode(this.context, 'renderer');
worklet.onprocessorerror = (e: Event) => {
console.error("Audio worklet error:", e)
};
worklet.port.postMessage({ config })
// Connect the worklet to the volume node and then to the speakers
worklet.connect(volume)
volume.connect(this.context.destination)
return worklet
}
onInit(init: Message.Init) {
this.worker.postMessage({ init }, [ init.buffer.buffer, init.reader ])
}
onSegment(segment: Message.Segment) {
this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ])
}
play(play: Message.Play) {
this.context.resume()
//this.worker.postMessage({ play })
}
}

5
player/src/media/message.ts → player/src/player/message.ts
View File

@ -1,5 +1,4 @@
import * as MP4 from "../mp4"
import { RingInit } from "../media/ring"
import * as Ring from "./ring"
export interface Config {
audio: AudioConfig;
@ -13,7 +12,7 @@ export interface VideoConfig {
export interface AudioConfig {
// audio stuff
sampleRate: number;
ring: RingInit;
ring: Ring.Buffer;
}
export interface Init {

11
player/src/media/renderer.ts → player/src/player/renderer.ts
View File

@ -68,8 +68,11 @@ export default class Renderer {
}
draw(now: DOMHighResTimeStamp) {
// Convert to microseconds
now *= 1000;
// Determine the target timestamp.
const target = 1000 * now - this.sync!
const target = now - this.sync!
this.drawAudio(now, target)
this.drawVideo(now, target)
@ -85,9 +88,11 @@ export default class Renderer {
// Check if we should skip some frames
while (this.audioQueue.length) {
const next = this.audioQueue[0]
if (next.timestamp >= target) {
if (next.timestamp > target) {
let ok = this.audioRing.write(next)
if (!ok) {
console.warn("ring buffer is full")
// No more space in the ring
break
}
@ -101,7 +106,7 @@ export default class Renderer {
}
drawVideo(now: DOMHighResTimeStamp, target: DOMHighResTimeStamp) {
if (this.videoQueue.length == 0) return;
if (!this.videoQueue.length) return;
let frame = this.videoQueue[0];
if (frame.timestamp >= target) {

10
player/src/media/ring.ts → player/src/player/ring.ts
View File

@ -11,15 +11,15 @@ export class Ring {
channels: Float32Array[];
capacity: number;
constructor(init: RingInit) {
this.state = new Int32Array(init.state)
constructor(buf: Buffer) {
this.state = new Int32Array(buf.state)
this.channels = []
for (let channel of init.channels) {
for (let channel of buf.channels) {
this.channels.push(new Float32Array(channel))
}
this.capacity = init.capacity
this.capacity = buf.capacity
}
// Add the samples for single audio frame
@ -121,7 +121,7 @@ export class Ring {
}
// No prototype to make this easier to send via postMessage
export class RingInit {
export class Buffer {
state: SharedArrayBuffer;
channels: SharedArrayBuffer[];

0
player/src/media/worker.ts → player/src/player/worker.ts
View File

6
player/src/media/worklet.ts → player/src/player/worklet.ts
View File

@ -20,12 +20,12 @@ class Renderer extends AudioWorkletProcessor {
onMessage(e: MessageEvent) {
if (e.data.config) {
this.config(e.data.config)
this.onConfig(e.data.config)
}
}
config(config: Message.Config) {
this.ring = new Ring(config.audio.ring)
onConfig(config: Message.AudioConfig) {
this.ring = new Ring(config.ring)
}
// Inputs and outputs in groups of 128 samples.

29
player/src/transport/index.ts
View File

@ -1,25 +1,18 @@
import * as Message from "./message"
import * as Stream from "../stream"
import * as MP4 from "../mp4"
import * as Interface from "./interface"
import Media from "../media"
export interface TransportInit {
export interface Config {
url: string;
fingerprint?: WebTransportHash; // the certificate fingerprint, temporarily needed for local development
media: Media;
}
export default class Transport {
quic: Promise<WebTransport>;
api: Promise<WritableStream>;
callback?: Interface.Callback;
media: Media;
constructor(props: TransportInit) {
this.media = props.media;
this.quic = this.connect(props)
constructor(config: Config) {
this.quic = this.connect(config)
// Create a unidirectional stream for all of our messages
this.api = this.quic.then((q) => {
@ -35,13 +28,13 @@ export default class Transport {
}
// Helper function to make creating a promise easier
private async connect(props: TransportInit): Promise<WebTransport> {
private async connect(config: Config): Promise<WebTransport> {
let options: WebTransportOptions = {};
if (props.fingerprint) {
options.serverCertificateHashes = [ props.fingerprint ]
if (config.fingerprint) {
options.serverCertificateHashes = [ config.fingerprint ]
}
const quic = new WebTransport(props.url, options)
const quic = new WebTransport(config.url, options)
await quic.ready
return quic
}
@ -86,12 +79,12 @@ export default class Transport {
const msg = JSON.parse(payload)
if (msg.init) {
return this.media.init({
return this.callback?.onInit({
buffer: r.buffer,
reader: r.reader,
})
} else if (msg.segment) {
return this.media.segment({
return this.callback?.onSegment({
buffer: r.buffer,
reader: r.reader,
})

14
player/src/transport/interface.ts Normal file
View File

@ -0,0 +1,14 @@
export interface Callback {
onInit(init: Init): any
onSegment(segment: Segment): any
}
export interface Init {
buffer: Uint8Array; // unread buffered data
reader: ReadableStream; // unread unbuffered data
}
export interface Segment {
buffer: Uint8Array; // unread buffered data
reader: ReadableStream; // unread unbuffered data
}

11
player/src/transport/message.ts
View File

@ -1,12 +1,5 @@
export interface Init {
id: string
}
export interface Segment {
init: string // id of the init segment
timestamp: number // presentation timestamp in milliseconds of the first sample
// TODO track would be nice
}
export interface Init {}
export interface Segment {}
export interface Debug {
max_bitrate: number