Merge pull request #10 from kixelated/webcodecs

WebCodecs
This commit is contained in:
kixelated 2023-04-06 14:09:06 -07:00 committed by GitHub
commit 5ba457bf65
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
45 changed files with 2240 additions and 14284 deletions

View File

@ -17,4 +17,4 @@ go run filippo.io/mkcert -ecdsa -install
go run filippo.io/mkcert -ecdsa -days 10 -cert-file "$CRT" -key-file "$KEY" localhost 127.0.0.1 ::1
# Compute the sha256 fingerprint of the certificate for WebTransport
openssl x509 -in "$CRT" -outform der | openssl dgst -sha256
openssl x509 -in "$CRT" -outform der | openssl dgst -sha256 > ../player/src/transport/fingerprint.hex

7
player/.proxyrc.js Normal file
View File

@ -0,0 +1,7 @@
module.exports = function (app) {
app.use((req, res, next) => {
res.setHeader('Cross-Origin-Opener-Policy', 'same-origin');
res.setHeader('Cross-Origin-Embedder-Policy', 'require-corp');
next();
});
};

4235
player/package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -6,8 +6,14 @@
"check": "tsc --noEmit"
},
"devDependencies": {
"@parcel/transformer-inline-string": "2.8.3",
"@parcel/validator-typescript": "^2.6.0",
"parcel": "^2.6.0",
"@types/audioworklet": "^0.0.41",
"@types/dom-webcodecs": "^0.1.6",
"parcel": "^2.8.0",
"typescript": ">=3.0.0"
},
"dependencies": {
"mp4box": "^0.5.2"
}
}

121
player/src/audio/decoder.ts Normal file
View File

@ -0,0 +1,121 @@
import * as Message from "./message";
import * as MP4 from "../mp4"
import * as Stream from "../stream"
import * as Util from "../util"
import Renderer from "./renderer"
export default class Decoder {
// Store the init message for each track
tracks: Map<string, Util.Deferred<Message.Init>>;
decoder: AudioDecoder; // TODO one per track
sync: Message.Sync;
constructor(config: Message.Config, renderer: Renderer) {
this.tracks = new Map();
this.decoder = new AudioDecoder({
output: renderer.emit.bind(renderer),
error: console.warn,
});
}
init(msg: Message.Init) {
let defer = this.tracks.get(msg.track);
if (!defer) {
defer = new Util.Deferred()
this.tracks.set(msg.track, defer)
}
if (msg.info.audioTracks.length != 1 || msg.info.videoTracks.length != 0) {
throw new Error("Expected a single audio track")
}
const track = msg.info.audioTracks[0]
const audio = track.audio
defer.resolve(msg)
}
async decode(msg: Message.Segment) {
let track = this.tracks.get(msg.track);
if (!track) {
track = new Util.Deferred()
this.tracks.set(msg.track, track)
}
// Wait for the init segment to be fully received and parsed
const init = await track.promise;
const audio = init.info.audioTracks[0]
if (this.decoder.state == "unconfigured") {
this.decoder.configure({
codec: audio.codec,
numberOfChannels: audio.audio.channel_count,
sampleRate: audio.audio.sample_rate,
})
}
const input = MP4.New();
input.onSamples = (id: number, user: any, samples: MP4.Sample[]) => {
for (let sample of samples) {
// Convert to microseconds
const timestamp = 1000 * 1000 * sample.dts / sample.timescale
const duration = 1000 * 1000 * sample.duration / sample.timescale
// This assumes that timescale == sample rate
this.decoder.decode(new EncodedAudioChunk({
type: sample.is_sync ? "key" : "delta",
data: sample.data,
duration: duration,
timestamp: timestamp,
}))
}
}
input.onReady = (info: any) => {
input.setExtractionOptions(info.tracks[0].id, {}, { nbSamples: 1 });
input.start();
}
// MP4box requires us to reparse the init segment unfortunately
let offset = 0;
for (let raw of init.raw) {
raw.fileStart = offset
input.appendBuffer(raw)
}
const stream = new Stream.Reader(msg.reader, msg.buffer)
/* TODO I'm not actually sure why this code doesn't work; something trips up the MP4 parser
while (1) {
const data = await stream.read()
if (!data) break
input.appendBuffer(data)
input.flush()
}
*/
// One day I'll figure it out; until then read one top-level atom at a time
while (!await stream.done()) {
const raw = await stream.peek(4)
const size = new DataView(raw.buffer, raw.byteOffset, raw.byteLength).getUint32(0)
const atom = await stream.bytes(size)
// Make a copy of the atom because mp4box only accepts an ArrayBuffer unfortunately
let box = new Uint8Array(atom.byteLength);
box.set(atom)
// and for some reason we need to modify the underlying ArrayBuffer with offset
let buffer = box.buffer as MP4.ArrayBuffer
buffer.fileStart = offset
// Parse the data
offset = input.appendBuffer(buffer)
input.flush()
}
}
}

77
player/src/audio/index.ts Normal file
View File

@ -0,0 +1,77 @@
import * as Message from "./message"
import Renderer from "./renderer"
import Decoder from "./decoder"
import { RingInit } from "./ring"
// Abstracts the Worker and Worklet into a simpler API
// This class must be created on the main thread due to AudioContext.
export default class Audio {
context: AudioContext;
worker: Worker;
worklet: Promise<AudioWorkletNode>;
constructor() {
// Assume 44.1kHz and two audio channels
const config = {
sampleRate: 44100,
ring: new RingInit(2, 4410), // 100ms at 44.1khz
}
this.context = new AudioContext({
latencyHint: "interactive",
sampleRate: config.sampleRate,
})
this.worker = this.setupWorker(config)
this.worklet = this.setupWorklet(config)
}
private setupWorker(config: Message.Config): Worker {
const url = new URL('worker.ts', import.meta.url)
const worker = new Worker(url, {
name: "audio",
type: "module",
})
worker.postMessage({ config })
return worker
}
private async setupWorklet(config: Message.Config): Promise<AudioWorkletNode> {
// Load the worklet source code.
const url = new URL('worklet.ts', import.meta.url)
await this.context.audioWorklet.addModule(url)
const volume = this.context.createGain()
volume.gain.value = 2.0;
// Create a worklet
const worklet = new AudioWorkletNode(this.context, 'renderer');
worklet.onprocessorerror = (e: Event) => {
console.error("Audio worklet error:", e)
};
worklet.port.postMessage({ config })
// Connect the worklet to the volume node and then to the speakers
worklet.connect(volume)
volume.connect(this.context.destination)
return worklet
}
init(init: Message.Init) {
this.worker.postMessage({ init })
}
segment(segment: Message.Segment) {
this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ])
}
play(play: Message.Play) {
this.context.resume()
//this.worker.postMessage({ play })
}
}

View File

@ -0,0 +1,30 @@
import * as MP4 from "../mp4"
import { RingInit } from "./ring"
export interface Config {
sampleRate: number;
ring: RingInit;
}
export interface Init {
track: string;
info: MP4.Info;
raw: MP4.ArrayBuffer[];
}
export interface Segment {
track: string;
buffer: Uint8Array; // unread buffered data
reader: ReadableStream; // unread unbuffered data
}
// Audio tells video when the given timestamp should be rendered.
export interface Sync {
origin: number;
clock: DOMHighResTimeStamp;
timestamp: number;
}
export interface Play {
timestamp?: number;
}

View File

@ -0,0 +1,85 @@
import * as Message from "./message"
import { Ring } from "./ring"
export default class Renderer {
ring: Ring;
queue: Array<AudioData>;
sync?: DOMHighResTimeStamp
running: number;
constructor(config: Message.Config) {
this.ring = new Ring(config.ring)
this.queue = [];
this.running = 0
}
emit(frame: AudioData) {
if (!this.sync) {
// Save the frame as the sync point
this.sync = 1000 * performance.now() - frame.timestamp
}
// Insert the frame into the queue sorted by timestamp.
if (this.queue.length > 0 && this.queue[this.queue.length-1].timestamp <= frame.timestamp) {
// Fast path because we normally append to the end.
this.queue.push(frame)
} else {
// Do a full binary search
let low = 0
let high = this.queue.length;
while (low < high) {
var mid = (low + high) >>> 1;
if (this.queue[mid].timestamp < frame.timestamp) low = mid + 1;
else high = mid;
}
this.queue.splice(low, 0, frame)
}
if (!this.running) {
// Wait for the next animation frame
this.running = self.requestAnimationFrame(this.render.bind(this))
}
}
render() {
// Determine the target timestamp.
const target = 1000 * performance.now() - this.sync!
// Check if we should skip some frames
while (this.queue.length) {
const next = this.queue[0]
if (next.timestamp >= target) {
break
}
console.warn("dropping audio")
this.queue.shift()
next.close()
}
// Push as many as we can to the ring buffer.
while (this.queue.length) {
let frame = this.queue[0]
let ok = this.ring.write(frame)
if (!ok) {
break
}
frame.close()
this.queue.shift()
}
if (this.queue.length) {
this.running = self.requestAnimationFrame(this.render.bind(this))
} else {
this.running = 0
}
}
play(play: Message.Play) {
this.ring.reset()
}
}

143
player/src/audio/ring.ts Normal file
View File

@ -0,0 +1,143 @@
// Ring buffer with audio samples.
enum STATE {
READ_INDEX = 0, // Index of the current read position (mod capacity)
WRITE_INDEX, // Index of the current write position (mod capacity)
LENGTH // Clever way of saving the total number of enums values.
}
export class Ring {
state: Int32Array;
channels: Float32Array[];
capacity: number;
constructor(init: RingInit) {
this.state = new Int32Array(init.state)
this.channels = []
for (let channel of init.channels) {
this.channels.push(new Float32Array(channel))
}
this.capacity = init.capacity
}
// Add the samples for single audio frame
write(frame: AudioData): boolean {
let count = frame.numberOfFrames;
let readIndex = Atomics.load(this.state, STATE.READ_INDEX)
let writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX)
let writeIndexNew = writeIndex + count;
// There's not enough space in the ring buffer
if (writeIndexNew - readIndex > this.capacity) {
return false
}
let startIndex = writeIndex % this.capacity;
let endIndex = writeIndexNew % this.capacity;
// Loop over each channel
for (let i = 0; i < this.channels.length; i += 1) {
const channel = this.channels[i]
if (startIndex < endIndex) {
// One continuous range to copy.
const full = channel.subarray(startIndex, endIndex)
frame.copyTo(full, {
planeIndex: i,
frameCount: count,
})
} else {
const first = channel.subarray(startIndex)
const second = channel.subarray(0, endIndex)
frame.copyTo(first, {
planeIndex: i,
frameCount: first.length,
})
frame.copyTo(second, {
planeIndex: i,
frameOffset: first.length,
frameCount: second.length,
})
}
}
Atomics.store(this.state, STATE.WRITE_INDEX, writeIndexNew)
return true
}
read(dst: Float32Array[]) {
let readIndex = Atomics.load(this.state, STATE.READ_INDEX)
let writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX)
if (readIndex >= writeIndex) {
// nothing to read
return
}
let readIndexNew = readIndex + dst[0].length
if (readIndexNew > writeIndex) {
// Partial read
readIndexNew = writeIndex
}
let startIndex = readIndex % this.capacity;
let endIndex = readIndexNew % this.capacity;
// Loop over each channel
for (let i = 0; i < dst.length; i += 1) {
if (i >= this.channels.length) {
// ignore excess channels
}
const input = this.channels[i]
const output = dst[i]
if (startIndex < endIndex) {
const full = input.subarray(startIndex, endIndex)
output.set(full)
} else {
const first = input.subarray(startIndex)
const second = input.subarray(0, endIndex)
output.set(first)
output.set(second, first.length)
}
}
Atomics.store(this.state, STATE.READ_INDEX, readIndexNew)
}
// TODO not thread safe
clear() {
const writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX)
Atomics.store(this.state, STATE.READ_INDEX, writeIndex)
}
}
// No prototype to make this easier to send via postMessage
export class RingInit {
state: SharedArrayBuffer;
channels: SharedArrayBuffer[];
capacity: number;
constructor(channels: number, capacity: number) {
// Store the current state in a separate ring buffer.
this.state = new SharedArrayBuffer(STATE.LENGTH * Int32Array.BYTES_PER_ELEMENT)
// Create a buffer for each audio channel
this.channels = []
for (let i = 0; i < channels; i += 1) {
const buffer = new SharedArrayBuffer(capacity * Float32Array.BYTES_PER_ELEMENT)
this.channels.push(buffer)
}
this.capacity = capacity
}
}

View File

@ -0,0 +1,26 @@
import Decoder from "./decoder"
import Renderer from "./renderer"
import * as Message from "./message"
let decoder: Decoder
let renderer: Renderer;
self.addEventListener('message', (e: MessageEvent) => {
if (e.data.config) {
renderer = new Renderer(e.data.config)
decoder = new Decoder(e.data.config, renderer)
}
if (e.data.init) {
decoder.init(e.data.init)
}
if (e.data.segment) {
decoder.decode(e.data.segment)
}
if (e.data.play) {
renderer.play(e.data.play)
}
})

View File

@ -0,0 +1,49 @@
// This is an AudioWorklet that acts as a media source.
// The renderer copies audio samples to a ring buffer read by this worklet.
// The worklet then outputs those samples to emit audio.
import * as Message from "./message"
import { Ring } from "./ring"
class Renderer extends AudioWorkletProcessor {
ring?: Ring;
base: number;
constructor(params: AudioWorkletNodeOptions) {
// The super constructor call is required.
super();
this.base = 0
this.port.onmessage = this.onMessage.bind(this)
}
onMessage(e: MessageEvent) {
if (e.data.config) {
this.config(e.data.config)
}
}
config(config: Message.Config) {
this.ring = new Ring(config.ring)
}
// Inputs and outputs in groups of 128 samples.
process(inputs: Float32Array[][], outputs: Float32Array[][], parameters: Record<string, Float32Array>): boolean {
if (!this.ring) {
// Not initialized yet
return true
}
if (inputs.length != 1 && outputs.length != 1) {
throw new Error("only a single track is supported")
}
const output = outputs[0]
this.ring.read(output)
return true;
}
}
registerProcessor("renderer", Renderer);

View File

@ -16,7 +16,7 @@ body {
position: relative;
}
#play {
#screen #play {
position: absolute;
width: 100%;
height: 100%;
@ -29,12 +29,6 @@ body {
z-index: 1;
}
#vid {
width: 100%;
height: 100%;
max-height: 100vh;
}
#controls {
display: flex;
flex-wrap: wrap;

View File

@ -11,8 +11,8 @@
<body>
<div id="player">
<div id="screen">
<div id="play"><span>click to play</span></div>
<video id="vid" controls></video>
<div id="play"><span>click for audio</span></div>
<canvas id="video" width="1280" height="720"></canvas>
</div>
<div id="controls">

View File

@ -1,48 +1,23 @@
import { Player } from "./player"
// This is so ghetto but I'm too lazy to improve it right now
const videoRef = document.querySelector<HTMLVideoElement>("video#vid")!;
const liveRef = document.querySelector<HTMLElement>("#live")!;
const throttleRef = document.querySelector<HTMLElement>("#throttle")!;
const statsRef = document.querySelector<HTMLElement>("#stats")!;
const playRef = document.querySelector<HTMLElement>("#play")!;
import Player from "./player"
const params = new URLSearchParams(window.location.search)
const url = params.get("url") || "https://localhost:4443/watch"
const canvas = document.querySelector<HTMLCanvasElement>("canvas#video")!
const player = new Player({
url: url,
videoRef: videoRef,
statsRef: statsRef,
throttleRef: throttleRef,
canvas: canvas,
})
liveRef.addEventListener("click", (e) => {
const play = document.querySelector<HTMLElement>("#screen #play")!
let playFunc = (e: Event) => {
player.play()
e.preventDefault()
player.goLive()
})
throttleRef.addEventListener("click", (e) => {
e.preventDefault()
player.throttle()
})
playRef.addEventListener('click', (e) => {
videoRef.play()
e.preventDefault()
})
function playFunc(e: Event) {
playRef.style.display = "none"
//player.goLive()
// Only fire once to restore pause/play functionality
videoRef.removeEventListener('play', playFunc)
play.removeEventListener('click', playFunc)
play.style.display = "none"
}
videoRef.addEventListener('play', playFunc)
videoRef.volume = 0.5
// Try to autoplay but ignore errors on mobile; they need to click
//vidRef.play().catch((e) => console.warn(e))
play.addEventListener('click', playFunc)

View File

@ -1,85 +0,0 @@
// Wrapper around MP4Box to play nicely with MP4Box.
// I tried getting a mp4box.all.d.ts file to work but just couldn't figure it out
import { createFile, ISOFile, DataStream, BoxParser } from "./mp4box.all"
// Rename some stuff so it's on brand.
export { createFile as MP4New, ISOFile as MP4File, DataStream as MP4Stream, BoxParser as MP4Parser }
export type MP4ArrayBuffer = ArrayBuffer & {fileStart: number};
export interface MP4MediaTrack {
id: number;
created: Date;
modified: Date;
movie_duration: number;
layer: number;
alternate_group: number;
volume: number;
track_width: number;
track_height: number;
timescale: number;
duration: number;
bitrate: number;
codec: string;
language: string;
nb_samples: number;
}
export interface MP4VideoData {
width: number;
height: number;
}
export interface MP4VideoTrack extends MP4MediaTrack {
video: MP4VideoData;
}
export interface MP4AudioData {
sample_rate: number;
channel_count: number;
sample_size: number;
}
export interface MP4AudioTrack extends MP4MediaTrack {
audio: MP4AudioData;
}
export type MP4Track = MP4VideoTrack | MP4AudioTrack;
export interface MP4Info {
duration: number;
timescale: number;
fragment_duration: number;
isFragmented: boolean;
isProgressive: boolean;
hasIOD: boolean;
brands: string[];
created: Date;
modified: Date;
tracks: MP4Track[];
mime: string;
videoTracks: MP4Track[];
audioTracks: MP4Track[];
}
export interface MP4Sample {
number: number;
track_id: number;
timescale: number;
description_index: number;
description: any;
data: ArrayBuffer;
size: number;
alreadyRead: number;
duration: number;
cts: number;
dts: number;
is_sync: boolean;
is_leading: number;
depends_on: number;
is_depended_on: number;
has_redundancy: number;
degration_priority: number;
offset: number;
subsamples: any;
}

11
player/src/mp4/index.ts Normal file
View File

@ -0,0 +1,11 @@
// Rename some stuff so it's on brand.
export {
createFile as New,
MP4File as File,
MP4ArrayBuffer as ArrayBuffer,
MP4Info as Info,
DataStream as Stream,
Sample,
} from "mp4box"
export { Init, InitParser } from "./init"

View File

@ -1,24 +1,28 @@
import { MP4New, MP4File, MP4ArrayBuffer, MP4Info } from "./mp4"
import * as MP4 from "./index"
export interface Init {
raw: MP4.ArrayBuffer;
info: MP4.Info;
}
export class InitParser {
mp4box: MP4File;
mp4box: MP4.File;
offset: number;
raw: MP4ArrayBuffer[];
ready: Promise<Init>;
raw: MP4.ArrayBuffer[];
info: Promise<MP4.Info>;
constructor() {
this.mp4box = MP4New()
this.mp4box = MP4.New()
this.raw = []
this.offset = 0
// Create a promise that gets resolved once the init segment has been parsed.
this.ready = new Promise((resolve, reject) => {
this.info = new Promise((resolve, reject) => {
this.mp4box.onError = reject
// https://github.com/gpac/mp4box.js#onreadyinfo
this.mp4box.onReady = (info: MP4Info) => {
this.mp4box.onReady = (info: MP4.Info) => {
if (!info.isFragmented) {
reject("expected a fragmented mp4")
}
@ -27,10 +31,7 @@ export class InitParser {
reject("expected a single track")
}
resolve({
info: info,
raw: this.raw,
})
resolve(info)
}
})
}
@ -41,7 +42,7 @@ export class InitParser {
box.set(data)
// and for some reason we need to modify the underlying ArrayBuffer with fileStart
let buffer = box.buffer as MP4ArrayBuffer
let buffer = box.buffer as MP4.ArrayBuffer
buffer.fileStart = this.offset
// Parse the data
@ -52,8 +53,3 @@ export class InitParser {
this.raw.push(buffer)
}
}
export interface Init {
raw: MP4ArrayBuffer[];
info: MP4Info;
}

148
player/src/mp4/mp4box.d.ts vendored Normal file
View File

@ -0,0 +1,148 @@
// https://github.com/gpac/mp4box.js/issues/233
declare module "mp4box" {
interface MP4MediaTrack {
id: number;
created: Date;
modified: Date;
movie_duration: number;
layer: number;
alternate_group: number;
volume: number;
track_width: number;
track_height: number;
timescale: number;
duration: number;
bitrate: number;
codec: string;
language: string;
nb_samples: number;
}
interface MP4VideoData {
width: number;
height: number;
}
interface MP4VideoTrack extends MP4MediaTrack {
video: MP4VideoData;
}
interface MP4AudioData {
sample_rate: number;
channel_count: number;
sample_size: number;
}
interface MP4AudioTrack extends MP4MediaTrack {
audio: MP4AudioData;
}
type MP4Track = MP4VideoTrack | MP4AudioTrack;
export interface MP4Info {
duration: number;
timescale: number;
fragment_duration: number;
isFragmented: boolean;
isProgressive: boolean;
hasIOD: boolean;
brands: string[];
created: Date;
modified: Date;
tracks: MP4Track[];
mime: string;
audioTracks: MP4AudioTrack[];
videoTracks: MP4VideoTrack[];
}
export type MP4ArrayBuffer = ArrayBuffer & {fileStart: number};
export interface MP4File {
onMoovStart?: () => void;
onReady?: (info: MP4Info) => void;
onError?: (e: string) => void;
onSamples?: (id: number, user: any, samples: Sample[]) => void;
appendBuffer(data: MP4ArrayBuffer): number;
start(): void;
stop(): void;
flush(): void;
setExtractionOptions(id: number, user: any, options: ExtractionOptions): void;
}
export function createFile(): MP4File;
export interface Sample {
number: number;
track_id: number;
timescale: number;
description_index: number;
description: any;
data: ArrayBuffer;
size: number;
alreadyRead: number;
duration: number;
cts: number;
dts: number;
is_sync: boolean;
is_leading: number;
depends_on: number;
is_depended_on: number;
has_redundancy: number;
degration_priority: number;
offset: number;
subsamples: any;
}
export interface ExtractionOptions {
nbSamples: number;
}
const BIG_ENDIAN: boolean;
const LITTLE_ENDIAN: boolean;
export class DataStream {
constructor(buffer: ArrayBuffer, byteOffset?: number, littleEndian?: boolean);
getPosition(): number;
get byteLength(): number;
get buffer(): ArrayBuffer;
set buffer(v: ArrayBuffer);
get byteOffset(): number;
set byteOffset(v: number);
get dataView(): DataView;
set dataView(v: DataView);
seek(pos: number): void;
isEof(): boolean;
mapUint8Array(length: number): Uint8Array;
readInt32Array(length: number, littleEndian: boolean): Int32Array;
readInt16Array(length: number, littleEndian: boolean): Int16Array;
readInt8(length: number): Int8Array;
readUint32Array(length: number, littleEndian: boolean): Uint32Array;
readUint16Array(length: number, littleEndian: boolean): Uint16Array;
readUint8(length: number): Uint8Array;
readFloat64Array(length: number, littleEndian: boolean): Float64Array;
readFloat32Array(length: number, littleEndian: boolean): Float32Array;
readInt32(littleEndian: boolean): number;
readInt16(littleEndian: boolean): number;
readInt8(): number;
readUint32(littleEndian: boolean): number;
readUint16(littleEndian: boolean): number;
readUint8(): number;
readFloat32(littleEndian: boolean): number;
readFloat64(littleEndian: boolean): number;
endianness: boolean;
memcpy(dst: ArrayBufferLike, dstOffset: number, src: ArrayBufferLike, srcOffset: number, byteLength: number): void;
// TODO I got bored porting the remaining functions
}
export { };
}

File diff suppressed because it is too large Load Diff

View File

@ -1,374 +0,0 @@
import { Source } from "./source"
import { StreamReader, StreamWriter } from "./stream"
import { InitParser } from "./init"
import { Segment } from "./segment"
import { Track } from "./track"
import { Message, MessageInit, MessageSegment } from "./message"
///<reference path="./types/webtransport.d.ts"/>
export interface PlayerInit {
url: string;
videoRef: HTMLVideoElement;
statsRef: HTMLElement;
throttleRef: HTMLElement;
}
/*
*/
export class Player {
mediaSource: MediaSource;
init: Map<string, InitParser>;
audio: Track;
video: Track;
quic: Promise<WebTransport>;
api: Promise<WritableStream>;
// References to elements in the DOM
vidRef: HTMLVideoElement; // The video element itself
statsRef: HTMLElement; // The stats div
throttleRef: HTMLElement; // The throttle button
throttleCount: number; // number of times we've clicked the button in a row
interval: number;
timeRef?: DOMHighResTimeStamp;
constructor(props: PlayerInit) {
this.vidRef = props.videoRef
this.statsRef = props.statsRef
this.throttleRef = props.throttleRef
this.throttleCount = 0
this.mediaSource = new MediaSource()
this.vidRef.src = URL.createObjectURL(this.mediaSource)
this.init = new Map()
this.audio = new Track(new Source(this.mediaSource));
this.video = new Track(new Source(this.mediaSource));
this.interval = setInterval(this.tick.bind(this), 100)
this.vidRef.addEventListener("waiting", this.tick.bind(this))
this.quic = this.connect(props.url)
// Create a unidirectional stream for all of our messages
this.api = this.quic.then((q) => {
return q.createUnidirectionalStream()
})
// async functions
this.receiveStreams()
// Limit to 4Mb/s
this.sendThrottle()
}
async close() {
clearInterval(this.interval);
(await this.quic).close()
}
async connect(url: string): Promise<WebTransport> {
// TODO remove this when WebTransport supports the system CA pool
const fingerprintURL = new URL(url);
fingerprintURL.pathname = "/fingerprint"
const response = await fetch(fingerprintURL)
if (!response.ok) {
throw new Error('failed to get server fingerprint');
}
const hex = await response.text()
// Convert the hex to binary.
let fingerprint = [];
for (let c = 0; c < hex.length; c += 2) {
fingerprint.push(parseInt(hex.substring(c, c+2), 16));
}
//const fingerprint = Uint8Array.from(atob(hex), c => c.charCodeAt(0))
const quic = new WebTransport(url, {
"serverCertificateHashes": [{
"algorithm": "sha-256",
"value": new Uint8Array(fingerprint),
}]
})
await quic.ready
return quic
}
async sendMessage(msg: any) {
const payload = JSON.stringify(msg)
const size = payload.length + 8
const stream = await this.api
const writer = new StreamWriter(stream)
await writer.uint32(size)
await writer.string("warp")
await writer.string(payload)
writer.release()
}
throttle() {
// Throttle is incremented each time we click the throttle button
this.throttleCount += 1
this.sendThrottle()
// After 5 seconds disable the throttling
setTimeout(() => {
this.throttleCount -= 1
this.sendThrottle()
}, 5000)
}
sendThrottle() {
let rate = 0;
if (this.throttleCount > 0) {
// TODO detect the incoming bitrate instead of hard-coding
// Right shift by throttle to divide by 2,4,8,16,etc each time
const bitrate = 4 * 1024 * 1024 // 4Mb/s
rate = bitrate >> (this.throttleCount-1)
const str = formatBits(rate) + "/s"
this.throttleRef.textContent = `Throttle: ${ str }`;
} else {
this.throttleRef.textContent = "Throttle: none";
}
// Send the server a message to fake network congestion.
this.sendMessage({
"debug": {
max_bitrate: rate,
},
})
}
tick() {
// Try skipping ahead if there's no data in the current buffer.
this.trySeek()
// Try skipping video if it would fix any desync.
this.trySkip()
// Update the stats at the end
this.updateStats()
}
goLive() {
const ranges = this.vidRef.buffered
if (!ranges.length) {
return
}
this.vidRef.currentTime = ranges.end(ranges.length-1);
this.vidRef.play();
}
// Try seeking ahead to the next buffered range if there's a gap
trySeek() {
if (this.vidRef.readyState > 2) { // HAVE_CURRENT_DATA
// No need to seek
return
}
const ranges = this.vidRef.buffered
if (!ranges.length) {
// Video has not started yet
return
}
for (let i = 0; i < ranges.length; i += 1) {
const pos = ranges.start(i)
if (this.vidRef.currentTime >= pos) {
// This would involve seeking backwards
continue
}
console.warn("seeking forward", pos - this.vidRef.currentTime)
this.vidRef.currentTime = pos
return
}
}
// Try dropping video frames if there is future data available.
trySkip() {
let playhead: number | undefined
if (this.vidRef.readyState > 2) {
// If we're not buffering, only skip video if it's before the current playhead
playhead = this.vidRef.currentTime
}
this.video.advance(playhead)
}
async receiveStreams() {
const q = await this.quic
const streams = q.incomingUnidirectionalStreams.getReader()
while (true) {
const result = await streams.read()
if (result.done) break
const stream = result.value
this.handleStream(stream) // don't await
}
}
async handleStream(stream: ReadableStream) {
let r = new StreamReader(stream.getReader())
while (!await r.done()) {
const size = await r.uint32();
const typ = new TextDecoder('utf-8').decode(await r.bytes(4));
if (typ != "warp") throw "expected warp atom"
if (size < 8) throw "atom too small"
const payload = new TextDecoder('utf-8').decode(await r.bytes(size - 8));
const msg = JSON.parse(payload) as Message
if (msg.init) {
return this.handleInit(r, msg.init)
} else if (msg.segment) {
return this.handleSegment(r, msg.segment)
}
}
}
async handleInit(stream: StreamReader, msg: MessageInit) {
let init = this.init.get(msg.id);
if (!init) {
init = new InitParser()
this.init.set(msg.id, init)
}
while (1) {
const data = await stream.read()
if (!data) break
init.push(data)
}
}
async handleSegment(stream: StreamReader, msg: MessageSegment) {
let pending = this.init.get(msg.init);
if (!pending) {
pending = new InitParser()
this.init.set(msg.init, pending)
}
// Wait for the init segment to be fully received and parsed
const init = await pending.ready;
let track: Track;
if (init.info.videoTracks.length) {
track = this.video
} else {
track = this.audio
}
const segment = new Segment(track.source, init, msg.timestamp)
// The track is responsible for flushing the segments in order
track.add(segment)
/* TODO I'm not actually sure why this code doesn't work; something trips up the MP4 parser
while (1) {
const data = await stream.read()
if (!data) break
segment.push(data)
track.flush() // Flushes if the active segment has samples
}
*/
// One day I'll figure it out; until then read one top-level atom at a time
while (!await stream.done()) {
const raw = await stream.peek(4)
const size = new DataView(raw.buffer, raw.byteOffset, raw.byteLength).getUint32(0)
const atom = await stream.bytes(size)
segment.push(atom)
track.flush() // Flushes if the active segment has new samples
}
segment.finish()
}
updateStats() {
for (const child of this.statsRef.children) {
if (child.className == "audio buffer") {
const ranges: any = (this.audio) ? this.audio.buffered() : { length: 0 }
this.visualizeBuffer(child as HTMLElement, ranges)
} else if (child.className == "video buffer") {
const ranges: any = (this.video) ? this.video.buffered() : { length: 0 }
this.visualizeBuffer(child as HTMLElement, ranges)
}
}
}
visualizeBuffer(element: HTMLElement, ranges: TimeRanges) {
const children = element.children
const max = 5
let index = 0
let prev = 0
for (let i = 0; i < ranges.length; i += 1) {
let start = ranges.start(i) - this.vidRef.currentTime
let end = ranges.end(i) - this.vidRef.currentTime
if (end < 0 || start > max) {
continue
}
let fill: HTMLElement;
if (index < children.length) {
fill = children[index] as HTMLElement;
} else {
fill = document.createElement("div")
element.appendChild(fill)
}
fill.className = "fill"
fill.innerHTML = end.toFixed(2)
fill.setAttribute('style', "left: " + (100 * Math.max(start, 0) / max) + "%; right: " + (100 - 100 * Math.min(end, max) / max) + "%")
index += 1
prev = end
}
for (let i = index; i < children.length; i += 1) {
element.removeChild(children[i])
}
}
}
// https://stackoverflow.com/questions/15900485/correct-way-to-convert-size-in-bytes-to-kb-mb-gb-in-javascript
function formatBits(bits: number, decimals: number = 1) {
if (bits === 0) return '0 bits';
const k = 1024;
const dm = decimals < 0 ? 0 : decimals;
const sizes = ['b', 'Kb', 'Mb', 'Gb', 'Tb', 'Pb', 'Eb', 'Zb', 'Yb'];
const i = Math.floor(Math.log(bits) / Math.log(k));
return parseFloat((bits / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
}

View File

@ -0,0 +1,46 @@
import Audio from "../audio"
import Transport from "../transport"
import Video from "../video"
export interface PlayerInit {
url: string;
canvas: HTMLCanvasElement;
}
export default class Player {
audio: Audio;
video: Video;
transport: Transport;
constructor(props: PlayerInit) {
this.audio = new Audio()
this.video = new Video({
canvas: props.canvas.transferControlToOffscreen(),
})
this.transport = new Transport({
url: props.url,
audio: this.audio,
video: this.video,
})
}
async close() {
this.transport.close()
}
async connect(url: string) {
await this.transport.connect(url)
}
play() {
this.audio.play({})
//this.video.play()
}
onMessage(msg: any) {
if (msg.sync) {
msg.sync
}
}
}

View File

@ -1,150 +0,0 @@
import { Source } from "./source"
import { Init } from "./init"
import { MP4New, MP4File, MP4Sample, MP4Stream, MP4Parser, MP4ArrayBuffer } from "./mp4"
// Manage a segment download, keeping a buffer of a single sample to potentially rewrite the duration.
export class Segment {
source: Source; // The SourceBuffer used to decode media.
offset: number; // The byte offset in the received file so far
samples: MP4Sample[]; // The samples ready to be flushed to the source.
timestamp: number; // The expected timestamp of the first sample in milliseconds
init: Init;
dts?: number; // The parsed DTS of the first sample
timescale?: number; // The parsed timescale of the segment
input: MP4File; // MP4Box file used to parse the incoming atoms.
output: MP4File; // MP4Box file used to write the outgoing atoms after modification.
done: boolean; // The segment has been completed
constructor(source: Source, init: Init, timestamp: number) {
this.source = source
this.offset = 0
this.done = false
this.timestamp = timestamp
this.init = init
this.input = MP4New();
this.output = MP4New();
this.samples = [];
this.input.onReady = (info: any) => {
this.input.setExtractionOptions(info.tracks[0].id, {}, { nbSamples: 1 });
this.input.onSamples = this.onSamples.bind(this)
this.input.start();
}
// We have to reparse the init segment to work with mp4box
for (let i = 0; i < init.raw.length; i += 1) {
this.offset = this.input.appendBuffer(init.raw[i])
// Also populate the output with our init segment so it knows about tracks
this.output.appendBuffer(init.raw[i])
}
this.input.flush()
this.output.flush()
}
push(data: Uint8Array) {
if (this.done) return; // ignore new data after marked done
// Make a copy of the atom because mp4box only accepts an ArrayBuffer unfortunately
let box = new Uint8Array(data.byteLength);
box.set(data)
// and for some reason we need to modify the underlying ArrayBuffer with offset
let buffer = box.buffer as MP4ArrayBuffer
buffer.fileStart = this.offset
// Parse the data
this.offset = this.input.appendBuffer(buffer)
this.input.flush()
}
onSamples(id: number, user: any, samples: MP4Sample[]) {
if (!samples.length) return;
if (this.dts === undefined) {
this.dts = samples[0].dts;
this.timescale = samples[0].timescale;
}
// Add the samples to a queue
this.samples.push(...samples)
}
// Flushes any pending samples, returning true if the stream has finished.
flush(): boolean {
let stream = new MP4Stream(new ArrayBuffer(0), 0, false); // big-endian
while (this.samples.length) {
// Keep a single sample if we're not done yet
if (!this.done && this.samples.length < 2) break;
const sample = this.samples.shift()
if (!sample) break;
let moof = this.output.createSingleSampleMoof(sample);
moof.write(stream);
// adjusting the data_offset now that the moof size is known
moof.trafs[0].truns[0].data_offset = moof.size+8; //8 is mdat header
stream.adjustUint32(moof.trafs[0].truns[0].data_offset_position, moof.trafs[0].truns[0].data_offset);
// @ts-ignore
var mdat = new MP4Parser.mdatBox();
mdat.data = sample.data;
mdat.write(stream);
}
this.source.initialize(this.init)
this.source.append(stream.buffer as ArrayBuffer)
return this.done
}
// The segment has completed
finish() {
this.done = true
this.flush()
// Trim the buffer to 30s long after each segment.
this.source.trim(30)
}
// Extend the last sample so it reaches the provided timestamp
skipTo(pts: number) {
if (this.samples.length == 0) return
let last = this.samples[this.samples.length-1]
const skip = pts - (last.dts + last.duration);
if (skip == 0) return;
if (skip < 0) throw "can't skip backwards"
last.duration += skip
if (this.timescale) {
console.warn("skipping video", skip / this.timescale)
}
}
buffered() {
// Ignore if we have a single sample
if (this.samples.length <= 1) return undefined;
if (!this.timescale) return undefined;
const first = this.samples[0];
const last = this.samples[this.samples.length-1]
return {
length: 1,
start: first.dts / this.timescale,
end: (last.dts + last.duration) / this.timescale,
}
}
}

View File

@ -1,147 +0,0 @@
import { Init } from "./init"
// Create a SourceBuffer with convenience methods
export class Source {
sourceBuffer?: SourceBuffer;
mediaSource: MediaSource;
queue: Array<SourceInit | SourceData | SourceTrim>;
init?: Init;
constructor(mediaSource: MediaSource) {
this.mediaSource = mediaSource;
this.queue = [];
}
// (re)initialize the source using the provided init segment.
initialize(init: Init) {
// Check if the init segment is already in the queue.
for (let i = this.queue.length - 1; i >= 0; i--) {
if ((this.queue[i] as SourceInit).init == init) {
// Already queued up.
return
}
}
// Check if the init segment has already been applied.
if (this.init == init) {
return
}
// Add the init segment to the queue so we call addSourceBuffer or changeType
this.queue.push({
kind: "init",
init: init,
})
for (let i = 0; i < init.raw.length; i += 1) {
this.queue.push({
kind: "data",
data: init.raw[i],
})
}
this.flush()
}
// Append the segment data to the buffer.
append(data: Uint8Array | ArrayBuffer) {
this.queue.push({
kind: "data",
data: data,
})
this.flush()
}
// Return the buffered range.
buffered() {
if (!this.sourceBuffer) {
return { length: 0 }
}
return this.sourceBuffer.buffered
}
// Delete any media older than x seconds from the buffer.
trim(duration: number) {
this.queue.push({
kind: "trim",
trim: duration,
})
this.flush()
}
// Flush any queued instructions
flush() {
while (1) {
// Check if the buffer is currently busy.
if (this.sourceBuffer && this.sourceBuffer.updating) {
break;
}
// Process the next item in the queue.
const next = this.queue.shift()
if (!next) {
break;
}
switch (next.kind) {
case "init":
this.init = next.init;
if (!this.sourceBuffer) {
// Create a new source buffer.
this.sourceBuffer = this.mediaSource.addSourceBuffer(this.init.info.mime)
// Call flush automatically after each update finishes.
this.sourceBuffer.addEventListener('updateend', this.flush.bind(this))
} else {
this.sourceBuffer.changeType(next.init.info.mime)
}
break;
case "data":
if (!this.sourceBuffer) {
throw "failed to call initailize before append"
}
this.sourceBuffer.appendBuffer(next.data)
break;
case "trim":
if (!this.sourceBuffer) {
throw "failed to call initailize before trim"
}
const end = this.sourceBuffer.buffered.end(this.sourceBuffer.buffered.length - 1) - next.trim;
const start = this.sourceBuffer.buffered.start(0)
if (end > start) {
this.sourceBuffer.remove(start, end)
}
break;
default:
throw "impossible; unknown SourceItem"
}
}
}
}
interface SourceItem {}
class SourceInit implements SourceItem {
kind!: "init";
init!: Init;
}
class SourceData implements SourceItem {
kind!: "data";
data!: Uint8Array | ArrayBuffer;
}
class SourceTrim implements SourceItem {
kind!: "trim";
trim!: number;
}

View File

@ -0,0 +1,2 @@
export { default as Reader } from "./reader"
export { default as Writer } from "./writer"

View File

@ -1,33 +1,64 @@
// Reader wraps a stream and provides convience methods for reading pieces from a stream
export class StreamReader {
reader: ReadableStreamDefaultReader; // TODO make a separate class without promises when null
export default class Reader {
reader: ReadableStream;
buffer: Uint8Array;
constructor(reader: ReadableStreamDefaultReader, buffer: Uint8Array = new Uint8Array(0)) {
constructor(reader: ReadableStream, buffer: Uint8Array = new Uint8Array(0)) {
this.reader = reader
this.buffer = buffer
}
// TODO implementing pipeTo seems more reasonable than releasing the lock
release() {
this.reader.releaseLock()
}
// Returns any number of bytes
async read(): Promise<Uint8Array | undefined> {
if (this.buffer.byteLength) {
const buffer = this.buffer;
this.buffer = new Uint8Array()
return buffer
}
const result = await this.reader.read()
const r = this.reader.getReader()
const result = await r.read()
r.releaseLock()
return result.value
}
async readAll(): Promise<Uint8Array> {
const r = this.reader.getReader()
while (1) {
const result = await r.read()
if (result.done) {
break
}
const buffer = new Uint8Array(result.value)
if (this.buffer.byteLength == 0) {
this.buffer = buffer
} else {
const temp = new Uint8Array(this.buffer.byteLength + buffer.byteLength)
temp.set(this.buffer)
temp.set(buffer, this.buffer.byteLength)
this.buffer = temp
}
}
const result = this.buffer
this.buffer = new Uint8Array()
r.releaseLock()
return result
}
async bytes(size: number): Promise<Uint8Array> {
const r = this.reader.getReader()
while (this.buffer.byteLength < size) {
const result = await this.reader.read()
const result = await r.read()
if (result.done) {
throw "short buffer"
}
@ -47,12 +78,16 @@ export class StreamReader {
const result = new Uint8Array(this.buffer.buffer, this.buffer.byteOffset, size)
this.buffer = new Uint8Array(this.buffer.buffer, this.buffer.byteOffset + size)
r.releaseLock()
return result
}
async peek(size: number): Promise<Uint8Array> {
const r = this.reader.getReader()
while (this.buffer.byteLength < size) {
const result = await this.reader.read()
const result = await r.read()
if (result.done) {
throw "short buffer"
}
@ -69,7 +104,11 @@ export class StreamReader {
}
}
return new Uint8Array(this.buffer.buffer, this.buffer.byteOffset, size)
const result = new Uint8Array(this.buffer.buffer, this.buffer.byteOffset, size)
r.releaseLock()
return result
}
async view(size: number): Promise<DataView> {
@ -151,104 +190,3 @@ export class StreamReader {
}
}
}
// StreamWriter wraps a stream and writes chunks of data
export class StreamWriter {
buffer: ArrayBuffer;
writer: WritableStreamDefaultWriter;
constructor(stream: WritableStream) {
this.buffer = new ArrayBuffer(8)
this.writer = stream.getWriter()
}
release() {
this.writer.releaseLock()
}
async close() {
return this.writer.close()
}
async uint8(v: number) {
const view = new DataView(this.buffer, 0, 1)
view.setUint8(0, v)
return this.writer.write(view)
}
async uint16(v: number) {
const view = new DataView(this.buffer, 0, 2)
view.setUint16(0, v)
return this.writer.write(view)
}
async uint24(v: number) {
const v1 = (v >> 16) & 0xff
const v2 = (v >> 8) & 0xff
const v3 = (v) & 0xff
const view = new DataView(this.buffer, 0, 3)
view.setUint8(0, v1)
view.setUint8(1, v2)
view.setUint8(2, v3)
return this.writer.write(view)
}
async uint32(v: number) {
const view = new DataView(this.buffer, 0, 4)
view.setUint32(0, v)
return this.writer.write(view)
}
async uint52(v: number) {
if (v > Number.MAX_SAFE_INTEGER) {
throw "value too large"
}
this.uint64(BigInt(v))
}
async vint52(v: number) {
if (v > Number.MAX_SAFE_INTEGER) {
throw "value too large"
}
if (v < (1 << 6)) {
return this.uint8(v)
} else if (v < (1 << 14)) {
return this.uint16(v|0x4000)
} else if (v < (1 << 30)) {
return this.uint32(v|0x80000000)
} else {
return this.uint64(BigInt(v) | 0xc000000000000000n)
}
}
async uint64(v: bigint) {
const view = new DataView(this.buffer, 0, 8)
view.setBigUint64(0, v)
return this.writer.write(view)
}
async vint64(v: bigint) {
if (v < (1 << 6)) {
return this.uint8(Number(v))
} else if (v < (1 << 14)) {
return this.uint16(Number(v)|0x4000)
} else if (v < (1 << 30)) {
return this.uint32(Number(v)|0x80000000)
} else {
return this.uint64(v | 0xc000000000000000n)
}
}
async bytes(buffer: ArrayBuffer) {
return this.writer.write(buffer)
}
async string(str: string) {
const data = new TextEncoder().encode(str)
return this.writer.write(data)
}
}

100
player/src/stream/writer.ts Normal file
View File

@ -0,0 +1,100 @@
// Writer wraps a stream and writes chunks of data
export default class Writer {
buffer: ArrayBuffer;
writer: WritableStreamDefaultWriter;
constructor(stream: WritableStream) {
this.buffer = new ArrayBuffer(8)
this.writer = stream.getWriter()
}
release() {
this.writer.releaseLock()
}
async close() {
return this.writer.close()
}
async uint8(v: number) {
const view = new DataView(this.buffer, 0, 1)
view.setUint8(0, v)
return this.writer.write(view)
}
async uint16(v: number) {
const view = new DataView(this.buffer, 0, 2)
view.setUint16(0, v)
return this.writer.write(view)
}
async uint24(v: number) {
const v1 = (v >> 16) & 0xff
const v2 = (v >> 8) & 0xff
const v3 = (v) & 0xff
const view = new DataView(this.buffer, 0, 3)
view.setUint8(0, v1)
view.setUint8(1, v2)
view.setUint8(2, v3)
return this.writer.write(view)
}
async uint32(v: number) {
const view = new DataView(this.buffer, 0, 4)
view.setUint32(0, v)
return this.writer.write(view)
}
async uint52(v: number) {
if (v > Number.MAX_SAFE_INTEGER) {
throw "value too large"
}
this.uint64(BigInt(v))
}
async vint52(v: number) {
if (v > Number.MAX_SAFE_INTEGER) {
throw "value too large"
}
if (v < (1 << 6)) {
return this.uint8(v)
} else if (v < (1 << 14)) {
return this.uint16(v|0x4000)
} else if (v < (1 << 30)) {
return this.uint32(v|0x80000000)
} else {
return this.uint64(BigInt(v) | 0xc000000000000000n)
}
}
async uint64(v: bigint) {
const view = new DataView(this.buffer, 0, 8)
view.setBigUint64(0, v)
return this.writer.write(view)
}
async vint64(v: bigint) {
if (v < (1 << 6)) {
return this.uint8(Number(v))
} else if (v < (1 << 14)) {
return this.uint16(Number(v)|0x4000)
} else if (v < (1 << 30)) {
return this.uint32(Number(v)|0x80000000)
} else {
return this.uint64(v | 0xc000000000000000n)
}
}
async bytes(buffer: ArrayBuffer) {
return this.writer.write(buffer)
}
async string(str: string) {
const data = new TextEncoder().encode(str)
return this.writer.write(data)
}
}

View File

@ -1,124 +0,0 @@
import { Source } from "./source"
import { Segment } from "./segment"
import { TimeRange } from "./util"
// An audio or video track that consists of multiple sequential segments.
//
// Instead of buffering, we want to drop video while audio plays uninterupted.
// Chrome actually plays up to 3s of audio without video before buffering when in low latency mode.
// Unforuntately, this does not recover correctly when there are gaps (pls fix).
// Our solution is to flush segments in decode order, buffering a single additional frame.
// We extend the duration of the buffered frame and flush it to cover any gaps.
export class Track {
source: Source;
segments: Segment[];
constructor(source: Source) {
this.source = source;
this.segments = [];
}
add(segment: Segment) {
// TODO don't add if the segment is out of date already
this.segments.push(segment)
// Sort by timestamp ascending
// NOTE: The timestamp is in milliseconds, and we need to parse the media to get the accurate PTS/DTS.
this.segments.sort((a: Segment, b: Segment): number => {
return a.timestamp - b.timestamp
})
}
buffered(): TimeRanges {
let ranges: TimeRange[] = []
const buffered = this.source.buffered() as TimeRanges
for (let i = 0; i < buffered.length; i += 1) {
// Convert the TimeRanges into an oject we can modify
ranges.push({
start: buffered.start(i),
end: buffered.end(i)
})
}
// Loop over segments and add in their ranges, merging if possible.
for (let segment of this.segments) {
const buffered = segment.buffered()
if (!buffered) continue;
if (ranges.length) {
// Try to merge with an existing range
const last = ranges[ranges.length-1];
if (buffered.start < last.start) {
// Network buffer is old; ignore it
continue
}
// Extend the end of the last range instead of pushing
if (buffered.start <= last.end && buffered.end > last.end) {
last.end = buffered.end
continue
}
}
ranges.push(buffered)
}
// TODO typescript
return {
length: ranges.length,
start: (x) => { return ranges[x].start },
end: (x) => { return ranges[x].end },
}
}
flush() {
while (1) {
if (!this.segments.length) break
const first = this.segments[0]
const done = first.flush()
if (!done) break
this.segments.shift()
}
}
// Given the current playhead, determine if we should drop any segments
// If playhead is undefined, it means we're buffering so skip to anything now.
advance(playhead: number | undefined) {
if (this.segments.length < 2) return
while (this.segments.length > 1) {
const current = this.segments[0];
const next = this.segments[1];
if (next.dts === undefined || next.timescale == undefined) {
// No samples have been parsed for the next segment yet.
break
}
if (current.dts === undefined) {
// No samples have been parsed for the current segment yet.
// We can't cover the gap by extending the sample so we have to seek.
// TODO I don't think this can happen, but I guess we have to seek past the gap.
break
}
if (playhead !== undefined) {
// Check if the next segment has playable media now.
// Otherwise give the current segment more time to catch up.
if ((next.dts / next.timescale) > playhead) {
return
}
}
current.skipTo(next.dts || 0) // tell typescript that it's not undefined; we already checked
current.finish()
// TODO cancel the QUIC stream to save bandwidth
this.segments.shift()
}
}
}

1
player/src/transport/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
fingerprint.hex

View File

@ -0,0 +1,175 @@
import * as Message from "./message"
import * as Stream from "../stream"
import * as MP4 from "../mp4"
import Audio from "../audio"
import Video from "../video"
// @ts-ignore bundler embeds data
import fingerprint from 'bundle-text:./fingerprint.hex';
export interface TransportInit {
url: string;
audio: Audio;
video: Video;
}
export default class Transport {
quic: Promise<WebTransport>;
api: Promise<WritableStream>;
tracks: Map<string, MP4.InitParser>
audio: Audio;
video: Video;
constructor(props: TransportInit) {
this.tracks = new Map();
this.audio = props.audio;
this.video = props.video;
this.quic = this.connect(props.url)
// Create a unidirectional stream for all of our messages
this.api = this.quic.then((q) => {
return q.createUnidirectionalStream()
})
// async functions
this.receiveStreams()
}
async close() {
(await this.quic).close()
}
async connect(url: string): Promise<WebTransport> {
// Convert the hex to binary.
let hash = [];
for (let c = 0; c < fingerprint.length-1; c += 2) {
hash.push(parseInt(fingerprint.substring(c, c+2), 16));
}
const quic = new WebTransport(url, {
"serverCertificateHashes": [{
"algorithm": "sha-256",
"value": new Uint8Array(hash),
}]
})
await quic.ready
return quic
}
async sendMessage(msg: any) {
const payload = JSON.stringify(msg)
const size = payload.length + 8
const stream = await this.api
const writer = new Stream.Writer(stream)
await writer.uint32(size)
await writer.string("warp")
await writer.string(payload)
writer.release()
}
async receiveStreams() {
const q = await this.quic
const streams = q.incomingUnidirectionalStreams.getReader()
while (true) {
const result = await streams.read()
if (result.done) break
const stream = result.value
this.handleStream(stream) // don't await
}
}
async handleStream(stream: ReadableStream) {
let r = new Stream.Reader(stream)
while (!await r.done()) {
const size = await r.uint32();
const typ = new TextDecoder('utf-8').decode(await r.bytes(4));
if (typ != "warp") throw "expected warp atom"
if (size < 8) throw "atom too small"
const payload = new TextDecoder('utf-8').decode(await r.bytes(size - 8));
const msg = JSON.parse(payload)
if (msg.init) {
return this.handleInit(r, msg.init as Message.Init)
} else if (msg.segment) {
return this.handleSegment(r, msg.segment as Message.Segment)
}
}
}
async handleInit(stream: Stream.Reader, msg: Message.Init) {
let track = this.tracks.get(msg.id);
if (!track) {
track = new MP4.InitParser()
this.tracks.set(msg.id, track)
}
while (1) {
const data = await stream.read()
if (!data) break
track.push(data)
}
const info = await track.info
if (info.audioTracks.length + info.videoTracks.length != 1) {
throw new Error("expected a single track")
}
if (info.audioTracks.length) {
this.audio.init({
track: msg.id,
info: info,
raw: track.raw,
})
} else if (info.videoTracks.length) {
this.video.init({
track: msg.id,
info: info,
raw: track.raw,
})
} else {
throw new Error("init is neither audio nor video")
}
}
async handleSegment(stream: Stream.Reader, msg: Message.Segment) {
let track = this.tracks.get(msg.init);
if (!track) {
track = new MP4.InitParser()
this.tracks.set(msg.init, track)
}
// Wait until we learn if this is an audio or video track
const info = await track.info
if (info.audioTracks.length) {
this.audio.segment({
track: msg.init,
buffer: stream.buffer,
reader: stream.reader,
})
} else if (info.videoTracks.length) {
this.video.segment({
track: msg.init,
buffer: stream.buffer,
reader: stream.reader,
})
} else {
throw new Error("segment is neither audio nor video")
}
}
}

View File

@ -1,13 +1,8 @@
export interface Message {
init?: MessageInit
segment?: MessageSegment
}
export interface MessageInit {
export interface Init {
id: string
}
export interface MessageSegment {
export interface Segment {
init: string // id of the init segment
timestamp: number // presentation timestamp in milliseconds of the first sample
// TODO track would be nice

View File

@ -1,4 +0,0 @@
export interface TimeRange {
start: number;
end: number;
}

View File

@ -0,0 +1,16 @@
export default class Deferred<T> {
promise: Promise<T>
resolve: (value: T | PromiseLike<T>) => void
reject: (value: T | PromiseLike<T>) => void
constructor() {
// Set initial values so TS stops being annoying.
this.resolve = (value: T | PromiseLike<T>) => {};
this.reject = (value: T | PromiseLike<T>) => {};
this.promise = new Promise((resolve, reject) => {
this.resolve = resolve
this.reject = reject
})
}
}

1
player/src/util/index.ts Normal file
View File

@ -0,0 +1 @@
export { default as Deferred } from "./deferred"

127
player/src/video/decoder.ts Normal file
View File

@ -0,0 +1,127 @@
import * as Message from "./message";
import * as MP4 from "../mp4"
import * as Stream from "../stream"
import * as Util from "../util"
import Renderer from "./renderer"
export default class Decoder {
// Store the init message for each track
tracks: Map<string, Util.Deferred<Message.Init>>
renderer: Renderer;
constructor(renderer: Renderer) {
this.tracks = new Map();
this.renderer = renderer;
}
async init(msg: Message.Init) {
let track = this.tracks.get(msg.track);
if (!track) {
track = new Util.Deferred()
this.tracks.set(msg.track, track)
}
if (msg.info.videoTracks.length != 1 || msg.info.audioTracks.length != 0) {
throw new Error("Expected a single video track")
}
track.resolve(msg)
}
async decode(msg: Message.Segment) {
let track = this.tracks.get(msg.track);
if (!track) {
track = new Util.Deferred()
this.tracks.set(msg.track, track)
}
// Wait for the init segment to be fully received and parsed
const init = await track.promise;
const info = init.info;
const video = info.videoTracks[0]
const decoder = new VideoDecoder({
output: (frame: VideoFrame) => {
this.renderer.emit(frame)
},
error: (err: Error) => {
console.warn(err)
}
});
const input = MP4.New();
input.onSamples = (id: number, user: any, samples: MP4.Sample[]) => {
for (let sample of samples) {
const timestamp = 1000 * sample.dts / sample.timescale // milliseconds
if (sample.is_sync) {
// Configure the decoder using the AVC box for H.264
const avcc = sample.description.avcC;
const description = new MP4.Stream(new Uint8Array(avcc.size), 0, false)
avcc.write(description)
decoder.configure({
codec: video.codec,
codedHeight: video.track_height,
codedWidth: video.track_width,
description: description.buffer?.slice(8),
// optimizeForLatency: true
})
}
decoder.decode(new EncodedVideoChunk({
data: sample.data,
duration: sample.duration,
timestamp: timestamp,
type: sample.is_sync ? "key" : "delta",
}))
}
}
input.onReady = (info: any) => {
input.setExtractionOptions(info.tracks[0].id, {}, { nbSamples: 1 });
input.start();
}
// MP4box requires us to reparse the init segment unfortunately
let offset = 0;
for (let raw of init.raw) {
raw.fileStart = offset
input.appendBuffer(raw)
}
const stream = new Stream.Reader(msg.reader, msg.buffer)
/* TODO I'm not actually sure why this code doesn't work; something trips up the MP4 parser
while (1) {
const data = await stream.read()
if (!data) break
input.appendBuffer(data)
input.flush()
}
*/
// One day I'll figure it out; until then read one top-level atom at a time
while (!await stream.done()) {
const raw = await stream.peek(4)
const size = new DataView(raw.buffer, raw.byteOffset, raw.byteLength).getUint32(0)
const atom = await stream.bytes(size)
// Make a copy of the atom because mp4box only accepts an ArrayBuffer unfortunately
let box = new Uint8Array(atom.byteLength);
box.set(atom)
// and for some reason we need to modify the underlying ArrayBuffer with offset
let buffer = box.buffer as MP4.ArrayBuffer
buffer.fileStart = offset
// Parse the data
offset = input.appendBuffer(buffer)
input.flush()
}
}
}

27
player/src/video/index.ts Normal file
View File

@ -0,0 +1,27 @@
import * as Message from "./message"
// Wrapper around the WebWorker API
export default class Video {
worker: Worker;
constructor(config: Message.Config) {
const url = new URL('worker.ts', import.meta.url)
this.worker = new Worker(url, {
type: "module",
name: "video",
})
this.worker.postMessage({ config }, [ config.canvas ])
}
init(init: Message.Init) {
this.worker.postMessage({ init }) // note: we copy the raw init bytes each time
}
segment(segment: Message.Segment) {
this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ])
}
play() {
// TODO
}
}

View File

@ -0,0 +1,17 @@
import * as MP4 from "../mp4"
export interface Config {
canvas: OffscreenCanvas;
}
export interface Init {
track: string;
info: MP4.Info;
raw: MP4.ArrayBuffer[];
}
export interface Segment {
track: string;
buffer: Uint8Array; // unread buffered data
reader: ReadableStream; // unread unbuffered data
}

View File

@ -0,0 +1,91 @@
import * as Message from "./message";
export default class Renderer {
canvas: OffscreenCanvas;
queue: Array<VideoFrame>;
render: number; // non-zero if requestAnimationFrame has been called
sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0
last?: number; // the timestamp of the last rendered frame
constructor(config: Message.Config) {
this.canvas = config.canvas;
this.queue = [];
this.render = 0;
}
emit(frame: VideoFrame) {
if (!this.sync) {
// Save the frame as the sync point
this.sync = performance.now() - frame.timestamp
}
// Drop any old frames
if (this.last && frame.timestamp <= this.last) {
frame.close()
return
}
// Insert the frame into the queue sorted by timestamp.
if (this.queue.length > 0 && this.queue[this.queue.length-1].timestamp <= frame.timestamp) {
// Fast path because we normally append to the end.
this.queue.push(frame)
} else {
// Do a full binary search
let low = 0
let high = this.queue.length;
while (low < high) {
var mid = (low + high) >>> 1;
if (this.queue[mid].timestamp < frame.timestamp) low = mid + 1;
else high = mid;
}
this.queue.splice(low, 0, frame)
}
// Queue up to render the next frame.
if (!this.render) {
this.render = self.requestAnimationFrame(this.draw.bind(this))
}
}
draw(now: DOMHighResTimeStamp) {
// Determine the target timestamp.
const target = now - this.sync!
let frame = this.queue[0]
if (frame.timestamp >= target) {
// nothing to render yet, wait for the next animation frame
this.render = self.requestAnimationFrame(this.draw.bind(this))
return
}
this.queue.shift()
// Check if we should skip some frames
while (this.queue.length) {
const next = this.queue[0]
if (next.timestamp > target) {
break
}
frame.close()
this.queue.shift()
frame = next
}
const ctx = this.canvas.getContext("2d");
ctx!.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height) // TODO aspect ratio
this.last = frame.timestamp;
frame.close()
if (this.queue.length > 0) {
this.render = self.requestAnimationFrame(this.draw.bind(this))
} else {
// Break the loop for now
this.render = 0
}
}
}

View File

@ -0,0 +1,22 @@
import Renderer from "./renderer"
import Decoder from "./decoder"
import * as Message from "./message"
let decoder: Decoder;
let renderer: Renderer;
self.addEventListener('message', async (e: MessageEvent) => {
if (e.data.config) {
const config = e.data.config as Message.Config
renderer = new Renderer(config)
decoder = new Decoder(renderer)
} else if (e.data.init) {
const init = e.data.init as Message.Init
await decoder.init(init)
} else if (e.data.segment) {
const segment = e.data.segment as Message.Segment
await decoder.decode(segment)
}
})

View File

@ -3,11 +3,9 @@
"src/**/*"
],
"compilerOptions": {
"target": "es2021",
"target": "es2022",
"module": "es2022",
"moduleResolution": "node",
"strict": true,
"typeRoots": [
"src/types"
],
"allowJs": true
}
}

File diff suppressed because it is too large Load Diff

View File

@ -20,7 +20,7 @@ require (
github.com/marten-seemann/qtls-go1-18 v0.1.3 // indirect
github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect
github.com/onsi/ginkgo/v2 v2.2.0 // indirect
golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 // indirect
golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect

View File

@ -77,7 +77,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lucas-clemente/quic-go v0.31.0 h1:MfNp3fk0wjWRajw6quMFA3ap1AVtlU+2mtwmbVogB2M=
github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/marten-seemann/qpack v0.3.0 h1:UiWstOgT8+znlkDPOg2+3rIuYXJ2CnGDkGUXN6ki6hE=
@ -153,8 +152,8 @@ golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 h1:/pEO3GD/ABYAjuakUS6xSEmmlyVS4kxBNkeA9tLJiTI=
golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 h1:tkVvjkPTB7pnW3jnid7kNyAMPVWllTNOf/qKDze4p9o=
golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA=
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA=

View File

@ -176,6 +176,11 @@ func (s *Session) writeInit(ctx context.Context, init *MediaInit) (err error) {
return fmt.Errorf("failed to create stream: %w", err)
}
if temp == nil {
// Not sure when this happens, perhaps when closing a connection?
return fmt.Errorf("received a nil stream from quic-go")
}
// Wrap the stream in an object that buffers writes instead of blocking.
stream := NewStream(temp)
s.streams.Add(stream.Run)
@ -200,6 +205,11 @@ func (s *Session) writeInit(ctx context.Context, init *MediaInit) (err error) {
return fmt.Errorf("failed to write init data: %w", err)
}
err = stream.Close()
if err != nil {
return fmt.Errorf("failed to close init stream: %w", err)
}
return nil
}
@ -210,6 +220,11 @@ func (s *Session) writeSegment(ctx context.Context, segment *MediaSegment) (err
return fmt.Errorf("failed to create stream: %w", err)
}
if temp == nil {
// Not sure when this happens, perhaps when closing a connection?
return fmt.Errorf("received a nil stream from quic-go")
}
// Wrap the stream in an object that buffers writes instead of blocking.
stream := NewStream(temp)
s.streams.Add(stream.Run)