diff --git a/HACKATHON.md b/HACKATHON.md new file mode 100644 index 0000000..36274ff --- /dev/null +++ b/HACKATHON.md @@ -0,0 +1,54 @@ +# Hackathon + +IETF Prague 118 + +## MoqTransport + +Reference libraries are available at [moq-rs](https://github.com/kixelated/moq-rs) and [moq-js](https://github.com/kixelated/moq-js). The Rust library is [well documented](https://docs.rs/moq-transport/latest/moq_transport/) but the web library, not so much. + +**TODO** Update both to draft-01. +**TODO** Switch any remaining forks over to extensions. ex: track_id in SUBSCRIBE + +The stream mapping right now is quite rigid: `stream == group == object`. + +**TODO** Support multiple objects per group. They MUST NOT use different priorities, different tracks, or out-of-order sequences. + +The API and cache aren't designed to send/receive arbitrary objects over arbitrary streams as specified in the draft. I don't think it should, and it wouldn't be possible to implement in time for the hackathon anyway. + +**TODO** Make an extension to require this stream mapping? + +## Generic Relay + +I'm hosting a simple CDN at: `relay.quic.video` + +The traffic is sharded based on the WebTransport path to avoid namespace collisions. Think of it like a customer ID, although it's completely unauthenticated for now. Use your username or whatever string you want: `CONNECT https://relay.quic.video/alan`. + +**TODO** Currently, it performs an implicit `ANNOUNCE ""` when `role=publisher`. This means there can only be a single publisher per shard and `role=both` is not supported. I should have explicit `ANNOUNCE` messages supported before the hackathon to remove this limitation. + +**TODO** I don't know if I will have subscribe hints fully working in time. They will be parsed but might be ignored. + +## CMAF Media + +You can [publish](https://quic.video/publish) and [watch](https://quic.video/watch) broadcasts. +You can also use `moq-pub` to publish your own media if you would like. + +I'm currently using a JSON catalog similar to the proposed [Warp catalog](https://datatracker.ietf.org/doc/draft-wilaw-moq-catalogformat/). + +**TODO** update to the proposed format. + +If you want to fetch from the relay directly, the name of the broadcast is path. For example, `https://quic.video/watch/bbb` can be accessed at `relay.quic.video/bbb`. The namespace is empty and the catalog track is `.catalog`. + +Each of the tracks uses a single object per groups. Video groups are per GoP, while audio groups are per frame. There's also an init track containing information required to initialize the decoder. + +**TODO** Base64 encode the init track in the catalog. + +**TODO** Add a flag for publishing LoC media? It shouldn't be difficult. + +## Clock + +**TODO** Host a clock demo that sends a group per second: + +``` +GROUP: YYYY-MM-DD HH:MM +OBJECT: SS +``` diff --git a/dev/README.md b/dev/README.md index 88e0a06..1c0d8fb 100644 --- a/dev/README.md +++ b/dev/README.md @@ -83,9 +83,16 @@ The following command runs a development instance, broadcasing `dev/source.mp4` ``` It will print out a URL when you can use to watch. -This will contain a random broadcast name so the below link won't work: +By default, the broadcast name is `dev` but you can overwrite it with the `NAME` env. -> Watch URL: https://quic.video/watch/REPLACE_WITH_NAME?server=localhost:4443 +> Watch URL: https://quic.video/watch/dev?server=localhost:4443 + +If you're debugging encoding issues, you can use this script to dump the file to disk instead, defaulting to +`dev/output.mp4`. + +```bash +./dev/pub-file +``` ### moq-api diff --git a/dev/pub b/dev/pub index c3bf8e0..2a5cb21 100755 --- a/dev/pub +++ b/dev/pub @@ -13,21 +13,28 @@ PORT="${PORT:-4443}" ADDR="${ADDR:-$HOST:$PORT}" # Generate a random 16 character name by default. -NAME="${NAME:-$(head /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9' | head -c 16)}" +#NAME="${NAME:-$(head /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9' | head -c 16)}" + +# JK use the name "dev" instead +# TODO use that random name if the host is not localhost +NAME="${NAME:-dev}" # Combine the host and name into a URL. URL="${URL:-"https://$ADDR/$NAME"}" # Default to a source video -MEDIA="${MEDIA:-dev/source.mp4}" +INPUT="${INPUT:-dev/source.mp4}" # Print out the watch URL echo "Watch URL: https://quic.video/watch/$NAME?server=$ADDR" # Run ffmpeg and pipe the output to moq-pub +# TODO enable audio again once fixed. ffmpeg -hide_banner -v quiet \ -stream_loop -1 -re \ - -i "$MEDIA" \ + -i "$INPUT" \ + -c copy \ -an \ - -f mp4 -movflags empty_moov+frag_every_frame+separate_moof+omit_tfhd_offset - \ - | cargo run --bin moq-pub -- "$URL" "$@" + -f mp4 -movflags cmaf+separate_moof+delay_moov+skip_trailer \ + -frag_duration 1 \ + - | cargo run --bin moq-pub -- "$URL" "$@" diff --git a/dev/pub-file b/dev/pub-file new file mode 100755 index 0000000..cef42ca --- /dev/null +++ b/dev/pub-file @@ -0,0 +1,90 @@ +#!/bin/bash +set -euo pipefail + +# Change directory to the root of the project +cd "$(dirname "$0")/.." + +# Default to a source video +INPUT="${INPUT:-dev/source.mp4}" + +# Output the fragmented MP4 to disk for testing. +OUTPUT="${OUTPUT:-dev/output.mp4}" + +# Run ffmpeg the same as dev/pub, but: +# - print any errors/warnings +# - only loop twice +# +# Note this is artificially slowed down to real-time using the -re flag; you can remove it. +ffmpeg \ + -re \ + -y \ + -i "$INPUT" \ + -c copy \ + -fps_mode passthrough \ + -f mp4 -movflags cmaf+separate_moof+delay_moov+skip_trailer \ + -frag_duration 1 \ + "${OUTPUT}" + +# % ffmpeg -f mp4 --ffmpeg -h muxer=mov +# +# ffmpeg version 6.0 Copyright (c) 2000-2023 the FFmpeg developers +# Muxer mov [QuickTime / MOV]: +# Common extensions: mov. +# Default video codec: h264. +# Default audio codec: aac. +# mov/mp4/tgp/psp/tg2/ipod/ismv/f4v muxer AVOptions: +# -movflags E.......... MOV muxer flags (default 0) +# rtphint E.......... Add RTP hint tracks +# empty_moov E.......... Make the initial moov atom empty +# frag_keyframe E.......... Fragment at video keyframes +# frag_every_frame E.......... Fragment at every frame +# separate_moof E.......... Write separate moof/mdat atoms for each track +# frag_custom E.......... Flush fragments on caller requests +# isml E.......... Create a live smooth streaming feed (for pushing to a publishing point) +# faststart E.......... Run a second pass to put the index (moov atom) at the beginning of the file +# omit_tfhd_offset E.......... Omit the base data offset in tfhd atoms +# disable_chpl E.......... Disable Nero chapter atom +# default_base_moof E.......... Set the default-base-is-moof flag in tfhd atoms +# dash E.......... Write DASH compatible fragmented MP4 +# cmaf E.......... Write CMAF compatible fragmented MP4 +# frag_discont E.......... Signal that the next fragment is discontinuous from earlier ones +# delay_moov E.......... Delay writing the initial moov until the first fragment is cut, or until the first fragment flush +# global_sidx E.......... Write a global sidx index at the start of the file +# skip_sidx E.......... Skip writing of sidx atom +# write_colr E.......... Write colr atom even if the color info is unspecified (Experimental, may be renamed or changed, do not use from scripts) +# prefer_icc E.......... If writing colr atom prioritise usage of ICC profile if it exists in stream packet side data +# write_gama E.......... Write deprecated gama atom +# use_metadata_tags E.......... Use mdta atom for metadata. +# skip_trailer E.......... Skip writing the mfra/tfra/mfro trailer for fragmented files +# negative_cts_offsets E.......... Use negative CTS offsets (reducing the need for edit lists) +# -moov_size E.......... maximum moov size so it can be placed at the begin (from 0 to INT_MAX) (default 0) +# -rtpflags E.......... RTP muxer flags (default 0) +# latm E.......... Use MP4A-LATM packetization instead of MPEG4-GENERIC for AAC +# rfc2190 E.......... Use RFC 2190 packetization instead of RFC 4629 for H.263 +# skip_rtcp E.......... Don't send RTCP sender reports +# h264_mode0 E.......... Use mode 0 for H.264 in RTP +# send_bye E.......... Send RTCP BYE packets when finishing +# -skip_iods E.......... Skip writing iods atom. (default true) +# -iods_audio_profile E.......... iods audio profile atom. (from -1 to 255) (default -1) +# -iods_video_profile E.......... iods video profile atom. (from -1 to 255) (default -1) +# -frag_duration E.......... Maximum fragment duration (from 0 to INT_MAX) (default 0) +# -min_frag_duration E.......... Minimum fragment duration (from 0 to INT_MAX) (default 0) +# -frag_size E.......... Maximum fragment size (from 0 to INT_MAX) (default 0) +# -ism_lookahead E.......... Number of lookahead entries for ISM files (from 0 to 255) (default 0) +# -video_track_timescale E.......... set timescale of all video tracks (from 0 to INT_MAX) (default 0) +# -brand E.......... Override major brand +# -use_editlist E.......... use edit list (default auto) +# -fragment_index E.......... Fragment number of the next fragment (from 1 to INT_MAX) (default 1) +# -mov_gamma E.......... gamma value for gama atom (from 0 to 10) (default 0) +# -frag_interleave E.......... Interleave samples within fragments (max number of consecutive samples, lower is tighter interleaving, but with more overhead) (from 0 to INT_MAX) (default 0) +# -encryption_scheme E.......... Configures the encryption scheme, allowed values are none, cenc-aes-ctr +# -encryption_key E.......... The media encryption key (hex) +# -encryption_kid E.......... The media encryption key identifier (hex) +# -use_stream_ids_as_track_ids E.......... use stream ids as track ids (default false) +# -write_btrt E.......... force or disable writing btrt (default auto) +# -write_tmcd E.......... force or disable writing tmcd (default auto) +# -write_prft E.......... Write producer reference time box with specified time source (from 0 to 2) (default 0) +# wallclock 1 E.......... +# pts 2 E.......... +# -empty_hdlr_name E.......... write zero-length name string in hdlr atoms within mdia and minf atoms (default false) +# -movie_timescale E.......... set movie timescale (from 1 to INT_MAX) (default 1000) diff --git a/moq-pub/README.md b/moq-pub/README.md index 7cf92a2..fe7327a 100644 --- a/moq-pub/README.md +++ b/moq-pub/README.md @@ -5,7 +5,7 @@ A command line tool for publishing media via Media over QUIC (MoQ). Expects to receive fragmented MP4 via standard input and connect to a MOQT relay. ``` -ffmpeg ... - | moq-pub -i - --host localhost:4443 +ffmpeg ... - | moq-pub https://localhost:4443 ``` ### Invoking `moq-pub`: @@ -13,7 +13,7 @@ ffmpeg ... - | moq-pub -i - --host localhost:4443 Here's how I'm currently testing things, with a local copy of Big Buck Bunny named `bbb_source.mp4`: ``` -$ ffmpeg -hide_banner -v quiet -stream_loop -1 -re -i bbb_source.mp4 -an -f mp4 -movflags empty_moov+frag_every_frame+separate_moof+omit_tfhd_offset - | RUST_LOG=moq_pub=info moq-pub -i - +$ ffmpeg -hide_banner -v quiet -stream_loop -1 -re -i bbb_source.mp4 -an -f mp4 -movflags empty_moov+frag_every_frame+separate_moof+omit_tfhd_offset - | RUST_LOG=moq_pub=info moq-pub https://localhost:4443 ``` This relies on having `moq-relay` (the relay server) already running locally in another shell. diff --git a/moq-pub/src/media.rs b/moq-pub/src/media.rs index d3babd6..cb2c5a7 100644 --- a/moq-pub/src/media.rs +++ b/moq-pub/src/media.rs @@ -4,6 +4,7 @@ use moq_transport::cache::{broadcast, segment, track}; use moq_transport::VarInt; use mp4::{self, ReadBox}; use serde_json::json; +use std::cmp::max; use std::collections::HashMap; use std::io::Cursor; use std::time; @@ -15,11 +16,12 @@ pub struct Media { _catalog: track::Publisher, _init: track::Publisher, - tracks: HashMap, + // Tracks based on their track ID. + tracks: HashMap, } impl Media { - pub async fn new(config: &Config, mut broadcast: broadcast::Publisher) -> anyhow::Result { + pub async fn new(_config: &Config, mut broadcast: broadcast::Publisher) -> anyhow::Result { let mut stdin = tokio::io::stdin(); let ftyp = read_atom(&mut stdin).await?; anyhow::ensure!(&ftyp[4..8] == b"ftyp", "expected ftyp atom"); @@ -39,7 +41,7 @@ impl Media { let moov = mp4::MoovBox::read_box(&mut moov_reader, moov_header.size)?; // Create the catalog track with a single segment. - let mut init_track = broadcast.create_track("1.mp4")?; + let mut init_track = broadcast.create_track("0.mp4")?; let mut init_segment = init_track.create_segment(segment::Info { sequence: VarInt::ZERO, priority: i32::MAX, @@ -52,20 +54,20 @@ impl Media { for trak in &moov.traks { let id = trak.tkhd.track_id; - let name = id.to_string(); + let name = format!("{}.m4s", id); let timescale = track_timescale(&moov, id); // Store the track publisher in a map so we can update it later. let track = broadcast.create_track(&name)?; let track = Track::new(track, timescale); - tracks.insert(name, track); + tracks.insert(id, track); } let mut catalog = broadcast.create_track(".catalog")?; // Create the catalog track - Self::serve_catalog(&mut catalog, config, init_track.name.to_string(), &moov, &tracks)?; + Self::serve_catalog(&mut catalog, &init_track.name, &moov)?; Ok(Media { _broadcast: broadcast, @@ -78,7 +80,7 @@ impl Media { pub async fn run(&mut self) -> anyhow::Result<()> { let mut stdin = tokio::io::stdin(); // The current track name - let mut track_name = None; + let mut current = None; loop { let atom = read_atom(&mut stdin).await?; @@ -92,22 +94,21 @@ impl Media { // Process the moof. let fragment = Fragment::new(moof)?; - let name = fragment.track.to_string(); // Get the track for this moof. - let track = self.tracks.get_mut(&name).context("failed to find track")?; + let track = self.tracks.get_mut(&fragment.track).context("failed to find track")?; // Save the track ID for the next iteration, which must be a mdat. - anyhow::ensure!(track_name.is_none(), "multiple moof atoms"); - track_name.replace(name); + anyhow::ensure!(current.is_none(), "multiple moof atoms"); + current.replace(fragment.track); // Publish the moof header, creating a new segment if it's a keyframe. track.header(atom, fragment).context("failed to publish moof")?; } mp4::BoxType::MdatBox => { // Get the track ID from the previous moof. - let name = track_name.take().context("missing moof")?; - let track = self.tracks.get_mut(&name).context("failed to find track")?; + let track = current.take().context("missing moof")?; + let track = self.tracks.get_mut(&track).context("failed to find track")?; // Publish the mdat atom. track.data(atom).context("failed to publish mdat")?; @@ -122,10 +123,8 @@ impl Media { fn serve_catalog( track: &mut track::Publisher, - config: &Config, - init_track_name: String, + init_track_name: &str, moov: &mp4::MoovBox, - _tracks: &HashMap, ) -> Result<(), anyhow::Error> { let mut segment = track.create_segment(segment::Info { sequence: VarInt::ZERO, @@ -133,47 +132,82 @@ impl Media { expires: None, })?; - // avc1[.PPCCLL] - // - // let profile = 0x64; - // let constraints = 0x00; - // let level = 0x1f; + let mut tracks = Vec::new(); - // TODO: do build multi-track catalog by looping through moov.traks - let trak = moov.traks[0].clone(); - let avc1 = trak - .mdia - .minf - .stbl - .stsd - .avc1 - .ok_or(anyhow::anyhow!("avc1 atom not found"))?; + for trak in &moov.traks { + let mut track = json!({ + "container": "mp4", + "init_track": init_track_name, + "data_track": format!("{}.m4s", trak.tkhd.track_id), + }); - let profile = avc1.avcc.avc_profile_indication; - let constraints = avc1.avcc.profile_compatibility; // Not 100% certain here, but it's 0x00 on my current test video - let level = avc1.avcc.avc_level_indication; + let stsd = &trak.mdia.minf.stbl.stsd; + if let Some(avc1) = &stsd.avc1 { + // avc1[.PPCCLL] + // + // let profile = 0x64; + // let constraints = 0x00; + // let level = 0x1f; + let profile = avc1.avcc.avc_profile_indication; + let constraints = avc1.avcc.profile_compatibility; // Not 100% certain here, but it's 0x00 on my current test video + let level = avc1.avcc.avc_level_indication; - let width = avc1.width; - let height = avc1.height; + let width = avc1.width; + let height = avc1.height; - let codec = rfc6381_codec::Codec::avc1(profile, constraints, level); - let codec_str = codec.to_string(); + let codec = rfc6381_codec::Codec::avc1(profile, constraints, level); + let codec_str = codec.to_string(); + + track["kind"] = json!("video"); + track["codec"] = json!(codec_str); + track["width"] = json!(width); + track["height"] = json!(height); + } else if let Some(_hev1) = &stsd.hev1 { + // TODO https://github.com/gpac/mp4box.js/blob/325741b592d910297bf609bc7c400fc76101077b/src/box-codecs.js#L106 + anyhow::bail!("HEVC not yet supported") + } else if let Some(mp4a) = &stsd.mp4a { + let desc = &mp4a + .esds + .as_ref() + .context("missing esds box for MP4a")? + .es_desc + .dec_config; + let codec_str = format!("mp4a.{:02x}.{}", desc.object_type_indication, desc.dec_specific.profile); + + track["kind"] = json!("audio"); + track["codec"] = json!(codec_str); + track["channel_count"] = json!(mp4a.channelcount); + track["sample_rate"] = json!(mp4a.samplerate.value()); + track["sample_size"] = json!(mp4a.samplesize); + + let bitrate = max(desc.max_bitrate, desc.avg_bitrate); + if bitrate > 0 { + track["bit_rate"] = json!(bitrate); + } + } else if let Some(vp09) = &stsd.vp09 { + // https://github.com/gpac/mp4box.js/blob/325741b592d910297bf609bc7c400fc76101077b/src/box-codecs.js#L238 + let vpcc = &vp09.vpcc; + let codec_str = format!("vp09.0.{:02x}.{:02x}.{:02x}", vpcc.profile, vpcc.level, vpcc.bit_depth); + + track["kind"] = json!("video"); + track["codec"] = json!(codec_str); + track["width"] = json!(vp09.width); // no idea if this needs to be multiplied + track["height"] = json!(vp09.height); // no idea if this needs to be multiplied + + // TODO Test if this actually works; I'm just guessing based on mp4box.js + anyhow::bail!("VP9 not yet supported") + } else { + // TODO add av01 support: https://github.com/gpac/mp4box.js/blob/325741b592d910297bf609bc7c400fc76101077b/src/box-codecs.js#L251 + anyhow::bail!("unknown codec for track: {}", trak.tkhd.track_id); + } + + tracks.push(track); + } let catalog = json!({ - "tracks": [ - { - "container": "mp4", - "kind": "video", - "init_track": init_track_name, - "data_track": "1", // assume just one track for now - "codec": codec_str, - "width": width, - "height": height, - "frame_rate": config.fps, - "bit_rate": config.bitrate, - } - ] + "tracks": tracks }); + let catalog_str = serde_json::to_string_pretty(&catalog)?; log::info!("catalog: {}", catalog_str);