From 2601c40b54ec0e40c7651a4e870e1f3f65854fbe Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 13 Apr 2023 10:20:17 -0700 Subject: [PATCH 01/23] Replace Go with Rust. No code written yet. --- server/.gitignore | 2 +- server/Cargo.lock | 279 +++++++++++++++++++++++ server/Cargo.toml | 9 + server/go.mod | 30 --- server/go.sum | 264 ---------------------- server/internal/warp/media.go | 380 -------------------------------- server/internal/warp/message.go | 20 -- server/internal/warp/server.go | 132 ----------- server/internal/warp/session.go | 279 ----------------------- server/internal/warp/stream.go | 144 ------------ server/internal/web/web.go | 64 ------ server/main.go | 71 ------ server/src/main.rs | 3 + 13 files changed, 292 insertions(+), 1385 deletions(-) create mode 100644 server/Cargo.lock create mode 100644 server/Cargo.toml delete mode 100644 server/go.mod delete mode 100644 server/go.sum delete mode 100644 server/internal/warp/media.go delete mode 100644 server/internal/warp/message.go delete mode 100644 server/internal/warp/server.go delete mode 100644 server/internal/warp/session.go delete mode 100644 server/internal/warp/stream.go delete mode 100644 server/internal/web/web.go delete mode 100644 server/main.go create mode 100644 server/src/main.rs diff --git a/server/.gitignore b/server/.gitignore index 333c1e9..eb5a316 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -1 +1 @@ -logs/ +target diff --git a/server/Cargo.lock b/server/Cargo.lock new file mode 100644 index 0000000..6110308 --- /dev/null +++ b/server/Cargo.lock @@ -0,0 +1,279 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bumpalo" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + +[[package]] +name = "js-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.141" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" + +[[package]] +name = "libm" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "octets" +version = "0.2.0" +source = "git+https://github.com/n8o/quiche.git?branch=master#0137dc3ca6f4f31e3175d0a0868acb9c64b46cc7" + +[[package]] +name = "once_cell" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" + +[[package]] +name = "proc-macro2" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quiche" +version = "0.17.1" +source = "git+https://github.com/n8o/quiche.git?branch=master#0137dc3ca6f4f31e3175d0a0868acb9c64b46cc7" +dependencies = [ + "cmake", + "lazy_static", + "libc", + "libm", + "log", + "octets", + "ring", + "slab", + "smallvec", + "winapi", +] + +[[package]] +name = "quote" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "serde" +version = "1.0.160" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" + +[[package]] +name = "slab" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +dependencies = [ + "serde", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "warp-server" +version = "0.1.0" +dependencies = [ + "quiche", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" + +[[package]] +name = "web-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/server/Cargo.toml b/server/Cargo.toml new file mode 100644 index 0000000..d4dcbb7 --- /dev/null +++ b/server/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "warp-server" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +quiche = { git = "https://github.com/n8o/quiche.git", branch = "master" } # WebTransport fork diff --git a/server/go.mod b/server/go.mod deleted file mode 100644 index 7e5b4fb..0000000 --- a/server/go.mod +++ /dev/null @@ -1,30 +0,0 @@ -module github.com/kixelated/warp/server - -go 1.18 - -require ( - github.com/abema/go-mp4 v0.7.2 - github.com/kixelated/invoker v1.0.0 - github.com/kixelated/quic-go v1.31.0 - github.com/kixelated/webtransport-go v1.4.1 - github.com/zencoder/go-dash/v3 v3.0.2 -) - -require ( - github.com/francoispqt/gojay v1.2.13 // indirect - github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect - github.com/golang/mock v1.6.0 // indirect - github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect - github.com/google/uuid v1.1.2 // indirect - github.com/marten-seemann/qpack v0.3.0 // indirect - github.com/marten-seemann/qtls-go1-18 v0.1.3 // indirect - github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect - github.com/onsi/ginkgo/v2 v2.2.0 // indirect - golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect - golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect - golang.org/x/sys v0.1.1-0.20221102194838-fc697a31fa06 // indirect - golang.org/x/text v0.3.7 // indirect - golang.org/x/tools v0.1.12 // indirect -) diff --git a/server/go.sum b/server/go.sum deleted file mode 100644 index fed92be..0000000 --- a/server/go.sum +++ /dev/null @@ -1,264 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= -dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= -dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= -dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= -dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= -git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/abema/go-mp4 v0.7.2 h1:ugTC8gfEmjyaDKpXs3vi2QzgJbDu9B8m6UMMIpbYbGg= -github.com/abema/go-mp4 v0.7.2/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= -github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= -github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= -github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/kisielk/errcheck v1.4.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kixelated/invoker v1.0.0 h1:0wYlvK39yQPbkwIFy+YN41AhF89WOtGyWqV2pZB39xw= -github.com/kixelated/invoker v1.0.0/go.mod h1:RjG3iqm/sKwZjOpcW4SGq+l+4DJCDR/yUtc70VjCRB8= -github.com/kixelated/quic-go v1.31.0 h1:p2vq3Otvtmz+0EP23vjumnO/HU4Q/DFxNF6xNryVfmA= -github.com/kixelated/quic-go v1.31.0/go.mod h1:AO7pURnb8HXHmdalp5e09UxQfsuwseEhl0NLmwiSOFY= -github.com/kixelated/webtransport-go v1.4.1 h1:ZtY3P7hVe1wK5fAt71b+HHnNISFDcQ913v+bvaNATxA= -github.com/kixelated/webtransport-go v1.4.1/go.mod h1:6RV5pTXF7oP53T83bosSDsLdSdw31j5cfpMDqsO4D5k= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/marten-seemann/qpack v0.3.0 h1:UiWstOgT8+znlkDPOg2+3rIuYXJ2CnGDkGUXN6ki6hE= -github.com/marten-seemann/qpack v0.3.0/go.mod h1:cGfKPBiP4a9EQdxCwEwI/GEeWAsjSekBvx/X8mh58+g= -github.com/marten-seemann/qtls-go1-18 v0.1.3 h1:R4H2Ks8P6pAtUagjFty2p7BVHn3XiwDAl7TTQf5h7TI= -github.com/marten-seemann/qtls-go1-18 v0.1.3/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= -github.com/marten-seemann/qtls-go1-19 v0.1.1 h1:mnbxeq3oEyQxQXwI4ReCgW9DPoPR94sNlqWoDZnjRIE= -github.com/marten-seemann/qtls-go1-19 v0.1.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= -github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= -github.com/onsi/ginkgo/v2 v2.2.0 h1:3ZNA3L1c5FYDFTTxbFeVGGD8jYvjYauHD30YgLxVsNI= -github.com/onsi/ginkgo/v2 v2.2.0/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= -github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= -github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= -github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e h1:s2RNOM/IGdY0Y6qfTeUKhDawdHDpK9RGBdx80qN4Ttw= -github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e/go.mod h1:nBdnFKj15wFbf94Rwfq4m30eAcyY9V/IyKAGQFtqkW0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= -github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= -github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= -github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= -github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= -github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= -github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI= -github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU= -github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag= -github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg= -github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= -github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= -github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= -github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= -github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= -github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= -github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0= -github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= -github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= -github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= -github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= -github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= -github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/sunfish-shogi/bufseekio v0.0.0-20210207115823-a4185644b365/go.mod h1:dEzdXgvImkQ3WLI+0KQpmEx8T/C/ma9KeS3AfmU899I= -github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= -github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= -github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/zencoder/go-dash/v3 v3.0.2 h1:oP1+dOh+Gp57PkvdCyMfbHtrHaxfl3w4kR3KBBbuqQE= -github.com/zencoder/go-dash/v3 v3.0.2/go.mod h1:30R5bKy1aUYY45yesjtZ9l8trNc2TwNqbS17WVQmCzk= -go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= -go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= -golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= -golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 h1:tkVvjkPTB7pnW3jnid7kNyAMPVWllTNOf/qKDze4p9o= -golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= -golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.1-0.20221102194838-fc697a31fa06 h1:E1pm64FqQa4v8dHd/bAneyMkR4hk8LTJhoSlc5mc1cM= -golang.org/x/sys v0.1.1-0.20221102194838-fc697a31fa06/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= -google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= -gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzEYLhQB2YY= -sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= -sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/server/internal/warp/media.go b/server/internal/warp/media.go deleted file mode 100644 index 86d6f93..0000000 --- a/server/internal/warp/media.go +++ /dev/null @@ -1,380 +0,0 @@ -package warp - -import ( - "bytes" - "context" - "encoding/binary" - "errors" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "strings" - "time" - - "github.com/abema/go-mp4" - "github.com/kixelated/invoker" - "github.com/zencoder/go-dash/v3/mpd" -) - -// This is a demo; you should actually fetch media from a live backend. -// It's just much easier to read from disk and "fake" being live. -type Media struct { - base fs.FS - inits map[string]*MediaInit - video []*mpd.Representation - audio []*mpd.Representation -} - -func NewMedia(playlistPath string) (m *Media, err error) { - m = new(Media) - - // Create a fs.FS out of the folder holding the playlist - m.base = os.DirFS(filepath.Dir(playlistPath)) - - // Read the playlist file - playlist, err := mpd.ReadFromFile(playlistPath) - if err != nil { - return nil, fmt.Errorf("failed to open playlist: %w", err) - } - - if len(playlist.Periods) > 1 { - return nil, fmt.Errorf("multiple periods not supported") - } - - period := playlist.Periods[0] - - for _, adaption := range period.AdaptationSets { - representation := adaption.Representations[0] - - if representation.MimeType == nil { - return nil, fmt.Errorf("missing representation mime type") - } - - if representation.Bandwidth == nil { - return nil, fmt.Errorf("missing representation bandwidth") - } - - switch *representation.MimeType { - case "video/mp4": - m.video = append(m.video, representation) - case "audio/mp4": - m.audio = append(m.audio, representation) - } - } - - if len(m.video) == 0 { - return nil, fmt.Errorf("no video representation found") - } - - if len(m.audio) == 0 { - return nil, fmt.Errorf("no audio representation found") - } - - m.inits = make(map[string]*MediaInit) - - var reps []*mpd.Representation - reps = append(reps, m.audio...) - reps = append(reps, m.video...) - - for _, rep := range reps { - path := *rep.SegmentTemplate.Initialization - - // TODO Support the full template engine - path = strings.ReplaceAll(path, "$RepresentationID$", *rep.ID) - - f, err := fs.ReadFile(m.base, path) - if err != nil { - return nil, fmt.Errorf("failed to read init file: %w", err) - } - - init, err := newMediaInit(*rep.ID, f) - if err != nil { - return nil, fmt.Errorf("failed to create init segment: %w", err) - } - - m.inits[*rep.ID] = init - } - - return m, nil -} - -func (m *Media) Start(bitrate func() uint64) (inits map[string]*MediaInit, audio *MediaStream, video *MediaStream, err error) { - start := time.Now() - - audio, err = newMediaStream(m, m.audio, start, bitrate) - if err != nil { - return nil, nil, nil, err - } - - video, err = newMediaStream(m, m.video, start, bitrate) - if err != nil { - return nil, nil, nil, err - } - - return m.inits, audio, video, nil -} - -type MediaStream struct { - Media *Media - - start time.Time - reps []*mpd.Representation - sequence int - bitrate func() uint64 // returns the current estimated bitrate -} - -func newMediaStream(m *Media, reps []*mpd.Representation, start time.Time, bitrate func() uint64) (ms *MediaStream, err error) { - ms = new(MediaStream) - ms.Media = m - ms.reps = reps - ms.start = start - ms.bitrate = bitrate - return ms, nil -} - -func (ms *MediaStream) chooseRepresentation() (choice *mpd.Representation) { - bitrate := ms.bitrate() - - // Loop over the renditions and pick the highest bitrate we can support - for _, r := range ms.reps { - if uint64(*r.Bandwidth) <= bitrate && (choice == nil || *r.Bandwidth > *choice.Bandwidth) { - choice = r - } - } - - if choice != nil { - return choice - } - - // We can't support any of the bitrates, so find the lowest one. - for _, r := range ms.reps { - if choice == nil || *r.Bandwidth < *choice.Bandwidth { - choice = r - } - } - - return choice -} - -// Returns the next segment in the stream -func (ms *MediaStream) Next(ctx context.Context) (segment *MediaSegment, err error) { - rep := ms.chooseRepresentation() - - if rep.SegmentTemplate == nil { - return nil, fmt.Errorf("missing segment template") - } - - if rep.SegmentTemplate.Media == nil { - return nil, fmt.Errorf("no media template") - } - - if rep.SegmentTemplate.StartNumber == nil { - return nil, fmt.Errorf("missing start number") - } - - path := *rep.SegmentTemplate.Media - sequence := ms.sequence + int(*rep.SegmentTemplate.StartNumber) - - // TODO Support the full template engine - path = strings.ReplaceAll(path, "$RepresentationID$", *rep.ID) - path = strings.ReplaceAll(path, "$Number%05d$", fmt.Sprintf("%05d", sequence)) // TODO TODO - - // Try openning the file - f, err := ms.Media.base.Open(path) - if errors.Is(err, os.ErrNotExist) && ms.sequence != 0 { - // Return EOF if the next file is missing - return nil, nil - } else if err != nil { - return nil, fmt.Errorf("failed to open segment file: %w", err) - } - - duration := time.Duration(*rep.SegmentTemplate.Duration) / time.Nanosecond - timestamp := time.Duration(ms.sequence) * duration - - init := ms.Media.inits[*rep.ID] - - segment, err = newMediaSegment(ms, init, f, timestamp) - if err != nil { - return nil, fmt.Errorf("failed to create segment: %w", err) - } - - ms.sequence += 1 - - return segment, nil -} - -type MediaInit struct { - ID string - Raw []byte - Timescale int -} - -func newMediaInit(id string, raw []byte) (mi *MediaInit, err error) { - mi = new(MediaInit) - mi.ID = id - mi.Raw = raw - - err = mi.parse() - if err != nil { - return nil, fmt.Errorf("failed to parse init segment: %w", err) - } - - return mi, nil -} - -// Parse through the init segment, literally just to populate the timescale -func (mi *MediaInit) parse() (err error) { - r := bytes.NewReader(mi.Raw) - - _, err = mp4.ReadBoxStructure(r, func(h *mp4.ReadHandle) (interface{}, error) { - if !h.BoxInfo.IsSupportedType() { - return nil, nil - } - - payload, _, err := h.ReadPayload() - if err != nil { - return nil, err - } - - switch box := payload.(type) { - case *mp4.Mdhd: // Media Header; moov -> trak -> mdia > mdhd - if mi.Timescale != 0 { - // verify only one track - return nil, fmt.Errorf("multiple mdhd atoms") - } - - mi.Timescale = int(box.Timescale) - } - - // Expands children - return h.Expand() - }) - - if err != nil { - return fmt.Errorf("failed to parse MP4 file: %w", err) - } - - return nil -} - -type MediaSegment struct { - Stream *MediaStream - Init *MediaInit - - file fs.File - timestamp time.Duration -} - -func newMediaSegment(s *MediaStream, init *MediaInit, file fs.File, timestamp time.Duration) (ms *MediaSegment, err error) { - ms = new(MediaSegment) - ms.Stream = s - ms.Init = init - - ms.file = file - ms.timestamp = timestamp - - return ms, nil -} - -// Return the next atom, sleeping based on the PTS to simulate a live stream -func (ms *MediaSegment) Read(ctx context.Context) (chunk []byte, err error) { - // Read the next top-level box - var header [8]byte - - _, err = io.ReadFull(ms.file, header[:]) - if err != nil { - return nil, fmt.Errorf("failed to read header: %w", err) - } - - size := int(binary.BigEndian.Uint32(header[0:4])) - if size < 8 { - return nil, fmt.Errorf("box is too small") - } - - buf := make([]byte, size) - n := copy(buf, header[:]) - - _, err = io.ReadFull(ms.file, buf[n:]) - if err != nil { - return nil, fmt.Errorf("failed to read atom: %w", err) - } - - sample, err := ms.parseAtom(ctx, buf) - if err != nil { - return nil, fmt.Errorf("failed to parse atom: %w", err) - } - - if sample != nil { - // Simulate a live stream by sleeping before we write this sample. - // Figure out how much time has elapsed since the start - elapsed := time.Since(ms.Stream.start) - delay := sample.Timestamp - elapsed - - if delay > 0 { - // Sleep until we're supposed to see these samples - err = invoker.Sleep(delay)(ctx) - if err != nil { - return nil, err - } - } - } - - return buf, nil -} - -// Parse through the MP4 atom, returning infomation about the next fragmented sample -func (ms *MediaSegment) parseAtom(ctx context.Context, buf []byte) (sample *mediaSample, err error) { - r := bytes.NewReader(buf) - - _, err = mp4.ReadBoxStructure(r, func(h *mp4.ReadHandle) (interface{}, error) { - if !h.BoxInfo.IsSupportedType() { - return nil, nil - } - - payload, _, err := h.ReadPayload() - if err != nil { - return nil, err - } - - switch box := payload.(type) { - case *mp4.Moof: - sample = new(mediaSample) - case *mp4.Tfdt: // Track Fragment Decode Timestamp; moof -> traf -> tfdt - // TODO This box isn't required - // TODO we want the last PTS if there are multiple samples - var dts time.Duration - if box.FullBox.Version == 0 { - dts = time.Duration(box.BaseMediaDecodeTimeV0) - } else { - dts = time.Duration(box.BaseMediaDecodeTimeV1) - } - - if ms.Init.Timescale == 0 { - return nil, fmt.Errorf("missing timescale") - } - - // Convert to seconds - // TODO What about PTS? - sample.Timestamp = dts * time.Second / time.Duration(ms.Init.Timescale) - } - - // Expands children - return h.Expand() - }) - - if err != nil { - return nil, fmt.Errorf("failed to parse MP4 file: %w", err) - } - - return sample, nil -} - -func (ms *MediaSegment) Close() (err error) { - return ms.file.Close() -} - -type mediaSample struct { - Timestamp time.Duration // The timestamp of the first sample -} diff --git a/server/internal/warp/message.go b/server/internal/warp/message.go deleted file mode 100644 index 5514b78..0000000 --- a/server/internal/warp/message.go +++ /dev/null @@ -1,20 +0,0 @@ -package warp - -type Message struct { - Init *MessageInit `json:"init,omitempty"` - Segment *MessageSegment `json:"segment,omitempty"` - Debug *MessageDebug `json:"debug,omitempty"` -} - -type MessageInit struct { - Id string `json:"id"` // ID of the init segment -} - -type MessageSegment struct { - Init string `json:"init"` // ID of the init segment to use for this segment - Timestamp int `json:"timestamp"` // PTS of the first frame in milliseconds -} - -type MessageDebug struct { - MaxBitrate int `json:"max_bitrate"` // Artificially limit the QUIC max bitrate -} diff --git a/server/internal/warp/server.go b/server/internal/warp/server.go deleted file mode 100644 index 9ad6f18..0000000 --- a/server/internal/warp/server.go +++ /dev/null @@ -1,132 +0,0 @@ -package warp - -import ( - "context" - "crypto/tls" - "encoding/hex" - "fmt" - "io" - "log" - "net/http" - "os" - "path/filepath" - - "github.com/kixelated/invoker" - "github.com/kixelated/quic-go" - "github.com/kixelated/quic-go/http3" - "github.com/kixelated/quic-go/logging" - "github.com/kixelated/quic-go/qlog" - "github.com/kixelated/webtransport-go" -) - -type Server struct { - inner *webtransport.Server - media *Media - sessions invoker.Tasks - cert *tls.Certificate -} - -type Config struct { - Addr string - Cert *tls.Certificate - LogDir string - Media *Media -} - -func New(config Config) (s *Server, err error) { - s = new(Server) - s.cert = config.Cert - s.media = config.Media - - quicConfig := &quic.Config{} - - if config.LogDir != "" { - quicConfig.Tracer = qlog.NewTracer(func(p logging.Perspective, connectionID []byte) io.WriteCloser { - path := fmt.Sprintf("%s-%s.qlog", p, hex.EncodeToString(connectionID)) - - f, err := os.Create(filepath.Join(config.LogDir, path)) - if err != nil { - // lame - panic(err) - } - - return f - }) - } - - tlsConfig := &tls.Config{ - Certificates: []tls.Certificate{*s.cert}, - } - - // Host a HTTP/3 server to serve the WebTransport endpoint - mux := http.NewServeMux() - mux.HandleFunc("/watch", s.handleWatch) - - s.inner = &webtransport.Server{ - H3: http3.Server{ - TLSConfig: tlsConfig, - QuicConfig: quicConfig, - Addr: config.Addr, - Handler: mux, - }, - CheckOrigin: func(r *http.Request) bool { return true }, - } - - return s, nil -} - -func (s *Server) runServe(ctx context.Context) (err error) { - return s.inner.ListenAndServe() -} - -func (s *Server) runShutdown(ctx context.Context) (err error) { - <-ctx.Done() - s.inner.Close() // close on context shutdown - return ctx.Err() -} - -func (s *Server) Run(ctx context.Context) (err error) { - return invoker.Run(ctx, s.runServe, s.runShutdown, s.sessions.Repeat) -} - -func (s *Server) handleWatch(w http.ResponseWriter, r *http.Request) { - hijacker, ok := w.(http3.Hijacker) - if !ok { - panic("unable to hijack connection: must use kixelated/quic-go") - } - - conn := hijacker.Connection() - - sess, err := s.inner.Upgrade(w, r) - if err != nil { - http.Error(w, "failed to upgrade session", 500) - return - } - - err = s.serveSession(r.Context(), conn, sess) - if err != nil { - log.Println(err) - } -} - -func (s *Server) serveSession(ctx context.Context, conn quic.Connection, sess *webtransport.Session) (err error) { - defer func() { - if err != nil { - sess.CloseWithError(1, err.Error()) - } else { - sess.CloseWithError(0, "end of broadcast") - } - }() - - ss, err := NewSession(conn, sess, s.media) - if err != nil { - return fmt.Errorf("failed to create session: %w", err) - } - - err = ss.Run(ctx) - if err != nil { - return fmt.Errorf("terminated session: %w", err) - } - - return nil -} diff --git a/server/internal/warp/session.go b/server/internal/warp/session.go deleted file mode 100644 index a267619..0000000 --- a/server/internal/warp/session.go +++ /dev/null @@ -1,279 +0,0 @@ -package warp - -import ( - "context" - "encoding/binary" - "encoding/json" - "errors" - "fmt" - "io" - "log" - "math" - "time" - - "github.com/kixelated/invoker" - "github.com/kixelated/quic-go" - "github.com/kixelated/webtransport-go" -) - -// A single WebTransport session -type Session struct { - conn quic.Connection - inner *webtransport.Session - - media *Media - inits map[string]*MediaInit - audio *MediaStream - video *MediaStream - - streams invoker.Tasks -} - -func NewSession(connection quic.Connection, session *webtransport.Session, media *Media) (s *Session, err error) { - s = new(Session) - s.conn = connection - s.inner = session - s.media = media - return s, nil -} - -func (s *Session) Run(ctx context.Context) (err error) { - s.inits, s.audio, s.video, err = s.media.Start(s.conn.GetMaxBandwidth) - if err != nil { - return fmt.Errorf("failed to start media: %w", err) - } - - // Once we've validated the session, now we can start accessing the streams - return invoker.Run(ctx, s.runAccept, s.runAcceptUni, s.runInit, s.runAudio, s.runVideo, s.streams.Repeat) -} - -func (s *Session) runAccept(ctx context.Context) (err error) { - for { - stream, err := s.inner.AcceptStream(ctx) - if err != nil { - return fmt.Errorf("failed to accept bidirectional stream: %w", err) - } - - // Warp doesn't utilize bidirectional streams so just close them immediately. - // We might use them in the future so don't close the connection with an error. - stream.CancelRead(1) - } -} - -func (s *Session) runAcceptUni(ctx context.Context) (err error) { - for { - stream, err := s.inner.AcceptUniStream(ctx) - if err != nil { - return fmt.Errorf("failed to accept unidirectional stream: %w", err) - } - - s.streams.Add(func(ctx context.Context) (err error) { - return s.handleStream(ctx, stream) - }) - } -} - -func (s *Session) handleStream(ctx context.Context, stream webtransport.ReceiveStream) (err error) { - defer func() { - if err != nil { - stream.CancelRead(1) - } - }() - - var header [8]byte - for { - _, err = io.ReadFull(stream, header[:]) - if errors.Is(io.EOF, err) { - return nil - } else if err != nil { - return fmt.Errorf("failed to read atom header: %w", err) - } - - size := binary.BigEndian.Uint32(header[0:4]) - name := string(header[4:8]) - - if size < 8 { - return fmt.Errorf("atom size is too small") - } else if size > 42069 { // arbitrary limit - return fmt.Errorf("atom size is too large") - } else if name != "warp" { - return fmt.Errorf("only warp atoms are supported") - } - - payload := make([]byte, size-8) - - _, err = io.ReadFull(stream, payload) - if err != nil { - return fmt.Errorf("failed to read atom payload: %w", err) - } - - log.Println("received message:", string(payload)) - - msg := Message{} - - err = json.Unmarshal(payload, &msg) - if err != nil { - return fmt.Errorf("failed to decode json payload: %w", err) - } - - if msg.Debug != nil { - s.setDebug(msg.Debug) - } - } -} - -func (s *Session) runInit(ctx context.Context) (err error) { - for _, init := range s.inits { - err = s.writeInit(ctx, init) - if err != nil { - return fmt.Errorf("failed to write init stream: %w", err) - } - } - - return nil -} - -func (s *Session) runAudio(ctx context.Context) (err error) { - for { - segment, err := s.audio.Next(ctx) - if err != nil { - return fmt.Errorf("failed to get next segment: %w", err) - } - - if segment == nil { - return nil - } - - err = s.writeSegment(ctx, segment) - if err != nil { - return fmt.Errorf("failed to write segment stream: %w", err) - } - } -} - -func (s *Session) runVideo(ctx context.Context) (err error) { - for { - segment, err := s.video.Next(ctx) - if err != nil { - return fmt.Errorf("failed to get next segment: %w", err) - } - - if segment == nil { - return nil - } - - err = s.writeSegment(ctx, segment) - if err != nil { - return fmt.Errorf("failed to write segment stream: %w", err) - } - } -} - -// Create a stream for an INIT segment and write the container. -func (s *Session) writeInit(ctx context.Context, init *MediaInit) (err error) { - temp, err := s.inner.OpenUniStreamSync(ctx) - if err != nil { - return fmt.Errorf("failed to create stream: %w", err) - } - - if temp == nil { - // Not sure when this happens, perhaps when closing a connection? - return fmt.Errorf("received a nil stream from quic-go") - } - - // Wrap the stream in an object that buffers writes instead of blocking. - stream := NewStream(temp) - s.streams.Add(stream.Run) - - defer func() { - if err != nil { - stream.WriteCancel(1) - } - }() - - stream.SetPriority(math.MaxInt) - - err = stream.WriteMessage(Message{ - Init: &MessageInit{Id: init.ID}, - }) - if err != nil { - return fmt.Errorf("failed to write init header: %w", err) - } - - _, err = stream.Write(init.Raw) - if err != nil { - return fmt.Errorf("failed to write init data: %w", err) - } - - err = stream.Close() - if err != nil { - return fmt.Errorf("failed to close init stream: %w", err) - } - - return nil -} - -// Create a stream for a segment and write the contents, chunk by chunk. -func (s *Session) writeSegment(ctx context.Context, segment *MediaSegment) (err error) { - temp, err := s.inner.OpenUniStreamSync(ctx) - if err != nil { - return fmt.Errorf("failed to create stream: %w", err) - } - - if temp == nil { - // Not sure when this happens, perhaps when closing a connection? - return fmt.Errorf("received a nil stream from quic-go") - } - - // Wrap the stream in an object that buffers writes instead of blocking. - stream := NewStream(temp) - s.streams.Add(stream.Run) - - defer func() { - if err != nil { - stream.WriteCancel(1) - } - }() - - ms := int(segment.timestamp / time.Millisecond) - - // newer segments take priority - stream.SetPriority(ms) - - err = stream.WriteMessage(Message{ - Segment: &MessageSegment{ - Init: segment.Init.ID, - Timestamp: ms, - }, - }) - if err != nil { - return fmt.Errorf("failed to write segment header: %w", err) - } - - for { - // Get the next fragment - buf, err := segment.Read(ctx) - if errors.Is(err, io.EOF) { - break - } else if err != nil { - return fmt.Errorf("failed to read segment data: %w", err) - } - - // NOTE: This won't block because of our wrapper - _, err = stream.Write(buf) - if err != nil { - return fmt.Errorf("failed to write segment data: %w", err) - } - } - - err = stream.Close() - if err != nil { - return fmt.Errorf("failed to close segemnt stream: %w", err) - } - - return nil -} - -func (s *Session) setDebug(msg *MessageDebug) { - s.conn.SetMaxBandwidth(uint64(msg.MaxBitrate)) -} diff --git a/server/internal/warp/stream.go b/server/internal/warp/stream.go deleted file mode 100644 index 2ba56b6..0000000 --- a/server/internal/warp/stream.go +++ /dev/null @@ -1,144 +0,0 @@ -package warp - -import ( - "context" - "encoding/binary" - "encoding/json" - "fmt" - "sync" - - "github.com/kixelated/webtransport-go" -) - -// Wrapper around quic.SendStream to make Write non-blocking. -// Otherwise we can't write to multiple concurrent streams in the same goroutine. -type Stream struct { - inner webtransport.SendStream - - chunks [][]byte - closed bool - err error - - notify chan struct{} - mutex sync.Mutex -} - -func NewStream(inner webtransport.SendStream) (s *Stream) { - s = new(Stream) - s.inner = inner - s.notify = make(chan struct{}) - return s -} - -func (s *Stream) Run(ctx context.Context) (err error) { - defer func() { - s.mutex.Lock() - s.err = err - s.mutex.Unlock() - }() - - for { - s.mutex.Lock() - - chunks := s.chunks - notify := s.notify - closed := s.closed - - s.chunks = s.chunks[len(s.chunks):] - s.mutex.Unlock() - - for _, chunk := range chunks { - _, err = s.inner.Write(chunk) - if err != nil { - return err - } - } - - if closed { - return s.inner.Close() - } - - if len(chunks) == 0 { - select { - case <-ctx.Done(): - return ctx.Err() - case <-notify: - } - } - } -} - -func (s *Stream) Write(buf []byte) (n int, err error) { - s.mutex.Lock() - defer s.mutex.Unlock() - - if s.err != nil { - return 0, s.err - } - - if s.closed { - return 0, fmt.Errorf("closed") - } - - // Make a copy of the buffer so it's long lived - buf = append([]byte{}, buf...) - s.chunks = append(s.chunks, buf) - - // Wake up the writer - close(s.notify) - s.notify = make(chan struct{}) - - return len(buf), nil -} - -func (s *Stream) WriteMessage(msg Message) (err error) { - payload, err := json.Marshal(msg) - if err != nil { - return fmt.Errorf("failed to marshal message: %w", err) - } - - var size [4]byte - binary.BigEndian.PutUint32(size[:], uint32(len(payload)+8)) - - _, err = s.Write(size[:]) - if err != nil { - return fmt.Errorf("failed to write size: %w", err) - } - - _, err = s.Write([]byte("warp")) - if err != nil { - return fmt.Errorf("failed to write atom header: %w", err) - } - - _, err = s.Write(payload) - if err != nil { - return fmt.Errorf("failed to write payload: %w", err) - } - - return nil -} - -func (s *Stream) WriteCancel(code webtransport.StreamErrorCode) { - s.inner.CancelWrite(code) -} - -func (s *Stream) SetPriority(prio int) { - s.inner.SetPriority(prio) -} - -func (s *Stream) Close() (err error) { - s.mutex.Lock() - defer s.mutex.Unlock() - - if s.err != nil { - return s.err - } - - s.closed = true - - // Wake up the writer - close(s.notify) - s.notify = make(chan struct{}) - - return nil -} diff --git a/server/internal/web/web.go b/server/internal/web/web.go deleted file mode 100644 index dfd766b..0000000 --- a/server/internal/web/web.go +++ /dev/null @@ -1,64 +0,0 @@ -package web - -import ( - "context" - "log" - "net/http" - "time" - - "github.com/kixelated/invoker" -) - -type Server struct { - inner http.Server - config Config -} - -type Config struct { - Addr string - CertFile string - KeyFile string - Fingerprint string // the TLS certificate fingerprint -} - -func New(config Config) (s *Server) { - s = new(Server) - s.config = config - - s.inner = http.Server{ - Addr: config.Addr, - } - - http.HandleFunc("/fingerprint", s.handleFingerprint) - - return s -} - -func (s *Server) Run(ctx context.Context) (err error) { - return invoker.Run(ctx, s.runServe, s.runShutdown) -} - -func (s *Server) runServe(context.Context) (err error) { - // NOTE: Doesn't support context, which is why we need runShutdown - err = s.inner.ListenAndServeTLS(s.config.CertFile, s.config.KeyFile) - log.Println(err) - return err -} - -// Gracefully shut down the server when the context is cancelled -func (s *Server) runShutdown(ctx context.Context) (err error) { - <-ctx.Done() - - timeout, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _ = s.inner.Shutdown(timeout) - - return ctx.Err() -} - -// Return the sha256 of the certificate as a temporary work-around for local development. -// TODO remove this when WebTransport uses the system CA -func (s *Server) handleFingerprint(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Access-Control-Allow-Origin", "*") - _, _ = w.Write([]byte(s.config.Fingerprint)) -} diff --git a/server/main.go b/server/main.go deleted file mode 100644 index a0b5f9b..0000000 --- a/server/main.go +++ /dev/null @@ -1,71 +0,0 @@ -package main - -import ( - "context" - "crypto/sha256" - "crypto/tls" - "encoding/hex" - "flag" - "fmt" - "log" - - "github.com/kixelated/invoker" - "github.com/kixelated/warp/server/internal/warp" - "github.com/kixelated/warp/server/internal/web" -) - -func main() { - err := run(context.Background()) - if err != nil { - log.Fatal(err) - } -} - -func run(ctx context.Context) (err error) { - addr := flag.String("addr", ":4443", "HTTPS server address") - cert := flag.String("tls-cert", "../cert/localhost.crt", "TLS certificate file path") - key := flag.String("tls-key", "../cert/localhost.key", "TLS certificate file path") - logDir := flag.String("log-dir", "", "logs will be written to the provided directory") - - dash := flag.String("dash", "../media/playlist.mpd", "DASH playlist path") - - flag.Parse() - - media, err := warp.NewMedia(*dash) - if err != nil { - return fmt.Errorf("failed to open media: %w", err) - } - - tlsCert, err := tls.LoadX509KeyPair(*cert, *key) - if err != nil { - return fmt.Errorf("failed to load TLS certificate: %w", err) - } - - warpConfig := warp.Config{ - Addr: *addr, - Cert: &tlsCert, - LogDir: *logDir, - Media: media, - } - - warpServer, err := warp.New(warpConfig) - if err != nil { - return fmt.Errorf("failed to create warp server: %w", err) - } - - hash := sha256.Sum256(tlsCert.Certificate[0]) - fingerprint := hex.EncodeToString(hash[:]) - - webConfig := web.Config{ - Addr: *addr, - CertFile: *cert, - KeyFile: *key, - Fingerprint: fingerprint, - } - - webServer := web.New(webConfig) - - log.Printf("listening on %s", *addr) - - return invoker.Run(ctx, invoker.Interrupt, warpServer.Run, webServer.Run) -} diff --git a/server/src/main.rs b/server/src/main.rs new file mode 100644 index 0000000..e7a11a9 --- /dev/null +++ b/server/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} From 5204dbc19c06765c1d31d6124a261c28dfafec5f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 14 Apr 2023 13:32:02 -0700 Subject: [PATCH 02/23] Refactor and restructure the WebTransport code. Who knows if it actually works. --- player/src/index.ts | 2 +- server/Cargo.lock | 452 +++++++++++++++++++++++++++++++++++++++++- server/Cargo.toml | 7 +- server/src/error.rs | 44 ++++ server/src/lib.rs | 3 + server/src/main.rs | 35 +++- server/src/server.rs | 332 +++++++++++++++++++++++++++++++ server/src/session.rs | 104 ++++++++++ 8 files changed, 972 insertions(+), 7 deletions(-) create mode 100644 server/src/error.rs create mode 100644 server/src/lib.rs create mode 100644 server/src/server.rs create mode 100644 server/src/session.rs diff --git a/player/src/index.ts b/player/src/index.ts index e691ab5..89e2e22 100644 --- a/player/src/index.ts +++ b/player/src/index.ts @@ -11,7 +11,7 @@ for (let c = 0; c < fingerprintHex.length-1; c += 2) { const params = new URLSearchParams(window.location.search) -const url = params.get("url") || "https://localhost:4443/watch" +const url = params.get("url") || "https://127.0.0.1:4443/watch" const canvas = document.querySelector("canvas#video")! const player = new Player({ diff --git a/server/Cargo.lock b/server/Cargo.lock index 6110308..035e890 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -2,12 +2,87 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e579a7752471abc2a8268df8b20005e3eadd975f585398f17efcfd8d4927371" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is-terminal", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" + +[[package]] +name = "anstyle-parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcd8291a340dd8ac70e18878bc4501dd7b4ff970cfa21c207d36ece51ea88fd" +dependencies = [ + "anstyle", + "windows-sys 0.48.0", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bumpalo" version = "3.12.0" @@ -26,6 +101,48 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clap" +version = "4.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b802d85aaf3a1cdb02b224ba472ebdea62014fccfcb269b95a4d76443b5ee5a" +dependencies = [ + "clap_builder", + "clap_derive", + "once_cell", +] + +[[package]] +name = "clap_builder" +version = "4.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14a1a858f532119338887a4b8e1af9c60de8249cd7bafd68036a489e261e37b6" +dependencies = [ + "anstream", + "anstyle", + "bitflags", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "clap_lex" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" + [[package]] name = "cmake" version = "0.1.50" @@ -35,6 +152,96 @@ dependencies = [ "cc", ] +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "env_logger" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "io-lifetimes" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +dependencies = [ + "hermit-abi 0.3.1", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "is-terminal" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +dependencies = [ + "hermit-abi 0.3.1", + "io-lifetimes", + "rustix", + "windows-sys 0.48.0", +] + [[package]] name = "js-sys" version = "0.3.61" @@ -62,6 +269,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" +[[package]] +name = "linux-raw-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" + [[package]] name = "log" version = "0.4.17" @@ -71,6 +284,24 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "mio" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.45.0", +] + [[package]] name = "octets" version = "0.2.0" @@ -117,6 +348,23 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "regex" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "ring" version = "0.16.20" @@ -132,6 +380,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "rustix" +version = "0.37.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.48.0", +] + [[package]] name = "serde" version = "1.0.160" @@ -162,6 +424,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "syn" version = "1.0.109" @@ -173,6 +441,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +dependencies = [ + "winapi-util", +] + [[package]] name = "unicode-ident" version = "1.0.8" @@ -186,12 +474,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] -name = "warp-server" +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "warp" version = "0.1.0" dependencies = [ + "clap", + "env_logger", + "log", + "mio", "quiche", + "ring", ] +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wasm-bindgen" version = "0.2.84" @@ -213,7 +518,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-shared", ] @@ -235,7 +540,7 @@ checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -272,8 +577,149 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" diff --git a/server/Cargo.toml b/server/Cargo.toml index d4dcbb7..c5f4afd 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "warp-server" +name = "warp" version = "0.1.0" edition = "2021" @@ -7,3 +7,8 @@ edition = "2021" [dependencies] quiche = { git = "https://github.com/n8o/quiche.git", branch = "master" } # WebTransport fork +clap = { version = "4.0", features = [ "derive" ] } +log = { version = "0.4", features = ["std"] } +mio = { version = "0.8", features = ["net", "os-poll"] } +env_logger = "0.9.3" +ring = "0.16" \ No newline at end of file diff --git a/server/src/error.rs b/server/src/error.rs new file mode 100644 index 0000000..ef10de2 --- /dev/null +++ b/server/src/error.rs @@ -0,0 +1,44 @@ +use std::io; +use quiche::h3::webtransport; + +#[derive(Debug)] +pub enum Error { + Io(io::Error), + Quiche(quiche::Error), + WebTransport(webtransport::Error), + Server(Server), +} + +impl From for Error { + fn from(err: io::Error) -> Error { + Error::Io(err) + } +} + +impl From for Error { + fn from(err: quiche::Error) -> Error { + Error::Quiche(err) + } +} + +impl From for Error { + fn from(err: webtransport::Error) -> Error { + Error::WebTransport(err) + } +} + +// Custom server error messages. +#[derive(Debug)] +pub enum Server { + InvalidToken, + InvalidConnectionID, + UnknownConnectionID, +} + +impl From for Error { + fn from(err: Server) -> Error { + Error::Server(err) + } +} + +pub type Result = std::result::Result; \ No newline at end of file diff --git a/server/src/lib.rs b/server/src/lib.rs new file mode 100644 index 0000000..241d22a --- /dev/null +++ b/server/src/lib.rs @@ -0,0 +1,3 @@ +pub mod error; +pub mod server; +pub mod session; \ No newline at end of file diff --git a/server/src/main.rs b/server/src/main.rs index e7a11a9..233d2cc 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,3 +1,34 @@ -fn main() { - println!("Hello, world!"); +use warp::server::Server; + +use clap::Parser; + +/// Search for a pattern in a file and display the lines that contain it. +#[derive(Parser)] +struct Cli { + /// Listen on this address + #[arg(short, long, default_value = "127.0.0.1:4443")] + addr: String, + + /// Use the certificate file at this path + #[arg(short, long, default_value = "../cert/localhost.crt")] + cert: String, + + /// Use the private key at this path + #[arg(short, long, default_value = "../cert/localhost.key")] + key: String, } + +fn main() { + let args = Cli::parse(); + + let server_config = warp::server::Config{ + addr: args.addr, + cert: args.cert, + key: args.key, + }; + + let mut server = Server::new(server_config).unwrap(); + loop { + server.poll().unwrap() + } +} \ No newline at end of file diff --git a/server/src/server.rs b/server/src/server.rs new file mode 100644 index 0000000..b3f2359 --- /dev/null +++ b/server/src/server.rs @@ -0,0 +1,332 @@ +use crate::session; +use crate::error; + +use session::Session; +use error::{Error, Result}; + +use std::{io, net}; +use log; + +use quiche::h3::webtransport; + +const MAX_DATAGRAM_SIZE: usize = 1350; + +pub struct Server { + // IO stuff + socket: mio::net::UdpSocket, + poll: mio::Poll, + events: mio::Events, + + // QUIC stuff + quic: quiche::Config, + sessions: session::Map, + seed: ring::hmac::Key, // connection ID seed +} + +pub struct Config { + pub addr: String, + pub cert: String, + pub key: String, +} + +impl Server { + pub fn new(config: Config) -> io::Result { + // Listen on the provided socket address + let addr = config.addr.parse().unwrap(); + let mut socket = mio::net::UdpSocket::bind(addr).unwrap(); + + // Setup the event loop. + let poll = mio::Poll::new().unwrap(); + let events = mio::Events::with_capacity(1024); + let sessions = session::Map::new(); + + poll.registry().register( + &mut socket, + mio::Token(0), + mio::Interest::READABLE, + ).unwrap(); + + // Generate random values for connection IDs. + let rng = ring::rand::SystemRandom::new(); + let seed = ring::hmac::Key::generate(ring::hmac::HMAC_SHA256, &rng).unwrap(); + + // Create the configuration for the QUIC connections. + let mut quic = quiche::Config::new(quiche::PROTOCOL_VERSION).unwrap(); + quic.load_cert_chain_from_pem_file(&config.cert).unwrap(); + quic.load_priv_key_from_pem_file(&config.key).unwrap(); + quic.set_application_protos(quiche::h3::APPLICATION_PROTOCOL).unwrap(); + quic.set_max_idle_timeout(5000); + quic.set_max_recv_udp_payload_size(MAX_DATAGRAM_SIZE); + quic.set_max_send_udp_payload_size(MAX_DATAGRAM_SIZE); + quic.set_initial_max_data(10_000_000); + quic.set_initial_max_stream_data_bidi_local(1_000_000); + quic.set_initial_max_stream_data_bidi_remote(1_000_000); + quic.set_initial_max_stream_data_uni(1_000_000); + quic.set_initial_max_streams_bidi(100); + quic.set_initial_max_streams_uni(100); + quic.set_disable_active_migration(true); + quic.enable_early_data(); + quic.enable_dgram(true, 65536, 65536); + + Ok(Server { + socket, + poll, + events, + + quic, + sessions, + seed + }) + } + + pub fn poll(&mut self) -> io::Result<()> { + self.receive().unwrap(); + self.send().unwrap(); + self.cleanup().unwrap(); + + Ok(()) + } + + fn receive(&mut self) -> io::Result<()> { + // Find the shorter timeout from all the active connections. + // + // TODO: use event loop that properly supports timers + let timeout = self.sessions.values().filter_map(|c| c.conn.timeout()).min(); + + self.poll.poll(&mut self.events, timeout).unwrap(); + + // If the event loop reported no events, it means that the timeout + // has expired, so handle it without attempting to read packets. We + // will then proceed with the send loop. + if self.events.is_empty() { + self.sessions.values_mut().for_each(|session| { + session.conn.on_timeout() + }); + + return Ok(()) + } + + // Read incoming UDP packets from the socket and feed them to quiche, + // until there are no more packets to read. + loop { + match self.receive_once() { + Err(Error::Io(e)) if e.kind() == io::ErrorKind::WouldBlock => return Ok(()), + Err(e) => log::error!("{:?}", e), + Ok(_) => (), + } + } + } + + fn receive_once(&mut self) -> Result<()> { + let mut src= [0; MAX_DATAGRAM_SIZE]; + + let (len, from) = self.socket.recv_from(&mut src).unwrap(); + let src = &mut src[..len]; + + let info = quiche::RecvInfo { + to: self.socket.local_addr().unwrap(), + from, + }; + + // Lookup a connection based on the packet's connection ID. If there + // is no connection matching, create a new one. + let pair = match self.accept(src, from).unwrap() { + Some(v) => v, + None => return Ok(()), + }; + + let conn = &mut pair.conn; + + // Process potentially coalesced packets. + conn.recv(src, info).unwrap(); + + // Create a new HTTP/3 connection as soon as the QUIC connection + // is established. + if (conn.is_in_early_data() || conn.is_established()) && pair.session.is_none() { + let session = webtransport::ServerSession::with_transport(conn).unwrap(); + pair.session = Some(session); + } + + // The `poll` can pull out the events that occurred according to the data passed here. + for (_, session) in self.sessions.iter_mut() { + session.poll().unwrap(); + } + + Ok(()) + } + + fn accept(&mut self, src: &mut [u8], from: net::SocketAddr) -> error::Result> { + // Parse the QUIC packet's header. + let hdr = quiche::Header::from_slice(src, quiche::MAX_CONN_ID_LEN).unwrap(); + + let conn_id = ring::hmac::sign(&self.seed, &hdr.dcid); + let conn_id = &conn_id.as_ref()[..quiche::MAX_CONN_ID_LEN]; + let conn_id = conn_id.to_vec().into(); + + if self.sessions.contains_key(&hdr.dcid) { + let pair = self.sessions.get_mut(&hdr.dcid).unwrap(); + return Ok(Some(pair)) + } else if self.sessions.contains_key(&conn_id) { + let pair = self.sessions.get_mut(&conn_id).unwrap(); + return Ok(Some(pair)); + } + + if hdr.ty != quiche::Type::Initial { + return Err(error::Server::UnknownConnectionID.into()) + } + + let mut dst = [0; MAX_DATAGRAM_SIZE]; + + if !quiche::version_is_supported(hdr.version) { + let len = quiche::negotiate_version(&hdr.scid, &hdr.dcid, &mut dst).unwrap(); + let dst= &dst[..len]; + + self.socket.send_to(dst, from).unwrap(); + return Ok(None) + } + + let mut scid = [0; quiche::MAX_CONN_ID_LEN]; + scid.copy_from_slice(&conn_id); + + let scid = quiche::ConnectionId::from_ref(&scid); + + // Token is always present in Initial packets. + let token = hdr.token.as_ref().unwrap(); + + // Do stateless retry if the client didn't send a token. + if token.is_empty() { + let new_token = mint_token(&hdr, &from); + + let len = quiche::retry( + &hdr.scid, + &hdr.dcid, + &scid, + &new_token, + hdr.version, + &mut dst, + ) + .unwrap(); + + let dst= &dst[..len]; + + self.socket.send_to(dst, from).unwrap(); + return Ok(None) + } + + let odcid = validate_token(&from, token); + + // The token was not valid, meaning the retry failed, so + // drop the packet. + if odcid.is_none() { + return Err(error::Server::InvalidToken.into()) + } + + if scid.len() != hdr.dcid.len() { + return Err(error::Server::InvalidConnectionID.into()) + } + + // Reuse the source connection ID we sent in the Retry packet, + // instead of changing it again. + let conn_id= hdr.dcid.clone(); + let local_addr = self.socket.local_addr().unwrap(); + + let conn = + quiche::accept(&conn_id, odcid.as_ref(), local_addr, from, &mut self.quic) + .unwrap(); + + self.sessions.insert( + conn_id.clone(), + Session { + conn, + session: None, + }, + ); + + let pair = self.sessions.get_mut(&conn_id).unwrap(); + Ok(Some(pair)) + } + + fn send(&mut self) -> io::Result<()> { + let mut pkt = [0; MAX_DATAGRAM_SIZE]; + + // Generate outgoing QUIC packets for all active connections and send + // them on the UDP socket, until quiche reports that there are no more + // packets to be sent. + for session in self.sessions.values_mut() { + loop { + let (size , info) = session.conn.send(&mut pkt).unwrap(); + let pkt = &pkt[..size]; + + match self.socket.send_to(&pkt, info.to) { + Err(err) if err.kind() == io::ErrorKind::WouldBlock => break, + Err(err) => return Err(err), + Ok(_) => (), + } + } + } + + Ok(()) + } + + fn cleanup(&mut self) -> io::Result<()> { + // Garbage collect closed connections. + self.sessions.retain(|_, session| !session.conn.is_closed() ); + Ok(()) + } +} + +/// Generate a stateless retry token. +/// +/// The token includes the static string `"quiche"` followed by the IP address +/// of the client and by the original destination connection ID generated by the +/// client. +/// +/// Note that this function is only an example and doesn't do any cryptographic +/// authenticate of the token. *It should not be used in production system*. +fn mint_token(hdr: &quiche::Header, src: &std::net::SocketAddr) -> Vec { + let mut token = Vec::new(); + + token.extend_from_slice(b"quiche"); + + let addr = match src.ip() { + std::net::IpAddr::V4(a) => a.octets().to_vec(), + std::net::IpAddr::V6(a) => a.octets().to_vec(), + }; + + token.extend_from_slice(&addr); + token.extend_from_slice(&hdr.dcid); + + token +} + +/// Validates a stateless retry token. +/// +/// This checks that the ticket includes the `"quiche"` static string, and that +/// the client IP address matches the address stored in the ticket. +/// +/// Note that this function is only an example and doesn't do any cryptographic +/// authenticate of the token. *It should not be used in production system*. +fn validate_token<'a>( + src: &std::net::SocketAddr, token: &'a [u8], +) -> Option> { + if token.len() < 6 { + return None; + } + + if &token[..6] != b"quiche" { + return None; + } + + let token = &token[6..]; + + let addr = match src.ip() { + std::net::IpAddr::V4(a) => a.octets().to_vec(), + std::net::IpAddr::V6(a) => a.octets().to_vec(), + }; + + if token.len() < addr.len() || &token[..addr.len()] != addr.as_slice() { + return None; + } + + Some(quiche::ConnectionId::from_ref(&token[addr.len()..])) +} \ No newline at end of file diff --git a/server/src/session.rs b/server/src/session.rs new file mode 100644 index 0000000..e06bcfa --- /dev/null +++ b/server/src/session.rs @@ -0,0 +1,104 @@ +use crate::error; +use error::Result; + +use std::collections::HashMap; +use quiche::h3::webtransport; + +pub struct Session { + pub conn: quiche::Connection, + pub session: Option, +} + +pub type Map = HashMap, Session>; + +impl Session { + // Process any updates to a session. + pub fn poll(&mut self) -> Result<()> { + let session = match &mut self.session { + Some(s) => s, + None => return Ok(()), + }; + + loop { + let event = match session.poll(&mut self.conn) { + Err(webtransport::Error::Done) => return Ok(()), + Err(e) => return Err(e.into()), + Ok(e) => e, + }; + + match event { + webtransport::ServerEvent::ConnectRequest(_req) => { + // you can handle request with + // req.authority() + // req.path() + // and you can validate this request with req.origin() + session.accept_connect_request(&mut self.conn, None).unwrap(); + }, + webtransport::ServerEvent::StreamData(stream_id) => { + let mut buf = vec![0; 10000]; + while let Ok(len) = + session.recv_stream_data(&mut self.conn, stream_id, &mut buf) + { + let stream_data = &buf[0..len]; + + // handle stream_data + if (stream_id & 0x2) == 0 { + // bidirectional stream + // you can send data through this stream. + session + .send_stream_data(&mut self.conn, stream_id, stream_data) + .unwrap(); + } else { + // you cannot send data through client-initiated-unidirectional-stream. + // so, open new server-initiated-unidirectional-stream, and send data + // through it. + let new_stream_id = + session.open_stream(&mut self.conn, false).unwrap(); + session + .send_stream_data(&mut self.conn, new_stream_id, stream_data) + .unwrap(); + } + } + } + + webtransport::ServerEvent::StreamFinished(_stream_id) => { + // A WebTrnasport stream finished, handle it. + } + + webtransport::ServerEvent::Datagram => { + let mut buf = vec![0; 1500]; + while let Ok((in_session, offset, total)) = + session.recv_dgram(&mut self.conn, &mut buf) + { + if in_session { + let dgram = &buf[offset..total]; + dbg!(std::string::String::from_utf8_lossy(dgram)); + // handle this dgram + + // for instance, you can write echo-server like following + session.send_dgram(&mut self.conn, dgram).unwrap(); + } else { + // this dgram is not related to current WebTransport session. ignore. + } + } + } + + webtransport::ServerEvent::SessionReset(_e) => { + // Peer reset session stream, handle it. + } + + webtransport::ServerEvent::SessionFinished => { + // Peer finish session stream, handle it. + } + + webtransport::ServerEvent::SessionGoAway => { + // Peer signalled it is going away, handle it. + } + + webtransport::ServerEvent::Other(_stream_id, _event) => { + // Original h3::Event which is not related to WebTransport. + } + } + } + } +} \ No newline at end of file From bb0437a3bbc237e8c0ccad454f5dadd33167cd34 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 24 Apr 2023 10:18:55 -0700 Subject: [PATCH 03/23] More refactoring ofc. --- media/generate | 20 +- server/.vscode/settings.json | 3 + server/Cargo.lock | 136 ++++++++++++ server/Cargo.toml | 6 +- server/src/error.rs | 44 ---- server/src/lib.rs | 5 +- server/src/main.rs | 47 +++- server/src/media.rs | 108 ++++++++++ server/src/media/mod.rs | 1 + server/src/media/source.rs | 163 ++++++++++++++ server/src/message.rs | 40 ++++ server/src/server.rs | 332 ---------------------------- server/src/session.rs | 104 --------- server/src/transport/app.rs | 8 + server/src/transport/connection.rs | 15 ++ server/src/transport/mod.rs | 7 + server/src/transport/server.rs | 334 +++++++++++++++++++++++++++++ server/src/transport/session.rs | 252 ++++++++++++++++++++++ 18 files changed, 1118 insertions(+), 507 deletions(-) create mode 100644 server/.vscode/settings.json delete mode 100644 server/src/error.rs create mode 100644 server/src/media.rs create mode 100644 server/src/media/mod.rs create mode 100644 server/src/media/source.rs create mode 100644 server/src/message.rs delete mode 100644 server/src/server.rs delete mode 100644 server/src/session.rs create mode 100644 server/src/transport/app.rs create mode 100644 server/src/transport/connection.rs create mode 100644 server/src/transport/mod.rs create mode 100644 server/src/transport/server.rs create mode 100644 server/src/transport/session.rs diff --git a/media/generate b/media/generate index 110f401..b387ac9 100755 --- a/media/generate +++ b/media/generate @@ -1,18 +1,6 @@ #!/bin/bash ffmpeg -i source.mp4 \ - -f dash -ldash 1 \ - -c:v libx264 \ - -preset veryfast -tune zerolatency \ - -c:a aac \ - -b:a 128k -ac 2 -ar 44100 \ - -map v:0 -s:v:0 1280x720 -b:v:0 3M \ - -map v:0 -s:v:1 854x480 -b:v:1 1.1M \ - -map v:0 -s:v:2 640x360 -b:v:2 365k \ - -map 0:a \ - -force_key_frames "expr:gte(t,n_forced*2)" \ - -sc_threshold 0 \ - -streaming 1 \ - -use_timeline 0 \ - -seg_duration 2 -frag_duration 0.01 \ - -frag_type duration \ - playlist.mpd + -c:v copy \ + -an \ + -movflags frag_every_frame+empty_moov \ + fragmented.mp4 diff --git a/server/.vscode/settings.json b/server/.vscode/settings.json new file mode 100644 index 0000000..4d9636b --- /dev/null +++ b/server/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "rust-analyzer.showUnlinkedFileNotification": false +} \ No newline at end of file diff --git a/server/Cargo.lock b/server/Cargo.lock index 035e890..00d51d5 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -60,6 +60,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "anyhow" +version = "1.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" + [[package]] name = "atty" version = "0.2.14" @@ -89,6 +95,18 @@ version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + [[package]] name = "cc" version = "1.0.79" @@ -242,6 +260,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + [[package]] name = "js-sys" version = "0.3.61" @@ -302,6 +326,63 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "mp4" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "509348cba250e7b852a875100a2ddce7a36ee3abf881a681c756670c1774264d" +dependencies = [ + "byteorder", + "bytes", + "num-rational", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + [[package]] name = "octets" version = "0.2.0" @@ -394,11 +475,42 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + [[package]] name = "serde" version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.160" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "serde_json" +version = "1.0.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +dependencies = [ + "itoa", + "ryu", + "serde", +] [[package]] name = "slab" @@ -461,6 +573,26 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "thiserror" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + [[package]] name = "unicode-ident" version = "1.0.8" @@ -483,12 +615,16 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" name = "warp" version = "0.1.0" dependencies = [ + "anyhow", "clap", "env_logger", "log", "mio", + "mp4", "quiche", "ring", + "serde", + "serde_json", ] [[package]] diff --git a/server/Cargo.toml b/server/Cargo.toml index c5f4afd..b397086 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -11,4 +11,8 @@ clap = { version = "4.0", features = [ "derive" ] } log = { version = "0.4", features = ["std"] } mio = { version = "0.8", features = ["net", "os-poll"] } env_logger = "0.9.3" -ring = "0.16" \ No newline at end of file +ring = "0.16" +anyhow = "1.0.70" +mp4 = "0.13.0" +serde = "1.0.160" +serde_json = "1.0" \ No newline at end of file diff --git a/server/src/error.rs b/server/src/error.rs deleted file mode 100644 index ef10de2..0000000 --- a/server/src/error.rs +++ /dev/null @@ -1,44 +0,0 @@ -use std::io; -use quiche::h3::webtransport; - -#[derive(Debug)] -pub enum Error { - Io(io::Error), - Quiche(quiche::Error), - WebTransport(webtransport::Error), - Server(Server), -} - -impl From for Error { - fn from(err: io::Error) -> Error { - Error::Io(err) - } -} - -impl From for Error { - fn from(err: quiche::Error) -> Error { - Error::Quiche(err) - } -} - -impl From for Error { - fn from(err: webtransport::Error) -> Error { - Error::WebTransport(err) - } -} - -// Custom server error messages. -#[derive(Debug)] -pub enum Server { - InvalidToken, - InvalidConnectionID, - UnknownConnectionID, -} - -impl From for Error { - fn from(err: Server) -> Error { - Error::Server(err) - } -} - -pub type Result = std::result::Result; \ No newline at end of file diff --git a/server/src/lib.rs b/server/src/lib.rs index 241d22a..915e3b1 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -1,3 +1,2 @@ -pub mod error; -pub mod server; -pub mod session; \ No newline at end of file +pub mod transport; +//mod media; \ No newline at end of file diff --git a/server/src/main.rs b/server/src/main.rs index 233d2cc..9c44d0f 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,6 +1,10 @@ -use warp::server::Server; +use quiche::h3::webtransport; +use warp::transport; + +use std::time; use clap::Parser; +use env_logger; /// Search for a pattern in a file and display the lines that contain it. #[derive(Parser)] @@ -16,19 +20,48 @@ struct Cli { /// Use the private key at this path #[arg(short, long, default_value = "../cert/localhost.key")] key: String, + + /// Use the media file at this path + #[arg(short, long, default_value = "../media/fragmented.mp4")] + media: String, } -fn main() { +#[derive(Default)] +struct Connection { + webtransport: Option, +} + +impl transport::App for Connection { + fn poll(&mut self, conn: &mut quiche::Connection, session: &mut webtransport::ServerSession) -> anyhow::Result<()> { + if !conn.is_established() { + // Wait until the handshake finishes + return Ok(()) + } + + if self.webtransport.is_none() { + self.webtransport = Some(webtransport::ServerSession::with_transport(conn)?) + } + + let webtransport = self.webtransport.as_mut().unwrap(); + + Ok(()) + } + + fn timeout(&self) -> Option { + None + } +} +fn main() -> anyhow::Result<()> { + env_logger::init(); + let args = Cli::parse(); - let server_config = warp::server::Config{ + let server_config = transport::Config{ addr: args.addr, cert: args.cert, key: args.key, }; - let mut server = Server::new(server_config).unwrap(); - loop { - server.poll().unwrap() - } + let mut server = transport::Server::::new(server_config).unwrap(); + server.run() } \ No newline at end of file diff --git a/server/src/media.rs b/server/src/media.rs new file mode 100644 index 0000000..5e86adb --- /dev/null +++ b/server/src/media.rs @@ -0,0 +1,108 @@ +use std::{io,fs}; + +use mp4; +use anyhow; +use bytes; + +use mp4::ReadBox; + +pub struct Source { + pub segments: Vec, +} + +impl Source { + pub fn new(path: &str) -> anyhow::Result { + let f = fs::read(path)?; + let mut bytes = bytes::Bytes::from(f); + + let mut segments = Vec::new(); + let mut current = Segment::new(); + + while bytes.len() > 0 { + // NOTE: Cloning is cheap, since the underlying bytes are reference counted. + let mut reader = io::Cursor::new(bytes.clone()); + + let header = mp4::BoxHeader::read(&mut reader)?; + let size: usize = header.size as usize; + + assert!(size > 0, "empty box"); + + let frag = bytes.split_to(size); + let fragment = Fragment{ bytes: frag }; + + match header.name { + /* + mp4::BoxType::FtypBox => { + } + mp4::BoxType::MoovBox => { + moov = mp4::MoovBox::read_box(&mut reader, size)? + } + mp4::BoxType::EmsgBox => { + let emsg = mp4::EmsgBox::read_box(&mut reader, size)?; + emsgs.push(emsg); + } + mp4::BoxType::MdatBox => { + mp4::skip_box(&mut reader, size)?; + } + */ + mp4::BoxType::MoofBox => { + let moof = mp4::MoofBox::read_box(&mut reader, header.size)?; + if has_keyframe(moof) { + segments.push(current); + current = Segment::new(); + } + } + _ => (), + } + + current.fragments.push(fragment); + } + + segments.push(current); + + Ok(Self { segments }) + } +} + +fn has_keyframe(moof: mp4::MoofBox) -> bool { + for traf in moof.trafs { + // TODO trak default flags if this is None + let default_flags = traf.tfhd.default_sample_flags.unwrap_or_default(); + let trun = traf.trun.expect("missing trun box"); + + for i in 0..trun.sample_count { + let mut flags = match trun.sample_flags.get(i as usize) { + Some(f) => *f, + None => default_flags, + }; + + if i == 0 && trun.first_sample_flags.is_some() { + flags = trun.first_sample_flags.unwrap(); + } + + // https://chromium.googlesource.com/chromium/src/media/+/master/formats/mp4/track_run_iterator.cc#177 + let keyframe = (flags >> 24) & 0x3 == 0x2; // kSampleDependsOnNoOther + let non_sync = (flags >> 16) & 0x1 == 0x1; // kSampleIsNonSyncSample + + if keyframe && non_sync { + return true + } + } + } + + false +} + +pub struct Segment { + pub fragments: Vec, +} + +impl Segment { + fn new() -> Self { + Segment { fragments: Vec::new() } + } +} + +pub struct Fragment { + pub bytes: bytes::Bytes, +} diff --git a/server/src/media/mod.rs b/server/src/media/mod.rs new file mode 100644 index 0000000..b5cb700 --- /dev/null +++ b/server/src/media/mod.rs @@ -0,0 +1 @@ +pub mod source; \ No newline at end of file diff --git a/server/src/media/source.rs b/server/src/media/source.rs new file mode 100644 index 0000000..7d83b5b --- /dev/null +++ b/server/src/media/source.rs @@ -0,0 +1,163 @@ +use std::{io,fs,time}; +use io::Read; + +use mp4; +use anyhow; + +use mp4::ReadBox; + +pub struct Source { + reader: io::BufReader, + start: time::Instant, + + pending: Option, + sequence: u64, +} + +pub struct Fragment { + pub data: Vec, + pub segment_id: u64, + pub timestamp: u64, +} + +impl Source { + pub fn new(path: &str) -> io::Result { + let f = fs::File::open(path)?; + let reader = io::BufReader::new(f); + let start = time::Instant::now(); + + Ok(Self{ + reader, + start, + pending: None, + sequence: 0, + }) + } + + pub fn next(&mut self) -> anyhow::Result> { + let pending = match self.pending.take() { + Some(f) => f, + None => self.next_inner()?, + }; + + if pending.timestamp > 0 && pending.timestamp < self.start.elapsed().as_millis() as u64 { + self.pending = Some(pending); + return Ok(None) + } + + Ok(Some(pending)) + } + + fn next_inner(&mut self) -> anyhow::Result { + // Read the next full atom. + let atom = read_box(&mut self.reader)?; + let mut timestamp = 0; + + // Before we return it, let's do some simple parsing. + let mut reader = io::Cursor::new(&atom); + let header = mp4::BoxHeader::read(&mut reader)?; + + + match header.name { + mp4::BoxType::MoofBox => { + let moof = mp4::MoofBox::read_box(&mut reader, header.size)?; + + if has_keyframe(&moof) { + self.sequence += 1 + } + + timestamp = first_timestamp(&moof); + } + _ => (), + } + + Ok(Fragment { + data: atom, + segment_id: self.sequence, + timestamp: timestamp, + }) + } +} + +// Read a full MP4 atom into a vector. +fn read_box(reader: &mut R) -> anyhow::Result> { + // Read the 8 bytes for the size + type + let mut buf = [0u8; 8]; + reader.read_exact(&mut buf)?; + + // Convert the first 4 bytes into the size. + let size = u32::from_be_bytes(buf[0..4].try_into()?) as u64; + let mut out = buf.to_vec(); + + let mut limit = match size { + // Runs until the end of the file. + 0 => reader.take(u64::MAX), + + // The next 8 bytes are the extended size to be used instead. + 1 => { + reader.read_exact(&mut buf)?; + let size_large = u64::from_be_bytes(buf); + anyhow::ensure!(size_large >= 16, "impossible extended box size: {}", size_large); + + reader.take(size_large - 16) + }, + + 2..=7 => { + anyhow::bail!("impossible box size: {}", size) + } + + // Otherwise read based on the size. + size => reader.take(size - 8) + }; + + // Append to the vector and return it. + limit.read_to_end(&mut out)?; + + Ok(out) +} + +fn has_keyframe(moof: &mp4::MoofBox) -> bool { + for traf in &moof.trafs { + // TODO trak default flags if this is None + let default_flags = traf.tfhd.default_sample_flags.unwrap_or_default(); + let trun = match &traf.trun { + Some(t) => t, + None => return false, + }; + + for i in 0..trun.sample_count { + let mut flags = match trun.sample_flags.get(i as usize) { + Some(f) => *f, + None => default_flags, + }; + + if i == 0 && trun.first_sample_flags.is_some() { + flags = trun.first_sample_flags.unwrap(); + } + + // https://chromium.googlesource.com/chromium/src/media/+/master/formats/mp4/track_run_iterator.cc#177 + let keyframe = (flags >> 24) & 0x3 == 0x2; // kSampleDependsOnNoOther + let non_sync = (flags >> 16) & 0x1 == 0x1; // kSampleIsNonSyncSample + + if keyframe && non_sync { + return true + } + } + } + + false +} + +fn first_timestamp(moof: &mp4::MoofBox) -> u64 { + let traf = match moof.trafs.first() { + Some(t) => t, + None => return 0, + }; + + let tfdt = match &traf.tfdt { + Some(t) => t, + None => return 0, + }; + + tfdt.base_media_decode_time +} diff --git a/server/src/message.rs b/server/src/message.rs new file mode 100644 index 0000000..1421e91 --- /dev/null +++ b/server/src/message.rs @@ -0,0 +1,40 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +pub struct Message { + pub init: Option, + pub segment: Option, +} + +#[derive(Serialize, Deserialize)] +pub struct Init { + pub id: String, +} + +#[derive(Serialize, Deserialize)] +pub struct Segment { + pub init: String, + pub timestamp: u64, +} + +impl Message { + pub fn new() -> Self { + Message { + init: None, + segment: None, + } + } + + pub fn serialize(&self) -> anyhow::Result> { + let str = serde_json::to_string(self)?; + let bytes = str.as_bytes(); + let size = bytes.len() + 8; + + let mut out = Vec::with_capacity(size); + out.extend_from_slice(b"warp"); + out.extend_from_slice(&size.to_be_bytes()); + out.extend_from_slice(bytes); + + Ok(out) + } +} \ No newline at end of file diff --git a/server/src/server.rs b/server/src/server.rs deleted file mode 100644 index b3f2359..0000000 --- a/server/src/server.rs +++ /dev/null @@ -1,332 +0,0 @@ -use crate::session; -use crate::error; - -use session::Session; -use error::{Error, Result}; - -use std::{io, net}; -use log; - -use quiche::h3::webtransport; - -const MAX_DATAGRAM_SIZE: usize = 1350; - -pub struct Server { - // IO stuff - socket: mio::net::UdpSocket, - poll: mio::Poll, - events: mio::Events, - - // QUIC stuff - quic: quiche::Config, - sessions: session::Map, - seed: ring::hmac::Key, // connection ID seed -} - -pub struct Config { - pub addr: String, - pub cert: String, - pub key: String, -} - -impl Server { - pub fn new(config: Config) -> io::Result { - // Listen on the provided socket address - let addr = config.addr.parse().unwrap(); - let mut socket = mio::net::UdpSocket::bind(addr).unwrap(); - - // Setup the event loop. - let poll = mio::Poll::new().unwrap(); - let events = mio::Events::with_capacity(1024); - let sessions = session::Map::new(); - - poll.registry().register( - &mut socket, - mio::Token(0), - mio::Interest::READABLE, - ).unwrap(); - - // Generate random values for connection IDs. - let rng = ring::rand::SystemRandom::new(); - let seed = ring::hmac::Key::generate(ring::hmac::HMAC_SHA256, &rng).unwrap(); - - // Create the configuration for the QUIC connections. - let mut quic = quiche::Config::new(quiche::PROTOCOL_VERSION).unwrap(); - quic.load_cert_chain_from_pem_file(&config.cert).unwrap(); - quic.load_priv_key_from_pem_file(&config.key).unwrap(); - quic.set_application_protos(quiche::h3::APPLICATION_PROTOCOL).unwrap(); - quic.set_max_idle_timeout(5000); - quic.set_max_recv_udp_payload_size(MAX_DATAGRAM_SIZE); - quic.set_max_send_udp_payload_size(MAX_DATAGRAM_SIZE); - quic.set_initial_max_data(10_000_000); - quic.set_initial_max_stream_data_bidi_local(1_000_000); - quic.set_initial_max_stream_data_bidi_remote(1_000_000); - quic.set_initial_max_stream_data_uni(1_000_000); - quic.set_initial_max_streams_bidi(100); - quic.set_initial_max_streams_uni(100); - quic.set_disable_active_migration(true); - quic.enable_early_data(); - quic.enable_dgram(true, 65536, 65536); - - Ok(Server { - socket, - poll, - events, - - quic, - sessions, - seed - }) - } - - pub fn poll(&mut self) -> io::Result<()> { - self.receive().unwrap(); - self.send().unwrap(); - self.cleanup().unwrap(); - - Ok(()) - } - - fn receive(&mut self) -> io::Result<()> { - // Find the shorter timeout from all the active connections. - // - // TODO: use event loop that properly supports timers - let timeout = self.sessions.values().filter_map(|c| c.conn.timeout()).min(); - - self.poll.poll(&mut self.events, timeout).unwrap(); - - // If the event loop reported no events, it means that the timeout - // has expired, so handle it without attempting to read packets. We - // will then proceed with the send loop. - if self.events.is_empty() { - self.sessions.values_mut().for_each(|session| { - session.conn.on_timeout() - }); - - return Ok(()) - } - - // Read incoming UDP packets from the socket and feed them to quiche, - // until there are no more packets to read. - loop { - match self.receive_once() { - Err(Error::Io(e)) if e.kind() == io::ErrorKind::WouldBlock => return Ok(()), - Err(e) => log::error!("{:?}", e), - Ok(_) => (), - } - } - } - - fn receive_once(&mut self) -> Result<()> { - let mut src= [0; MAX_DATAGRAM_SIZE]; - - let (len, from) = self.socket.recv_from(&mut src).unwrap(); - let src = &mut src[..len]; - - let info = quiche::RecvInfo { - to: self.socket.local_addr().unwrap(), - from, - }; - - // Lookup a connection based on the packet's connection ID. If there - // is no connection matching, create a new one. - let pair = match self.accept(src, from).unwrap() { - Some(v) => v, - None => return Ok(()), - }; - - let conn = &mut pair.conn; - - // Process potentially coalesced packets. - conn.recv(src, info).unwrap(); - - // Create a new HTTP/3 connection as soon as the QUIC connection - // is established. - if (conn.is_in_early_data() || conn.is_established()) && pair.session.is_none() { - let session = webtransport::ServerSession::with_transport(conn).unwrap(); - pair.session = Some(session); - } - - // The `poll` can pull out the events that occurred according to the data passed here. - for (_, session) in self.sessions.iter_mut() { - session.poll().unwrap(); - } - - Ok(()) - } - - fn accept(&mut self, src: &mut [u8], from: net::SocketAddr) -> error::Result> { - // Parse the QUIC packet's header. - let hdr = quiche::Header::from_slice(src, quiche::MAX_CONN_ID_LEN).unwrap(); - - let conn_id = ring::hmac::sign(&self.seed, &hdr.dcid); - let conn_id = &conn_id.as_ref()[..quiche::MAX_CONN_ID_LEN]; - let conn_id = conn_id.to_vec().into(); - - if self.sessions.contains_key(&hdr.dcid) { - let pair = self.sessions.get_mut(&hdr.dcid).unwrap(); - return Ok(Some(pair)) - } else if self.sessions.contains_key(&conn_id) { - let pair = self.sessions.get_mut(&conn_id).unwrap(); - return Ok(Some(pair)); - } - - if hdr.ty != quiche::Type::Initial { - return Err(error::Server::UnknownConnectionID.into()) - } - - let mut dst = [0; MAX_DATAGRAM_SIZE]; - - if !quiche::version_is_supported(hdr.version) { - let len = quiche::negotiate_version(&hdr.scid, &hdr.dcid, &mut dst).unwrap(); - let dst= &dst[..len]; - - self.socket.send_to(dst, from).unwrap(); - return Ok(None) - } - - let mut scid = [0; quiche::MAX_CONN_ID_LEN]; - scid.copy_from_slice(&conn_id); - - let scid = quiche::ConnectionId::from_ref(&scid); - - // Token is always present in Initial packets. - let token = hdr.token.as_ref().unwrap(); - - // Do stateless retry if the client didn't send a token. - if token.is_empty() { - let new_token = mint_token(&hdr, &from); - - let len = quiche::retry( - &hdr.scid, - &hdr.dcid, - &scid, - &new_token, - hdr.version, - &mut dst, - ) - .unwrap(); - - let dst= &dst[..len]; - - self.socket.send_to(dst, from).unwrap(); - return Ok(None) - } - - let odcid = validate_token(&from, token); - - // The token was not valid, meaning the retry failed, so - // drop the packet. - if odcid.is_none() { - return Err(error::Server::InvalidToken.into()) - } - - if scid.len() != hdr.dcid.len() { - return Err(error::Server::InvalidConnectionID.into()) - } - - // Reuse the source connection ID we sent in the Retry packet, - // instead of changing it again. - let conn_id= hdr.dcid.clone(); - let local_addr = self.socket.local_addr().unwrap(); - - let conn = - quiche::accept(&conn_id, odcid.as_ref(), local_addr, from, &mut self.quic) - .unwrap(); - - self.sessions.insert( - conn_id.clone(), - Session { - conn, - session: None, - }, - ); - - let pair = self.sessions.get_mut(&conn_id).unwrap(); - Ok(Some(pair)) - } - - fn send(&mut self) -> io::Result<()> { - let mut pkt = [0; MAX_DATAGRAM_SIZE]; - - // Generate outgoing QUIC packets for all active connections and send - // them on the UDP socket, until quiche reports that there are no more - // packets to be sent. - for session in self.sessions.values_mut() { - loop { - let (size , info) = session.conn.send(&mut pkt).unwrap(); - let pkt = &pkt[..size]; - - match self.socket.send_to(&pkt, info.to) { - Err(err) if err.kind() == io::ErrorKind::WouldBlock => break, - Err(err) => return Err(err), - Ok(_) => (), - } - } - } - - Ok(()) - } - - fn cleanup(&mut self) -> io::Result<()> { - // Garbage collect closed connections. - self.sessions.retain(|_, session| !session.conn.is_closed() ); - Ok(()) - } -} - -/// Generate a stateless retry token. -/// -/// The token includes the static string `"quiche"` followed by the IP address -/// of the client and by the original destination connection ID generated by the -/// client. -/// -/// Note that this function is only an example and doesn't do any cryptographic -/// authenticate of the token. *It should not be used in production system*. -fn mint_token(hdr: &quiche::Header, src: &std::net::SocketAddr) -> Vec { - let mut token = Vec::new(); - - token.extend_from_slice(b"quiche"); - - let addr = match src.ip() { - std::net::IpAddr::V4(a) => a.octets().to_vec(), - std::net::IpAddr::V6(a) => a.octets().to_vec(), - }; - - token.extend_from_slice(&addr); - token.extend_from_slice(&hdr.dcid); - - token -} - -/// Validates a stateless retry token. -/// -/// This checks that the ticket includes the `"quiche"` static string, and that -/// the client IP address matches the address stored in the ticket. -/// -/// Note that this function is only an example and doesn't do any cryptographic -/// authenticate of the token. *It should not be used in production system*. -fn validate_token<'a>( - src: &std::net::SocketAddr, token: &'a [u8], -) -> Option> { - if token.len() < 6 { - return None; - } - - if &token[..6] != b"quiche" { - return None; - } - - let token = &token[6..]; - - let addr = match src.ip() { - std::net::IpAddr::V4(a) => a.octets().to_vec(), - std::net::IpAddr::V6(a) => a.octets().to_vec(), - }; - - if token.len() < addr.len() || &token[..addr.len()] != addr.as_slice() { - return None; - } - - Some(quiche::ConnectionId::from_ref(&token[addr.len()..])) -} \ No newline at end of file diff --git a/server/src/session.rs b/server/src/session.rs deleted file mode 100644 index e06bcfa..0000000 --- a/server/src/session.rs +++ /dev/null @@ -1,104 +0,0 @@ -use crate::error; -use error::Result; - -use std::collections::HashMap; -use quiche::h3::webtransport; - -pub struct Session { - pub conn: quiche::Connection, - pub session: Option, -} - -pub type Map = HashMap, Session>; - -impl Session { - // Process any updates to a session. - pub fn poll(&mut self) -> Result<()> { - let session = match &mut self.session { - Some(s) => s, - None => return Ok(()), - }; - - loop { - let event = match session.poll(&mut self.conn) { - Err(webtransport::Error::Done) => return Ok(()), - Err(e) => return Err(e.into()), - Ok(e) => e, - }; - - match event { - webtransport::ServerEvent::ConnectRequest(_req) => { - // you can handle request with - // req.authority() - // req.path() - // and you can validate this request with req.origin() - session.accept_connect_request(&mut self.conn, None).unwrap(); - }, - webtransport::ServerEvent::StreamData(stream_id) => { - let mut buf = vec![0; 10000]; - while let Ok(len) = - session.recv_stream_data(&mut self.conn, stream_id, &mut buf) - { - let stream_data = &buf[0..len]; - - // handle stream_data - if (stream_id & 0x2) == 0 { - // bidirectional stream - // you can send data through this stream. - session - .send_stream_data(&mut self.conn, stream_id, stream_data) - .unwrap(); - } else { - // you cannot send data through client-initiated-unidirectional-stream. - // so, open new server-initiated-unidirectional-stream, and send data - // through it. - let new_stream_id = - session.open_stream(&mut self.conn, false).unwrap(); - session - .send_stream_data(&mut self.conn, new_stream_id, stream_data) - .unwrap(); - } - } - } - - webtransport::ServerEvent::StreamFinished(_stream_id) => { - // A WebTrnasport stream finished, handle it. - } - - webtransport::ServerEvent::Datagram => { - let mut buf = vec![0; 1500]; - while let Ok((in_session, offset, total)) = - session.recv_dgram(&mut self.conn, &mut buf) - { - if in_session { - let dgram = &buf[offset..total]; - dbg!(std::string::String::from_utf8_lossy(dgram)); - // handle this dgram - - // for instance, you can write echo-server like following - session.send_dgram(&mut self.conn, dgram).unwrap(); - } else { - // this dgram is not related to current WebTransport session. ignore. - } - } - } - - webtransport::ServerEvent::SessionReset(_e) => { - // Peer reset session stream, handle it. - } - - webtransport::ServerEvent::SessionFinished => { - // Peer finish session stream, handle it. - } - - webtransport::ServerEvent::SessionGoAway => { - // Peer signalled it is going away, handle it. - } - - webtransport::ServerEvent::Other(_stream_id, _event) => { - // Original h3::Event which is not related to WebTransport. - } - } - } - } -} \ No newline at end of file diff --git a/server/src/transport/app.rs b/server/src/transport/app.rs new file mode 100644 index 0000000..b55ce53 --- /dev/null +++ b/server/src/transport/app.rs @@ -0,0 +1,8 @@ +use std::time; + +use quiche::h3::webtransport; + +pub trait App: Default { + fn poll(&mut self, conn: &mut quiche::Connection, session: &mut webtransport::ServerSession) -> anyhow::Result<()>; + fn timeout(&self) -> Option; +} diff --git a/server/src/transport/connection.rs b/server/src/transport/connection.rs new file mode 100644 index 0000000..2fb12d9 --- /dev/null +++ b/server/src/transport/connection.rs @@ -0,0 +1,15 @@ +use quiche; +use quiche::h3::webtransport; + +use std::collections::hash_map as hmap; + +pub type Id = quiche::ConnectionId<'static>; + +use super::app; + +pub type Map = hmap::HashMap>; +pub struct Connection { + pub quiche: quiche::Connection, + pub session: Option, + pub app: T, +} \ No newline at end of file diff --git a/server/src/transport/mod.rs b/server/src/transport/mod.rs new file mode 100644 index 0000000..c003fa3 --- /dev/null +++ b/server/src/transport/mod.rs @@ -0,0 +1,7 @@ +mod server; +mod session; +mod connection; +mod app; + +pub use app::App; +pub use server::{Config, Server}; \ No newline at end of file diff --git a/server/src/transport/server.rs b/server/src/transport/server.rs new file mode 100644 index 0000000..bbc2a94 --- /dev/null +++ b/server/src/transport/server.rs @@ -0,0 +1,334 @@ +use std::io; + +use quiche::h3::webtransport; + +use super::connection; +use super::app; + +const MAX_DATAGRAM_SIZE: usize = 1350; + +pub struct Server { + // IO stuff + socket: mio::net::UdpSocket, + poll: mio::Poll, + events: mio::Events, + + // QUIC stuff + quic: quiche::Config, + seed: ring::hmac::Key, // connection ID seed + + conns: connection::Map, +} + +pub struct Config { + pub addr: String, + pub cert: String, + pub key: String, +} + +impl Server { + pub fn new(config: Config) -> io::Result { + // Listen on the provided socket address + let addr = config.addr.parse().unwrap(); + let mut socket = mio::net::UdpSocket::bind(addr).unwrap(); + + // Setup the event loop. + let poll = mio::Poll::new().unwrap(); + let events = mio::Events::with_capacity(1024); + + poll.registry().register( + &mut socket, + mio::Token(0), + mio::Interest::READABLE, + ).unwrap(); + + // Generate random values for connection IDs. + let rng = ring::rand::SystemRandom::new(); + let seed = ring::hmac::Key::generate(ring::hmac::HMAC_SHA256, &rng).unwrap(); + + // Create the configuration for the QUIC conns. + let mut quic = quiche::Config::new(quiche::PROTOCOL_VERSION).unwrap(); + quic.load_cert_chain_from_pem_file(&config.cert).unwrap(); + quic.load_priv_key_from_pem_file(&config.key).unwrap(); + quic.set_application_protos(quiche::h3::APPLICATION_PROTOCOL).unwrap(); + quic.set_max_idle_timeout(5000); + quic.set_max_recv_udp_payload_size(MAX_DATAGRAM_SIZE); + quic.set_max_send_udp_payload_size(MAX_DATAGRAM_SIZE); + quic.set_initial_max_data(10_000_000); + quic.set_initial_max_stream_data_bidi_local(1_000_000); + quic.set_initial_max_stream_data_bidi_remote(1_000_000); + quic.set_initial_max_stream_data_uni(1_000_000); + quic.set_initial_max_streams_bidi(100); + quic.set_initial_max_streams_uni(100); + quic.set_disable_active_migration(true); + quic.enable_early_data(); + quic.enable_dgram(true, 65536, 65536); + + let conns = Default::default(); + + Ok(Server { + socket, + poll, + events, + + quic, + seed, + + conns, + }) + } + + pub fn run(&mut self) -> anyhow::Result<()> { + loop { + self.wait()?; + self.receive()?; + self.app()?; + self.send()?; + } + } + + pub fn wait(&mut self) -> anyhow::Result<()> { + // Find the shorter timeout from all the active connections. + // + // TODO: use event loop that properly supports timers + let timeout = self.conns.values().filter_map(|c| { + let timeout = c.quiche.timeout(); + let expires = c.app.timeout(); + + match (timeout, expires) { + (Some(a), Some(b)) => Some(a.min(b)), + (Some(a), None) => Some(a), + (None, Some(b)) => Some(b), + (None, None) => None, + } + }).min(); + + self.poll.poll(&mut self.events, timeout).unwrap(); + + // If the event loop reported no events, it means that the timeout + // has expired, so handle it without attempting to read packets. We + // will then proceed with the send loop. + if self.events.is_empty() { + for conn in self.conns.values_mut() { + conn.quiche.on_timeout(); + } + } + + Ok(()) + } + + // Reads packets from the socket, updating any internal connection state. + fn receive(&mut self) -> anyhow::Result<()> { + let mut src= [0; MAX_DATAGRAM_SIZE]; + + // Try reading any data currently available on the socket. + loop { + let (len, from) = match self.socket.recv_from(&mut src) { + Ok(v) => v, + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => return Ok(()), + Err(e) => return Err(e.into()), + }; + + let src = &mut src[..len]; + + let info = quiche::RecvInfo { + to: self.socket.local_addr().unwrap(), + from, + }; + + // Parse the QUIC packet's header. + let hdr = quiche::Header::from_slice(src, quiche::MAX_CONN_ID_LEN).unwrap(); + + let conn_id = ring::hmac::sign(&self.seed, &hdr.dcid); + let conn_id = &conn_id.as_ref()[..quiche::MAX_CONN_ID_LEN]; + let conn_id = conn_id.to_vec().into(); + + // Check if it's an existing connection. + if let Some(conn) = self.conns.get_mut(&hdr.dcid) { + // initial or handshake traffic. + conn.quiche.recv(src, info)?; + + if conn.session.is_none() && conn.quiche.is_established() { + conn.session = Some(webtransport::ServerSession::with_transport(&mut conn.quiche)?) + } + + continue + } else if let Some(conn) = self.conns.get_mut(&conn_id) { + // 1-RTT traffic. + conn.quiche.recv(src, info)?; + + // TODO is this needed here? + if conn.session.is_none() && conn.quiche.is_established() { + conn.session = Some(webtransport::ServerSession::with_transport(&mut conn.quiche)?) + } + + continue + } + + if hdr.ty != quiche::Type::Initial { + anyhow::bail!("unknown connection ID"); + } + + let mut dst = [0; MAX_DATAGRAM_SIZE]; + + if !quiche::version_is_supported(hdr.version) { + let len = quiche::negotiate_version(&hdr.scid, &hdr.dcid, &mut dst).unwrap(); + let dst= &dst[..len]; + + self.socket.send_to(dst, from).unwrap(); + continue + } + + let mut scid = [0; quiche::MAX_CONN_ID_LEN]; + scid.copy_from_slice(&conn_id); + + let scid = quiche::ConnectionId::from_ref(&scid); + + // Token is always present in Initial packets. + let token = hdr.token.as_ref().unwrap(); + + // Do stateless retry if the client didn't send a token. + if token.is_empty() { + let new_token = mint_token(&hdr, &from); + + let len = quiche::retry( + &hdr.scid, + &hdr.dcid, + &scid, + &new_token, + hdr.version, + &mut dst, + ) + .unwrap(); + + let dst= &dst[..len]; + + self.socket.send_to(dst, from).unwrap(); + continue + } + + let odcid = validate_token(&from, token); + + // The token was not valid, meaning the retry failed, so + // drop the packet. + if odcid.is_none() { + anyhow::bail!("invalid token"); + } + + if scid.len() != hdr.dcid.len() { + anyhow::bail!("invalid connection ID"); + } + + // Reuse the source connection ID we sent in the Retry packet, + // instead of changing it again. + let conn_id= hdr.dcid.clone(); + let local_addr = self.socket.local_addr().unwrap(); + + let mut conn = quiche::accept(&conn_id, odcid.as_ref(), local_addr, from, &mut self.quic)?; + + // Process potentially coalesced packets. + conn.recv(src, info)?; + + let user = connection::Connection{ + quiche: conn, + session: None, + app: T::default(), + }; + + self.conns.insert(user.quiche.source_id().into_owned(), user); + } + } + + pub fn app(&mut self) -> anyhow::Result<()> { + for (_, conn) in &mut self.conns { + if let Some(session) = &mut conn.session { + conn.app.poll(&mut conn.quiche, session)?; + } + } + + Ok(()) + } + + // Generate outgoing QUIC packets for all active connections and send + // them on the UDP socket, until quiche reports that there are no more + // packets to be sent. + pub fn send(&mut self) -> anyhow::Result<()> { + let mut pkt = [0; MAX_DATAGRAM_SIZE]; + + for conn in self.conns.values_mut() { + loop { + let (size , info) = match conn.quiche.send(&mut pkt) { + Ok(v) => v, + Err(quiche::Error::Done) => return Ok(()), + Err(e) => return Err(e.into()), + }; + + let pkt = &pkt[..size]; + + match self.socket.send_to(&pkt, info.to) { + Err(err) if err.kind() == io::ErrorKind::WouldBlock => break, + Err(err) => return Err(err.into()), + Ok(_) => (), + } + } + } + + Ok(()) + } +} + +/// Generate a stateless retry token. +/// +/// The token includes the static string `"quiche"` followed by the IP address +/// of the client and by the original destination connection ID generated by the +/// client. +/// +/// Note that this function is only an example and doesn't do any cryptographic +/// authenticate of the token. *It should not be used in production system*. +fn mint_token(hdr: &quiche::Header, src: &std::net::SocketAddr) -> Vec { + let mut token = Vec::new(); + + token.extend_from_slice(b"quiche"); + + let addr = match src.ip() { + std::net::IpAddr::V4(a) => a.octets().to_vec(), + std::net::IpAddr::V6(a) => a.octets().to_vec(), + }; + + token.extend_from_slice(&addr); + token.extend_from_slice(&hdr.dcid); + + token +} + +/// Validates a stateless retry token. +/// +/// This checks that the ticket includes the `"quiche"` static string, and that +/// the client IP address matches the address stored in the ticket. +/// +/// Note that this function is only an example and doesn't do any cryptographic +/// authenticate of the token. *It should not be used in production system*. +fn validate_token<'a>( + src: &std::net::SocketAddr, token: &'a [u8], +) -> Option> { + if token.len() < 6 { + return None; + } + + if &token[..6] != b"quiche" { + return None; + } + + let token = &token[6..]; + + let addr = match src.ip() { + std::net::IpAddr::V4(a) => a.octets().to_vec(), + std::net::IpAddr::V6(a) => a.octets().to_vec(), + }; + + if token.len() < addr.len() || &token[..addr.len()] != addr.as_slice() { + return None; + } + + Some(quiche::ConnectionId::from_ref(&token[addr.len()..])) +} \ No newline at end of file diff --git a/server/src/transport/session.rs b/server/src/transport/session.rs new file mode 100644 index 0000000..1334704 --- /dev/null +++ b/server/src/transport/session.rs @@ -0,0 +1,252 @@ +use std::collections::hash_map as hmap; +use quiche::h3::webtransport; + +type Session = webtransport::ServerSession; +type Map = hmap::HashMap, Session>; + +/* +impl Session { + pub fn with_transport(conn: &mut quiche::Connection) -> anyhow::Result { + let session = webtransport::ServerSession::with_transport(conn)?; + + Ok(Self{ + session + }) + } + + // Process any updates to a session. + pub fn poll(&mut self) -> anyhow::Result<()> { + log::debug!("poll conn"); + while self.poll_once()? {} + + log::debug!("poll streams"); + self.poll_streams()?; + + Ok(()) + } + + // Process any updates to a session. + pub fn poll_once(&mut self) -> anyhow::Result { + let session = match &mut self.session { + Some(s) => s, + None => return Ok(false), + }; + + let event = match session.poll(&mut self.conn) { + Err(webtransport::Error::Done) => return Ok(false), + Err(e) => return Err(e.into()), + Ok(e) => e, + }; + + match event { + webtransport::ServerEvent::ConnectRequest(req) => { + log::debug!("new connect {:?}", req); + // you can handle request with + // req.authority() + // req.path() + // and you can validate this request with req.origin() + session.accept_connect_request(&mut self.conn, None).unwrap(); + }, + webtransport::ServerEvent::StreamData(stream_id) => { + log::debug!("on stream data {}", stream_id); + + let mut buf = vec![0; 10000]; + while let Ok(len) = + session.recv_stream_data(&mut self.conn, stream_id, &mut buf) + { + let stream_data = &buf[0..len]; + log::debug!("stream data {:?}", stream_data); + +/* + // handle stream_data + if (stream_id & 0x2) == 0 { + // bidirectional stream + // you can send data through this stream. + session + .send_stream_data(&mut self.conn, stream_id, stream_data) + .unwrap(); + } else { + // you cannot send data through client-initiated-unidirectional-stream. + // so, open new server-initiated-unidirectional-stream, and send data + // through it. + let new_stream_id = + session.open_stream(&mut self.conn, false).unwrap(); + session + .send_stream_data(&mut self.conn, new_stream_id, stream_data) + .unwrap(); + } + */ + } + } + + webtransport::ServerEvent::StreamFinished(stream_id) => { + // A WebTrnasport stream finished, handle it. + log::debug!("stream finished {}", stream_id); + } + + webtransport::ServerEvent::Datagram => { + log::debug!("datagram"); + } + + webtransport::ServerEvent::SessionReset(e) => { + log::debug!("session reset {}", e); + // Peer reset session stream, handle it. + } + + webtransport::ServerEvent::SessionFinished => { + log::debug!("session finished"); + // Peer finish session stream, handle it. + } + + webtransport::ServerEvent::SessionGoAway => { + log::debug!("session go away"); + // Peer signalled it is going away, handle it. + } + + webtransport::ServerEvent::Other(stream_id, event) => { + log::debug!("session other: {} {:?}", stream_id, event); + // Original h3::Event which is not related to WebTransport. + } + } + + Ok(true) + } + +/* + fn poll_source(&mut self) -> anyhow::Result<()> { + let media = match &mut self.media { + Some(m) => m, + None => return Ok(()), + }; + + let fragment = match media.next()? { + Some(f) => f, + None => return Ok(()), + }; + + // Get or create a new stream for each unique segment ID. + let stream_id = match self.segments.entry(fragment.segment_id) { + map::Entry::Occupied(e) => e.into_mut(), + map::Entry::Vacant(e) => { + let stream_id = self.start_stream(&fragment)?; + e.insert(stream_id) + }, + }; + + // Get or create a buffered object for each unique stream ID. + let buffered = match self.streams.entry(*stream_id) { + map::Entry::Occupied(e) => e.into_mut(), + map::Entry::Vacant(e) => e.insert(Buffered::new()), + }; + + let session = match &mut self.session { + Some(s) => s, + None => return Ok(()), + }; + + let data = fragment.data.as_slice(); + + match self.conn.stream_writable(*stream_id, data.len()) { + Ok(true) if buffered.len() == 0 => { + session.send_stream_data(&mut self.conn, *stream_id, data)?; + }, + Ok(_) => buffered.push_back(fragment.data), + Err(quiche::Error::Done) => {}, // stream closed? + Err(e) => anyhow::bail!(e), + }; + + Ok(()) + } + + fn start_stream(&mut self, fragment: &source::Fragment) -> anyhow::Result { + let conn = &mut self.conn; + let session = self.session.as_mut().unwrap(); + + let stream_id = session.open_stream(conn, false)?; + + // TODO: conn.stream_priority(stream_id, urgency, incremental) + + let mut message = message::Message::new(); + if fragment.segment_id == 0 { + message.init = Some(message::Init{ + id: "video".to_string(), + }); + } else { + message.segment = Some(message::Segment{ + init: "video".to_string(), + timestamp: fragment.timestamp, + }); + } + + let data= message.serialize()?; + match conn.stream_writable(stream_id, data.len()) { + Ok(true) => { + session.send_stream_data(conn, stream_id, data.as_slice())?; + }, + Ok(false) => { + let mut buffered = Buffered::new(); + buffered.push_back(data); + + self.streams.insert(stream_id, buffered); + }, + Err(quiche::Error::Done) => {}, + Err(e) => anyhow::bail!(e), + }; + + Ok(stream_id) + } +*/ + + fn poll_streams(&mut self) -> anyhow::Result<()> { + // TODO make sure this loops in priority order + for stream_id in self.conn.writable() { + self.poll_stream(stream_id)?; + } + + // Remove any entry buffered values. + self.streams.retain(|_, buffered| buffered.len() > 0 ); + + Ok(()) + } + + pub fn poll_stream(&mut self, stream_id: u64) -> anyhow::Result<()> { + let buffered = match self.streams.get_mut(&stream_id) { + Some(b) => b, + None => return Ok(()), + }; + + let conn = &mut self.conn; + + let session = match &mut self.session { + Some(s) => s, + None => return Ok(()), + }; + + while let Some(data) = buffered.pop_front() { + match conn.stream_writable(stream_id, data.len()) { + Ok(true) => { + session.send_stream_data(conn, stream_id, data.as_slice())?; + }, + Ok(false) => { + buffered.push_front(data); + return Ok(()); + }, + Err(quiche::Error::Done) => {}, + Err(e) => anyhow::bail!(e), + }; + } + + Ok(()) + } + + pub fn timeout(&self) -> Option { + self.conn.timeout() + } + + pub fn on_timeout(&mut self) { + self.conn.on_timeout() + + // custom stuff here + } +} +*/ \ No newline at end of file From c3dd45b7a7716b95ca84942b154c511a04775800 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 24 Apr 2023 11:45:46 -0700 Subject: [PATCH 04/23] Proggers. --- player/src/transport/index.ts | 9 +- player/src/video/decoder.ts | 2 + server/src/lib.rs | 3 +- server/src/main.rs | 32 +--- server/src/media.rs | 108 ------------ server/src/media/mod.rs | 4 +- server/src/media/source.rs | 29 ++-- server/src/{ => session}/message.rs | 2 +- server/src/session/mod.rs | 4 + server/src/session/session.rs | 128 ++++++++++++++ server/src/transport/mod.rs | 1 - server/src/transport/session.rs | 252 ---------------------------- 12 files changed, 160 insertions(+), 414 deletions(-) delete mode 100644 server/src/media.rs rename server/src/{ => session}/message.rs (92%) create mode 100644 server/src/session/mod.rs create mode 100644 server/src/session/session.rs delete mode 100644 server/src/transport/session.rs diff --git a/player/src/transport/index.ts b/player/src/transport/index.ts index 90bb29b..ee58ed1 100644 --- a/player/src/transport/index.ts +++ b/player/src/transport/index.ts @@ -44,7 +44,6 @@ export default class Transport { // Helper function to make creating a promise easier private async connect(props: TransportInit): Promise { - let options: WebTransportOptions = {}; if (props.fingerprint) { options.serverCertificateHashes = [ props.fingerprint ] @@ -98,11 +97,15 @@ export default class Transport { return this.handleInit(r, msg.init as Message.Init) } else if (msg.segment) { return this.handleSegment(r, msg.segment as Message.Segment) + } else { + console.warn("unknown message", msg); } } } async handleInit(stream: Stream.Reader, msg: Message.Init) { + console.log("handle init", msg); + let track = this.tracks.get(msg.id); if (!track) { track = new MP4.InitParser() @@ -118,6 +121,8 @@ export default class Transport { const info = await track.info + console.log(info); + if (info.audioTracks.length + info.videoTracks.length != 1) { throw new Error("expected a single track") } @@ -140,6 +145,8 @@ export default class Transport { } async handleSegment(stream: Stream.Reader, msg: Message.Segment) { + console.log("handle segment", msg); + let track = this.tracks.get(msg.init); if (!track) { track = new MP4.InitParser() diff --git a/player/src/video/decoder.ts b/player/src/video/decoder.ts index 376f99a..61c8113 100644 --- a/player/src/video/decoder.ts +++ b/player/src/video/decoder.ts @@ -53,6 +53,8 @@ export default class Decoder { const input = MP4.New(); input.onSamples = (id: number, user: any, samples: MP4.Sample[]) => { + console.log(samples) + for (let sample of samples) { const timestamp = 1000 * sample.dts / sample.timescale // milliseconds diff --git a/server/src/lib.rs b/server/src/lib.rs index 915e3b1..2c973b8 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -1,2 +1,3 @@ pub mod transport; -//mod media; \ No newline at end of file +pub mod session; +pub mod media; \ No newline at end of file diff --git a/server/src/main.rs b/server/src/main.rs index 9c44d0f..1ffa226 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,7 +1,4 @@ -use quiche::h3::webtransport; -use warp::transport; - -use std::time; +use warp::{session,transport}; use clap::Parser; use env_logger; @@ -26,31 +23,6 @@ struct Cli { media: String, } -#[derive(Default)] -struct Connection { - webtransport: Option, -} - -impl transport::App for Connection { - fn poll(&mut self, conn: &mut quiche::Connection, session: &mut webtransport::ServerSession) -> anyhow::Result<()> { - if !conn.is_established() { - // Wait until the handshake finishes - return Ok(()) - } - - if self.webtransport.is_none() { - self.webtransport = Some(webtransport::ServerSession::with_transport(conn)?) - } - - let webtransport = self.webtransport.as_mut().unwrap(); - - Ok(()) - } - - fn timeout(&self) -> Option { - None - } -} fn main() -> anyhow::Result<()> { env_logger::init(); @@ -62,6 +34,6 @@ fn main() -> anyhow::Result<()> { key: args.key, }; - let mut server = transport::Server::::new(server_config).unwrap(); + let mut server = transport::Server::::new(server_config).unwrap(); server.run() } \ No newline at end of file diff --git a/server/src/media.rs b/server/src/media.rs deleted file mode 100644 index 5e86adb..0000000 --- a/server/src/media.rs +++ /dev/null @@ -1,108 +0,0 @@ -use std::{io,fs}; - -use mp4; -use anyhow; -use bytes; - -use mp4::ReadBox; - -pub struct Source { - pub segments: Vec, -} - -impl Source { - pub fn new(path: &str) -> anyhow::Result { - let f = fs::read(path)?; - let mut bytes = bytes::Bytes::from(f); - - let mut segments = Vec::new(); - let mut current = Segment::new(); - - while bytes.len() > 0 { - // NOTE: Cloning is cheap, since the underlying bytes are reference counted. - let mut reader = io::Cursor::new(bytes.clone()); - - let header = mp4::BoxHeader::read(&mut reader)?; - let size: usize = header.size as usize; - - assert!(size > 0, "empty box"); - - let frag = bytes.split_to(size); - let fragment = Fragment{ bytes: frag }; - - match header.name { - /* - mp4::BoxType::FtypBox => { - } - mp4::BoxType::MoovBox => { - moov = mp4::MoovBox::read_box(&mut reader, size)? - } - mp4::BoxType::EmsgBox => { - let emsg = mp4::EmsgBox::read_box(&mut reader, size)?; - emsgs.push(emsg); - } - mp4::BoxType::MdatBox => { - mp4::skip_box(&mut reader, size)?; - } - */ - mp4::BoxType::MoofBox => { - let moof = mp4::MoofBox::read_box(&mut reader, header.size)?; - if has_keyframe(moof) { - segments.push(current); - current = Segment::new(); - } - } - _ => (), - } - - current.fragments.push(fragment); - } - - segments.push(current); - - Ok(Self { segments }) - } -} - -fn has_keyframe(moof: mp4::MoofBox) -> bool { - for traf in moof.trafs { - // TODO trak default flags if this is None - let default_flags = traf.tfhd.default_sample_flags.unwrap_or_default(); - let trun = traf.trun.expect("missing trun box"); - - for i in 0..trun.sample_count { - let mut flags = match trun.sample_flags.get(i as usize) { - Some(f) => *f, - None => default_flags, - }; - - if i == 0 && trun.first_sample_flags.is_some() { - flags = trun.first_sample_flags.unwrap(); - } - - // https://chromium.googlesource.com/chromium/src/media/+/master/formats/mp4/track_run_iterator.cc#177 - let keyframe = (flags >> 24) & 0x3 == 0x2; // kSampleDependsOnNoOther - let non_sync = (flags >> 16) & 0x1 == 0x1; // kSampleIsNonSyncSample - - if keyframe && non_sync { - return true - } - } - } - - false -} - -pub struct Segment { - pub fragments: Vec, -} - -impl Segment { - fn new() -> Self { - Segment { fragments: Vec::new() } - } -} - -pub struct Fragment { - pub bytes: bytes::Bytes, -} diff --git a/server/src/media/mod.rs b/server/src/media/mod.rs index b5cb700..80a709f 100644 --- a/server/src/media/mod.rs +++ b/server/src/media/mod.rs @@ -1 +1,3 @@ -pub mod source; \ No newline at end of file +mod source; + +pub use source::{Fragment,Source}; \ No newline at end of file diff --git a/server/src/media/source.rs b/server/src/media/source.rs index 7d83b5b..45320b1 100644 --- a/server/src/media/source.rs +++ b/server/src/media/source.rs @@ -9,15 +9,13 @@ use mp4::ReadBox; pub struct Source { reader: io::BufReader, start: time::Instant, - pending: Option, - sequence: u64, } pub struct Fragment { pub data: Vec, - pub segment_id: u64, - pub timestamp: u64, + pub keyframe: bool, + pub timestamp: u64, // only used to simulate a live stream } impl Source { @@ -30,7 +28,6 @@ impl Source { reader, start, pending: None, - sequence: 0, }) } @@ -52,29 +49,23 @@ impl Source { // Read the next full atom. let atom = read_box(&mut self.reader)?; let mut timestamp = 0; + let mut keyframe = false; // Before we return it, let's do some simple parsing. let mut reader = io::Cursor::new(&atom); let header = mp4::BoxHeader::read(&mut reader)?; + if header.name == mp4::BoxType::MoofBox { + let moof = mp4::MoofBox::read_box(&mut reader, header.size)?; - match header.name { - mp4::BoxType::MoofBox => { - let moof = mp4::MoofBox::read_box(&mut reader, header.size)?; - - if has_keyframe(&moof) { - self.sequence += 1 - } - - timestamp = first_timestamp(&moof); - } - _ => (), + keyframe = has_keyframe(&moof); + timestamp = first_timestamp(&moof); } Ok(Fragment { data: atom, - segment_id: self.sequence, - timestamp: timestamp, + keyframe, + timestamp, }) } } @@ -139,7 +130,7 @@ fn has_keyframe(moof: &mp4::MoofBox) -> bool { let keyframe = (flags >> 24) & 0x3 == 0x2; // kSampleDependsOnNoOther let non_sync = (flags >> 16) & 0x1 == 0x1; // kSampleIsNonSyncSample - if keyframe && non_sync { + if keyframe && !non_sync { return true } } diff --git a/server/src/message.rs b/server/src/session/message.rs similarity index 92% rename from server/src/message.rs rename to server/src/session/message.rs index 1421e91..314660a 100644 --- a/server/src/message.rs +++ b/server/src/session/message.rs @@ -31,8 +31,8 @@ impl Message { let size = bytes.len() + 8; let mut out = Vec::with_capacity(size); + out.extend_from_slice(&(size as u32).to_be_bytes()); out.extend_from_slice(b"warp"); - out.extend_from_slice(&size.to_be_bytes()); out.extend_from_slice(bytes); Ok(out) diff --git a/server/src/session/mod.rs b/server/src/session/mod.rs new file mode 100644 index 0000000..031805c --- /dev/null +++ b/server/src/session/mod.rs @@ -0,0 +1,4 @@ +mod session; +mod message; + +pub use session::Session; \ No newline at end of file diff --git a/server/src/session/session.rs b/server/src/session/session.rs new file mode 100644 index 0000000..d694956 --- /dev/null +++ b/server/src/session/session.rs @@ -0,0 +1,128 @@ +use std::time; + +use quiche; +use quiche::h3::webtransport; + +use crate::{media,transport}; +use super::message; + +#[derive(Default)] +pub struct Session { + media: Option, + stream_id: Option, // stream ID of the current segment +} + +impl transport::App for Session { + // Process any updates to a session. + fn poll(&mut self, conn: &mut quiche::Connection, session: &mut webtransport::ServerSession) -> anyhow::Result<()> { + loop { + let event = match session.poll(conn) { + Err(webtransport::Error::Done) => break, + Err(e) => return Err(e.into()), + Ok(e) => e, + }; + + log::debug!("webtransport event: {:?}", event); + + match event { + webtransport::ServerEvent::ConnectRequest(req) => { + log::debug!("new connect {:?}", req); + // you can handle request with + // req.authority() + // req.path() + // and you can validate this request with req.origin() + + // TODO + let media = media::Source::new("../media/fragmented.mp4")?; + self.media = Some(media); + + session.accept_connect_request(conn, None).unwrap(); + }, + webtransport::ServerEvent::StreamData(stream_id) => { + let mut buf = vec![0; 10000]; + while let Ok(len) = + session.recv_stream_data(conn, stream_id, &mut buf) + { + let stream_data = &buf[0..len]; + log::debug!("stream data {:?}", stream_data); + } + } + + _ => {}, + } + } + + self.poll_source(conn, session)?; + + Ok(()) + } + + fn timeout(&self) -> Option { + None + } +} + +impl Session { + fn poll_source(&mut self, conn: &mut quiche::Connection, session: &mut webtransport::ServerSession) -> anyhow::Result<()> { + let media = match &mut self.media { + Some(m) => m, + None => return Ok(()), + }; + + let fragment = match media.next()? { + Some(f) => f, + None => return Ok(()), + }; + + log::debug!("{} {}", fragment.keyframe, fragment.timestamp); + + let mut stream_id = match self.stream_id { + Some(stream_id) => stream_id, + None => { + let mut message = message::Message::new(); + message.init = Some(message::Init{ + id: "video".to_string(), + }); + + let data = message.serialize()?; + + // TODO handle when stream is full + let stream_id = session.open_stream(conn, false)?; + session.send_stream_data(conn, stream_id, data.as_slice())?; + + stream_id + }, + }; + + if fragment.keyframe { + // Close the prior stream. + conn.stream_send(stream_id, &[], true)?; + + let mut message = message::Message::new(); + message.segment = Some(message::Segment{ + init: "video".to_string(), + timestamp: fragment.timestamp, + }); + + let data = message.serialize()?; + + // TODO: conn.stream_priority(stream_id, urgency, incremental) + + // TODO handle when stream is full + stream_id = session.open_stream(conn, false)?; + session.send_stream_data(conn, stream_id, data.as_slice())?; + } + + let data = fragment.data.as_slice(); + + // TODO check if stream is writable + session.send_stream_data(conn, stream_id, data)?; + + log::debug!("wrote {} to {}", std::str::from_utf8(&data[4..8]).unwrap(), stream_id); + + // Save for the next fragment + self.stream_id = Some(stream_id); + + Ok(()) + } +} \ No newline at end of file diff --git a/server/src/transport/mod.rs b/server/src/transport/mod.rs index c003fa3..c7b8325 100644 --- a/server/src/transport/mod.rs +++ b/server/src/transport/mod.rs @@ -1,5 +1,4 @@ mod server; -mod session; mod connection; mod app; diff --git a/server/src/transport/session.rs b/server/src/transport/session.rs deleted file mode 100644 index 1334704..0000000 --- a/server/src/transport/session.rs +++ /dev/null @@ -1,252 +0,0 @@ -use std::collections::hash_map as hmap; -use quiche::h3::webtransport; - -type Session = webtransport::ServerSession; -type Map = hmap::HashMap, Session>; - -/* -impl Session { - pub fn with_transport(conn: &mut quiche::Connection) -> anyhow::Result { - let session = webtransport::ServerSession::with_transport(conn)?; - - Ok(Self{ - session - }) - } - - // Process any updates to a session. - pub fn poll(&mut self) -> anyhow::Result<()> { - log::debug!("poll conn"); - while self.poll_once()? {} - - log::debug!("poll streams"); - self.poll_streams()?; - - Ok(()) - } - - // Process any updates to a session. - pub fn poll_once(&mut self) -> anyhow::Result { - let session = match &mut self.session { - Some(s) => s, - None => return Ok(false), - }; - - let event = match session.poll(&mut self.conn) { - Err(webtransport::Error::Done) => return Ok(false), - Err(e) => return Err(e.into()), - Ok(e) => e, - }; - - match event { - webtransport::ServerEvent::ConnectRequest(req) => { - log::debug!("new connect {:?}", req); - // you can handle request with - // req.authority() - // req.path() - // and you can validate this request with req.origin() - session.accept_connect_request(&mut self.conn, None).unwrap(); - }, - webtransport::ServerEvent::StreamData(stream_id) => { - log::debug!("on stream data {}", stream_id); - - let mut buf = vec![0; 10000]; - while let Ok(len) = - session.recv_stream_data(&mut self.conn, stream_id, &mut buf) - { - let stream_data = &buf[0..len]; - log::debug!("stream data {:?}", stream_data); - -/* - // handle stream_data - if (stream_id & 0x2) == 0 { - // bidirectional stream - // you can send data through this stream. - session - .send_stream_data(&mut self.conn, stream_id, stream_data) - .unwrap(); - } else { - // you cannot send data through client-initiated-unidirectional-stream. - // so, open new server-initiated-unidirectional-stream, and send data - // through it. - let new_stream_id = - session.open_stream(&mut self.conn, false).unwrap(); - session - .send_stream_data(&mut self.conn, new_stream_id, stream_data) - .unwrap(); - } - */ - } - } - - webtransport::ServerEvent::StreamFinished(stream_id) => { - // A WebTrnasport stream finished, handle it. - log::debug!("stream finished {}", stream_id); - } - - webtransport::ServerEvent::Datagram => { - log::debug!("datagram"); - } - - webtransport::ServerEvent::SessionReset(e) => { - log::debug!("session reset {}", e); - // Peer reset session stream, handle it. - } - - webtransport::ServerEvent::SessionFinished => { - log::debug!("session finished"); - // Peer finish session stream, handle it. - } - - webtransport::ServerEvent::SessionGoAway => { - log::debug!("session go away"); - // Peer signalled it is going away, handle it. - } - - webtransport::ServerEvent::Other(stream_id, event) => { - log::debug!("session other: {} {:?}", stream_id, event); - // Original h3::Event which is not related to WebTransport. - } - } - - Ok(true) - } - -/* - fn poll_source(&mut self) -> anyhow::Result<()> { - let media = match &mut self.media { - Some(m) => m, - None => return Ok(()), - }; - - let fragment = match media.next()? { - Some(f) => f, - None => return Ok(()), - }; - - // Get or create a new stream for each unique segment ID. - let stream_id = match self.segments.entry(fragment.segment_id) { - map::Entry::Occupied(e) => e.into_mut(), - map::Entry::Vacant(e) => { - let stream_id = self.start_stream(&fragment)?; - e.insert(stream_id) - }, - }; - - // Get or create a buffered object for each unique stream ID. - let buffered = match self.streams.entry(*stream_id) { - map::Entry::Occupied(e) => e.into_mut(), - map::Entry::Vacant(e) => e.insert(Buffered::new()), - }; - - let session = match &mut self.session { - Some(s) => s, - None => return Ok(()), - }; - - let data = fragment.data.as_slice(); - - match self.conn.stream_writable(*stream_id, data.len()) { - Ok(true) if buffered.len() == 0 => { - session.send_stream_data(&mut self.conn, *stream_id, data)?; - }, - Ok(_) => buffered.push_back(fragment.data), - Err(quiche::Error::Done) => {}, // stream closed? - Err(e) => anyhow::bail!(e), - }; - - Ok(()) - } - - fn start_stream(&mut self, fragment: &source::Fragment) -> anyhow::Result { - let conn = &mut self.conn; - let session = self.session.as_mut().unwrap(); - - let stream_id = session.open_stream(conn, false)?; - - // TODO: conn.stream_priority(stream_id, urgency, incremental) - - let mut message = message::Message::new(); - if fragment.segment_id == 0 { - message.init = Some(message::Init{ - id: "video".to_string(), - }); - } else { - message.segment = Some(message::Segment{ - init: "video".to_string(), - timestamp: fragment.timestamp, - }); - } - - let data= message.serialize()?; - match conn.stream_writable(stream_id, data.len()) { - Ok(true) => { - session.send_stream_data(conn, stream_id, data.as_slice())?; - }, - Ok(false) => { - let mut buffered = Buffered::new(); - buffered.push_back(data); - - self.streams.insert(stream_id, buffered); - }, - Err(quiche::Error::Done) => {}, - Err(e) => anyhow::bail!(e), - }; - - Ok(stream_id) - } -*/ - - fn poll_streams(&mut self) -> anyhow::Result<()> { - // TODO make sure this loops in priority order - for stream_id in self.conn.writable() { - self.poll_stream(stream_id)?; - } - - // Remove any entry buffered values. - self.streams.retain(|_, buffered| buffered.len() > 0 ); - - Ok(()) - } - - pub fn poll_stream(&mut self, stream_id: u64) -> anyhow::Result<()> { - let buffered = match self.streams.get_mut(&stream_id) { - Some(b) => b, - None => return Ok(()), - }; - - let conn = &mut self.conn; - - let session = match &mut self.session { - Some(s) => s, - None => return Ok(()), - }; - - while let Some(data) = buffered.pop_front() { - match conn.stream_writable(stream_id, data.len()) { - Ok(true) => { - session.send_stream_data(conn, stream_id, data.as_slice())?; - }, - Ok(false) => { - buffered.push_front(data); - return Ok(()); - }, - Err(quiche::Error::Done) => {}, - Err(e) => anyhow::bail!(e), - }; - } - - Ok(()) - } - - pub fn timeout(&self) -> Option { - self.conn.timeout() - } - - pub fn on_timeout(&mut self) { - self.conn.on_timeout() - - // custom stuff here - } -} -*/ \ No newline at end of file From 15c3352d80627348d303cea7e5d315e81f844897 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 24 Apr 2023 13:07:06 -0700 Subject: [PATCH 05/23] Pretty gud. --- server/src/media/source.rs | 63 +++++++++++++++++++++-------------- server/src/session/message.rs | 1 - server/src/session/session.rs | 26 +++++++++++---- 3 files changed, 57 insertions(+), 33 deletions(-) diff --git a/server/src/media/source.rs b/server/src/media/source.rs index 45320b1..6566f6a 100644 --- a/server/src/media/source.rs +++ b/server/src/media/source.rs @@ -8,14 +8,17 @@ use mp4::ReadBox; pub struct Source { reader: io::BufReader, - start: time::Instant, pending: Option, + + start: time::Instant, + timescale: Option, } pub struct Fragment { + pub typ: mp4::BoxType, pub data: Vec, pub keyframe: bool, - pub timestamp: u64, // only used to simulate a live stream + pub timestamp: Option, // only used to simulate a live stream } impl Source { @@ -28,46 +31,66 @@ impl Source { reader, start, pending: None, + timescale: None, }) } pub fn next(&mut self) -> anyhow::Result> { - let pending = match self.pending.take() { - Some(f) => f, - None => self.next_inner()?, + if self.pending.is_none() { + self.pending = Some(self.next_inner()?); }; - if pending.timestamp > 0 && pending.timestamp < self.start.elapsed().as_millis() as u64 { - self.pending = Some(pending); + if self.timeout().is_some() { return Ok(None) } - Ok(Some(pending)) + let pending = self.pending.take(); + Ok(pending) } fn next_inner(&mut self) -> anyhow::Result { // Read the next full atom. let atom = read_box(&mut self.reader)?; - let mut timestamp = 0; + let mut timestamp = None; let mut keyframe = false; // Before we return it, let's do some simple parsing. let mut reader = io::Cursor::new(&atom); let header = mp4::BoxHeader::read(&mut reader)?; - if header.name == mp4::BoxType::MoofBox { - let moof = mp4::MoofBox::read_box(&mut reader, header.size)?; + match header.name { + mp4::BoxType::MoovBox => { + // We need to parse the moov to get the timescale. + let moov = mp4::MoovBox::read_box(&mut reader, header.size)?; + self.timescale = Some(moov.traks[0].mdia.mdhd.timescale.into()); + }, + mp4::BoxType::MoofBox => { + let moof = mp4::MoofBox::read_box(&mut reader, header.size)?; - keyframe = has_keyframe(&moof); - timestamp = first_timestamp(&moof); + keyframe = has_keyframe(&moof); + timestamp = first_timestamp(&moof); + } + _ => {}, } Ok(Fragment { + typ: header.name, data: atom, keyframe, timestamp, }) } + + // Simulate a live stream by sleeping until the next timestamp in the media. + pub fn timeout(&self) -> Option { + let timestamp = self.pending.as_ref()?.timestamp?; + let timescale = self.timescale?; + + let delay = time::Duration::from_millis(1000 * timestamp / timescale); + let elapsed = self.start.elapsed(); + + delay.checked_sub(elapsed) + } } // Read a full MP4 atom into a vector. @@ -139,16 +162,6 @@ fn has_keyframe(moof: &mp4::MoofBox) -> bool { false } -fn first_timestamp(moof: &mp4::MoofBox) -> u64 { - let traf = match moof.trafs.first() { - Some(t) => t, - None => return 0, - }; - - let tfdt = match &traf.tfdt { - Some(t) => t, - None => return 0, - }; - - tfdt.base_media_decode_time +fn first_timestamp(moof: &mp4::MoofBox) -> Option { + Some(moof.trafs.first()?.tfdt.as_ref()?.base_media_decode_time) } diff --git a/server/src/session/message.rs b/server/src/session/message.rs index 314660a..8718b81 100644 --- a/server/src/session/message.rs +++ b/server/src/session/message.rs @@ -14,7 +14,6 @@ pub struct Init { #[derive(Serialize, Deserialize)] pub struct Segment { pub init: String, - pub timestamp: u64, } impl Message { diff --git a/server/src/session/session.rs b/server/src/session/session.rs index d694956..9edd481 100644 --- a/server/src/session/session.rs +++ b/server/src/session/session.rs @@ -6,10 +6,13 @@ use quiche::h3::webtransport; use crate::{media,transport}; use super::message; +use mp4; + #[derive(Default)] pub struct Session { media: Option, stream_id: Option, // stream ID of the current segment + styp: Option>, } impl transport::App for Session { @@ -58,7 +61,7 @@ impl transport::App for Session { } fn timeout(&self) -> Option { - None + self.media.as_ref().and_then(|m| m.timeout()) } } @@ -74,8 +77,6 @@ impl Session { None => return Ok(()), }; - log::debug!("{} {}", fragment.keyframe, fragment.timestamp); - let mut stream_id = match self.stream_id { Some(stream_id) => stream_id, None => { @@ -101,7 +102,6 @@ impl Session { let mut message = message::Message::new(); message.segment = Some(message::Segment{ init: "video".to_string(), - timestamp: fragment.timestamp, }); let data = message.serialize()?; @@ -111,18 +111,30 @@ impl Session { // TODO handle when stream is full stream_id = session.open_stream(conn, false)?; session.send_stream_data(conn, stream_id, data.as_slice())?; + + let styp = self.styp.as_ref().expect("missing ftyp mox"); + session.send_stream_data(conn, stream_id, &styp)?; } let data = fragment.data.as_slice(); // TODO check if stream is writable - session.send_stream_data(conn, stream_id, data)?; - - log::debug!("wrote {} to {}", std::str::from_utf8(&data[4..8]).unwrap(), stream_id); + let size = session.send_stream_data(conn, stream_id, data)?; + if size < data.len() { + anyhow::bail!("partial write: {} < {}", size, data.len()); + } // Save for the next fragment self.stream_id = Some(stream_id); + // Save the ftyp fragment but modify it to be a styp for furture segments. + if fragment.typ == mp4::BoxType::FtypBox { + let mut data = fragment.data; + data[4] = b's'; // ftyp to styp + + self.styp = Some(data); + } + Ok(()) } } \ No newline at end of file From 2b1a3adecc4464a38abba39c17d12d51ca2fa0f5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 27 Apr 2023 13:21:16 -0700 Subject: [PATCH 06/23] Video woooorks. --- media/generate | 9 +++- player/src/video/decoder.ts | 4 +- server/src/session/session.rs | 86 ++++++++++++++++----------------- server/src/transport/mod.rs | 4 +- server/src/transport/streams.rs | 79 ++++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 49 deletions(-) create mode 100644 server/src/transport/streams.rs diff --git a/media/generate b/media/generate index b387ac9..922d2bc 100755 --- a/media/generate +++ b/media/generate @@ -1,6 +1,13 @@ #!/bin/bash +cd "$(dirname "$0")" + +# empty_moov: Uses moof fragments instead of one giant moov/mdat pair. +# frag_every_frame: Creates a moof for each frame. +# separate_moof: Splits audio and video into separate moof flags. +# omit_tfhd_offset: Removes absolute byte offsets so we can fragment. + ffmpeg -i source.mp4 \ + -movflags empty_moov+frag_every_frame+separate_moof+omit_tfhd_offset \ -c:v copy \ -an \ - -movflags frag_every_frame+empty_moov \ fragmented.mp4 diff --git a/player/src/video/decoder.ts b/player/src/video/decoder.ts index 61c8113..582dcd2 100644 --- a/player/src/video/decoder.ts +++ b/player/src/video/decoder.ts @@ -53,8 +53,6 @@ export default class Decoder { const input = MP4.New(); input.onSamples = (id: number, user: any, samples: MP4.Sample[]) => { - console.log(samples) - for (let sample of samples) { const timestamp = 1000 * sample.dts / sample.timescale // milliseconds @@ -92,7 +90,7 @@ export default class Decoder { for (let raw of init.raw) { raw.fileStart = offset - input.appendBuffer(raw) + offset = input.appendBuffer(raw) } const stream = new Stream.Reader(msg.reader, msg.buffer) diff --git a/server/src/session/session.rs b/server/src/session/session.rs index 9edd481..d01dd00 100644 --- a/server/src/session/session.rs +++ b/server/src/session/session.rs @@ -6,13 +6,12 @@ use quiche::h3::webtransport; use crate::{media,transport}; use super::message; -use mp4; - #[derive(Default)] pub struct Session { media: Option, stream_id: Option, // stream ID of the current segment - styp: Option>, + + streams: transport::Streams, // An easy way of buffering stream data. } impl transport::App for Session { @@ -55,6 +54,10 @@ impl transport::App for Session { } } + // Send any pending stream data. + self.streams.poll(conn)?; + + // Fetch the next media fragment, possibly queuing up stream data. self.poll_source(conn, session)?; Ok(()) @@ -67,19 +70,46 @@ impl transport::App for Session { impl Session { fn poll_source(&mut self, conn: &mut quiche::Connection, session: &mut webtransport::ServerSession) -> anyhow::Result<()> { + // Get the media source once the connection is established. let media = match &mut self.media { Some(m) => m, None => return Ok(()), }; + // Get the next media fragment. let fragment = match media.next()? { Some(f) => f, None => return Ok(()), }; - let mut stream_id = match self.stream_id { - Some(stream_id) => stream_id, + // Check if we have already created a stream for this fragment. + let stream_id = match self.stream_id { + Some(old_stream_id) if fragment.keyframe => { + // This is the start of a new segment. + + // Close the prior stream. + self.streams.send(conn, old_stream_id, &[], true)?; + + // Encode a JSON header indicating this is the video track. + let mut message = message::Message::new(); + message.segment = Some(message::Segment{ + init: "video".to_string(), + }); + + // Open a new stream. + let stream_id = session.open_stream(conn, false)?; + // TODO: conn.stream_priority(stream_id, urgency, incremental) + + // Write the header. + let data = message.serialize()?; + self.streams.send(conn, stream_id, &data, false)?; + + stream_id + }, None => { + // This is the start of an init segment. + + // Create a JSON header. let mut message = message::Message::new(); message.init = Some(message::Init{ id: "video".to_string(), @@ -87,54 +117,22 @@ impl Session { let data = message.serialize()?; - // TODO handle when stream is full + // Create a new stream and write the header. let stream_id = session.open_stream(conn, false)?; - session.send_stream_data(conn, stream_id, data.as_slice())?; + self.streams.send(conn, stream_id, data.as_slice(), false)?; stream_id - }, + } + Some(stream_id) => stream_id, // Continuation of init or segment }; - if fragment.keyframe { - // Close the prior stream. - conn.stream_send(stream_id, &[], true)?; - - let mut message = message::Message::new(); - message.segment = Some(message::Segment{ - init: "video".to_string(), - }); - - let data = message.serialize()?; - - // TODO: conn.stream_priority(stream_id, urgency, incremental) - - // TODO handle when stream is full - stream_id = session.open_stream(conn, false)?; - session.send_stream_data(conn, stream_id, data.as_slice())?; - - let styp = self.styp.as_ref().expect("missing ftyp mox"); - session.send_stream_data(conn, stream_id, &styp)?; - } - + // Write the current fragment. let data = fragment.data.as_slice(); + self.streams.send(conn, stream_id, data, false)?; - // TODO check if stream is writable - let size = session.send_stream_data(conn, stream_id, data)?; - if size < data.len() { - anyhow::bail!("partial write: {} < {}", size, data.len()); - } - - // Save for the next fragment + // Save the stream ID for the next fragment. self.stream_id = Some(stream_id); - // Save the ftyp fragment but modify it to be a styp for furture segments. - if fragment.typ == mp4::BoxType::FtypBox { - let mut data = fragment.data; - data[4] = b's'; // ftyp to styp - - self.styp = Some(data); - } - Ok(()) } } \ No newline at end of file diff --git a/server/src/transport/mod.rs b/server/src/transport/mod.rs index c7b8325..f8d86e8 100644 --- a/server/src/transport/mod.rs +++ b/server/src/transport/mod.rs @@ -1,6 +1,8 @@ mod server; mod connection; mod app; +mod streams; pub use app::App; -pub use server::{Config, Server}; \ No newline at end of file +pub use server::{Config, Server}; +pub use streams::Streams; \ No newline at end of file diff --git a/server/src/transport/streams.rs b/server/src/transport/streams.rs new file mode 100644 index 0000000..6b78233 --- /dev/null +++ b/server/src/transport/streams.rs @@ -0,0 +1,79 @@ +use std::collections::hash_map as hmap; +use std::collections::VecDeque; + +use quiche; +use anyhow; + +#[derive(Default)] +pub struct Streams { + lookup: hmap::HashMap, +} + +#[derive(Default)] +struct State { + buffer: VecDeque, + fin: bool, +} + +impl Streams { + pub fn send(&mut self, conn: &mut quiche::Connection, id: u64, buf: &[u8], fin: bool) -> anyhow::Result<()> { + match self.lookup.entry(id) { + hmap::Entry::Occupied(mut entry) => { + // Add to the existing buffer. + let state = entry.get_mut(); + state.buffer.extend(buf); + state.fin |= fin; + }, + hmap::Entry::Vacant(entry) => { + let size = conn.stream_send(id, buf, fin)?; + + if size < buf.len() { + // Short write, save the rest for later. + let mut buffer = VecDeque::with_capacity(buf.len()); + buffer.extend(&buf[size..]); + + entry.insert(State{buffer, fin}); + } + }, + }; + + Ok(()) + } + + pub fn poll(&mut self, conn: &mut quiche::Connection) -> anyhow::Result<()> { + 'outer: for id in conn.writable() { + // Check if there's any buffered data for this stream. + let mut entry = match self.lookup.entry(id) { + hmap::Entry::Occupied(entry) => entry, + hmap::Entry::Vacant(_) => continue, + }; + + let state = entry.get_mut(); + + // Keep reading from the buffer until it's empty. + while state.buffer.len() > 0 { + // VecDeque is a ring buffer, so we can't write the whole thing at once. + let parts = state.buffer.as_slices(); + + let size = conn.stream_send(id, parts.0, false)?; + if size == 0 { + // No more space available for this stream. + continue 'outer + } + + // Remove the bytes that were written. + state.buffer.drain(..size); + } + + if state.fin { + // Write the stream done signal. + conn.stream_send(id, &[], true)?; + } + + // We can remove the value from the lookup once we've flushed everything. + entry.remove(); + } + + Ok(()) + } +} \ No newline at end of file From e578b757e59a074f38ea4f7c243f767bb8a35374 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 2 May 2023 11:05:05 -0700 Subject: [PATCH 07/23] wip --- server/Cargo.lock | 3 +- server/Cargo.toml | 2 +- server/src/media/source.rs | 160 +++++++++++++++++++++++++-------- server/src/transport/server.rs | 12 ++- 4 files changed, 135 insertions(+), 42 deletions(-) diff --git a/server/Cargo.lock b/server/Cargo.lock index 00d51d5..b470484 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -329,8 +329,7 @@ dependencies = [ [[package]] name = "mp4" version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "509348cba250e7b852a875100a2ddce7a36ee3abf881a681c756670c1774264d" +source = "git+https://github.com/kixelated/mp4-rust.git?branch=trexs#efefcc47353f477518bff01493785ae0daa8efd4" dependencies = [ "byteorder", "bytes", diff --git a/server/Cargo.toml b/server/Cargo.toml index b397086..660ab93 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -13,6 +13,6 @@ mio = { version = "0.8", features = ["net", "os-poll"] } env_logger = "0.9.3" ring = "0.16" anyhow = "1.0.70" -mp4 = "0.13.0" +mp4 = { git = "https://github.com/kixelated/mp4-rust.git", branch = "trexs" } serde = "1.0.160" serde_json = "1.0" \ No newline at end of file diff --git a/server/src/media/source.rs b/server/src/media/source.rs index 6566f6a..0a7d019 100644 --- a/server/src/media/source.rs +++ b/server/src/media/source.rs @@ -1,24 +1,46 @@ use std::{io,fs,time}; use io::Read; +use std::collections::{VecDeque}; + +use std::io::Write; use mp4; use anyhow; -use mp4::ReadBox; +use mp4::{ReadBox,WriteBox}; pub struct Source { + // We read the file once, in order, and don't seek backwards. reader: io::BufReader, - pending: Option, + // Any fragments parsed and ready to be returned by next(). + fragments: VecDeque, + + // The timestamp when the broadcast "started", so we can sleep to simulate a live stream. start: time::Instant, - timescale: Option, + + // The raw ftyp box, which we need duplicate for each track, but we don't know how many tracks exist yet. + ftyp: Vec, + + // The parsed moov box, so we can look up track information later. + moov: Option, } pub struct Fragment { + // The track ID for the fragment. + pub track: u32, + + // The type of the fragment. pub typ: mp4::BoxType, + + // The data of the fragment. pub data: Vec, + + // Whether this fragment is a keyframe. pub keyframe: bool, - pub timestamp: Option, // only used to simulate a live stream + + // The timestamp of the fragment, in milliseconds, to simulate a live stream. + pub timestamp: Option } impl Source { @@ -30,61 +52,123 @@ impl Source { Ok(Self{ reader, start, - pending: None, - timescale: None, + fragments: VecDeque::new(), + ftyp: Vec::new(), + moov: None, }) } pub fn next(&mut self) -> anyhow::Result> { - if self.pending.is_none() { - self.pending = Some(self.next_inner()?); + if self.fragments.is_empty() { + self.parse()?; }; if self.timeout().is_some() { return Ok(None) } - let pending = self.pending.take(); - Ok(pending) + Ok(self.fragments.pop_front()) } - fn next_inner(&mut self) -> anyhow::Result { - // Read the next full atom. - let atom = read_box(&mut self.reader)?; - let mut timestamp = None; - let mut keyframe = false; + fn parse(&mut self) -> anyhow::Result<()> { + loop { + // Read the next full atom. + let atom = read_box(&mut self.reader)?; - // Before we return it, let's do some simple parsing. - let mut reader = io::Cursor::new(&atom); - let header = mp4::BoxHeader::read(&mut reader)?; + // Before we return it, let's do some simple parsing. + let mut reader = io::Cursor::new(&atom); + let header = mp4::BoxHeader::read(&mut reader)?; - match header.name { - mp4::BoxType::MoovBox => { - // We need to parse the moov to get the timescale. - let moov = mp4::MoovBox::read_box(&mut reader, header.size)?; - self.timescale = Some(moov.traks[0].mdia.mdhd.timescale.into()); - }, - mp4::BoxType::MoofBox => { - let moof = mp4::MoofBox::read_box(&mut reader, header.size)?; + match header.name { + mp4::BoxType::FtypBox => { + // Don't return anything until we know the total number of tracks. + // To be honest, I didn't expect the borrow checker to allow this, but it does! + self.ftyp = atom; + }, + mp4::BoxType::MoovBox => { + // We need to split the moov based on the tracks. + let moov = mp4::MoovBox::read_box(&mut reader, header.size)?; - keyframe = has_keyframe(&moof); - timestamp = first_timestamp(&moof); + for trak in &moov.traks { + let track_id = trak.tkhd.track_id; + + // Push the styp atom for each track. + self.fragments.push_back(Fragment { + track: track_id, + typ: mp4::BoxType::FtypBox, + data: self.ftyp.clone(), + keyframe: false, + timestamp: None, + }); + + // Unfortunately, we need to create a brand new moov atom for each track. + // We remove every box for other track IDs. + let mut toov = moov.clone(); + toov.traks.retain(|t| t.tkhd.track_id == track_id); + toov.mvex.as_mut().expect("missing mvex").trexs.retain(|f| f.track_id == track_id); + + // Marshal the box. + let mut toov_data = Vec::new(); + toov.write_box(&mut toov_data)?; + + let mut file = std::fs::File::create(format!("track{}.mp4", track_id))?; + file.write_all(toov_data.as_slice()); + + self.fragments.push_back(Fragment { + track: track_id, + typ: mp4::BoxType::MoovBox, + data: toov_data, + keyframe: false, + timestamp: None, + }); + } + + self.moov = Some(moov); + }, + mp4::BoxType::MoofBox => { + let moof = mp4::MoofBox::read_box(&mut reader, header.size)?; + + if moof.trafs.len() != 1 { + // We can't split the mdat atom, so this is impossible to support + anyhow::bail!("multiple tracks per moof atom") + } + + self.fragments.push_back(Fragment{ + track: moof.trafs[0].tfhd.track_id, + typ: mp4::BoxType::MoofBox, + data: atom, + keyframe: has_keyframe(&moof), + timestamp: first_timestamp(&moof), + }) + }, + mp4::BoxType::MdatBox => { + let moof = self.fragments.back().expect("no atom before mdat"); + assert!(moof.typ == mp4::BoxType::MoofBox, "no moof before mdat"); + + self.fragments.push_back(Fragment{ + track: moof.track, + typ: mp4::BoxType::MoofBox, + data: atom, + keyframe: false, + timestamp: None, + }); + + // We have some media data, return so we can start sending it. + return Ok(()) + }, + _ => anyhow::bail!("unknown top-level atom: {:?}", header.name), } - _ => {}, } - - Ok(Fragment { - typ: header.name, - data: atom, - keyframe, - timestamp, - }) } // Simulate a live stream by sleeping until the next timestamp in the media. pub fn timeout(&self) -> Option { - let timestamp = self.pending.as_ref()?.timestamp?; - let timescale = self.timescale?; + let next = self.fragments.front()?; + let timestamp = next.timestamp?; + + // Find the timescale for the track. + let track = self.moov.as_ref()?.traks.iter().find(|t| t.tkhd.track_id == next.track)?; + let timescale = track.mdia.mdhd.timescale as u64; let delay = time::Duration::from_millis(1000 * timestamp / timescale); let elapsed = self.start.elapsed(); diff --git a/server/src/transport/server.rs b/server/src/transport/server.rs index bbc2a94..6c3eebc 100644 --- a/server/src/transport/server.rs +++ b/server/src/transport/server.rs @@ -84,6 +84,7 @@ impl Server { self.receive()?; self.app()?; self.send()?; + self.cleanup(); } } @@ -242,7 +243,11 @@ impl Server { pub fn app(&mut self) -> anyhow::Result<()> { for (_, conn) in &mut self.conns { if let Some(session) = &mut conn.session { - conn.app.poll(&mut conn.quiche, session)?; + if let Err(e) = conn.app.poll(&mut conn.quiche, session) { + // Close the connection on any application error + let reason = format!("app error: {:?}", e); + conn.quiche.close(true, 0xff, reason.as_bytes()).ok(); + } } } @@ -275,6 +280,11 @@ impl Server { Ok(()) } + + pub fn cleanup(&mut self) { + // Garbage collect closed connections. + self.conns.retain(|_, ref mut c| !c.quiche.is_closed() ); + } } /// Generate a stateless retry token. From b5b7ffedfa64653895071bb06f85a23d18c949cb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 2 May 2023 11:05:21 -0700 Subject: [PATCH 08/23] cargo fmt --- server/src/lib.rs | 4 +- server/src/main.rs | 6 +-- server/src/media/mod.rs | 2 +- server/src/media/source.rs | 53 ++++++++++++-------- server/src/session/message.rs | 2 +- server/src/session/mod.rs | 4 +- server/src/session/session.rs | 30 +++++++----- server/src/transport/app.rs | 6 ++- server/src/transport/connection.rs | 2 +- server/src/transport/mod.rs | 6 +-- server/src/transport/server.rs | 78 +++++++++++++++++------------- server/src/transport/streams.rs | 20 +++++--- 12 files changed, 126 insertions(+), 87 deletions(-) diff --git a/server/src/lib.rs b/server/src/lib.rs index 2c973b8..6715979 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -1,3 +1,3 @@ -pub mod transport; +pub mod media; pub mod session; -pub mod media; \ No newline at end of file +pub mod transport; diff --git a/server/src/main.rs b/server/src/main.rs index 1ffa226..a6a1119 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,4 +1,4 @@ -use warp::{session,transport}; +use warp::{session, transport}; use clap::Parser; use env_logger; @@ -28,7 +28,7 @@ fn main() -> anyhow::Result<()> { let args = Cli::parse(); - let server_config = transport::Config{ + let server_config = transport::Config { addr: args.addr, cert: args.cert, key: args.key, @@ -36,4 +36,4 @@ fn main() -> anyhow::Result<()> { let mut server = transport::Server::::new(server_config).unwrap(); server.run() -} \ No newline at end of file +} diff --git a/server/src/media/mod.rs b/server/src/media/mod.rs index 80a709f..e7ec5eb 100644 --- a/server/src/media/mod.rs +++ b/server/src/media/mod.rs @@ -1,3 +1,3 @@ mod source; -pub use source::{Fragment,Source}; \ No newline at end of file +pub use source::{Fragment, Source}; diff --git a/server/src/media/source.rs b/server/src/media/source.rs index 0a7d019..6ce9dcb 100644 --- a/server/src/media/source.rs +++ b/server/src/media/source.rs @@ -1,13 +1,13 @@ -use std::{io,fs,time}; use io::Read; -use std::collections::{VecDeque}; +use std::collections::VecDeque; +use std::{fs, io, time}; use std::io::Write; -use mp4; use anyhow; +use mp4; -use mp4::{ReadBox,WriteBox}; +use mp4::{ReadBox, WriteBox}; pub struct Source { // We read the file once, in order, and don't seek backwards. @@ -40,7 +40,7 @@ pub struct Fragment { pub keyframe: bool, // The timestamp of the fragment, in milliseconds, to simulate a live stream. - pub timestamp: Option + pub timestamp: Option, } impl Source { @@ -49,7 +49,7 @@ impl Source { let reader = io::BufReader::new(f); let start = time::Instant::now(); - Ok(Self{ + Ok(Self { reader, start, fragments: VecDeque::new(), @@ -64,7 +64,7 @@ impl Source { }; if self.timeout().is_some() { - return Ok(None) + return Ok(None); } Ok(self.fragments.pop_front()) @@ -84,7 +84,7 @@ impl Source { // Don't return anything until we know the total number of tracks. // To be honest, I didn't expect the borrow checker to allow this, but it does! self.ftyp = atom; - }, + } mp4::BoxType::MoovBox => { // We need to split the moov based on the tracks. let moov = mp4::MoovBox::read_box(&mut reader, header.size)?; @@ -105,7 +105,11 @@ impl Source { // We remove every box for other track IDs. let mut toov = moov.clone(); toov.traks.retain(|t| t.tkhd.track_id == track_id); - toov.mvex.as_mut().expect("missing mvex").trexs.retain(|f| f.track_id == track_id); + toov.mvex + .as_mut() + .expect("missing mvex") + .trexs + .retain(|f| f.track_id == track_id); // Marshal the box. let mut toov_data = Vec::new(); @@ -124,7 +128,7 @@ impl Source { } self.moov = Some(moov); - }, + } mp4::BoxType::MoofBox => { let moof = mp4::MoofBox::read_box(&mut reader, header.size)?; @@ -133,19 +137,19 @@ impl Source { anyhow::bail!("multiple tracks per moof atom") } - self.fragments.push_back(Fragment{ + self.fragments.push_back(Fragment { track: moof.trafs[0].tfhd.track_id, typ: mp4::BoxType::MoofBox, data: atom, keyframe: has_keyframe(&moof), timestamp: first_timestamp(&moof), }) - }, + } mp4::BoxType::MdatBox => { let moof = self.fragments.back().expect("no atom before mdat"); assert!(moof.typ == mp4::BoxType::MoofBox, "no moof before mdat"); - self.fragments.push_back(Fragment{ + self.fragments.push_back(Fragment { track: moof.track, typ: mp4::BoxType::MoofBox, data: atom, @@ -154,8 +158,8 @@ impl Source { }); // We have some media data, return so we can start sending it. - return Ok(()) - }, + return Ok(()); + } _ => anyhow::bail!("unknown top-level atom: {:?}", header.name), } } @@ -167,7 +171,12 @@ impl Source { let timestamp = next.timestamp?; // Find the timescale for the track. - let track = self.moov.as_ref()?.traks.iter().find(|t| t.tkhd.track_id == next.track)?; + let track = self + .moov + .as_ref()? + .traks + .iter() + .find(|t| t.tkhd.track_id == next.track)?; let timescale = track.mdia.mdhd.timescale as u64; let delay = time::Duration::from_millis(1000 * timestamp / timescale); @@ -195,17 +204,21 @@ fn read_box(reader: &mut R) -> anyhow::Result> { 1 => { reader.read_exact(&mut buf)?; let size_large = u64::from_be_bytes(buf); - anyhow::ensure!(size_large >= 16, "impossible extended box size: {}", size_large); + anyhow::ensure!( + size_large >= 16, + "impossible extended box size: {}", + size_large + ); reader.take(size_large - 16) - }, + } 2..=7 => { anyhow::bail!("impossible box size: {}", size) } // Otherwise read based on the size. - size => reader.take(size - 8) + size => reader.take(size - 8), }; // Append to the vector and return it. @@ -238,7 +251,7 @@ fn has_keyframe(moof: &mp4::MoofBox) -> bool { let non_sync = (flags >> 16) & 0x1 == 0x1; // kSampleIsNonSyncSample if keyframe && !non_sync { - return true + return true; } } } diff --git a/server/src/session/message.rs b/server/src/session/message.rs index 8718b81..74243e8 100644 --- a/server/src/session/message.rs +++ b/server/src/session/message.rs @@ -36,4 +36,4 @@ impl Message { Ok(out) } -} \ No newline at end of file +} diff --git a/server/src/session/mod.rs b/server/src/session/mod.rs index 031805c..592f525 100644 --- a/server/src/session/mod.rs +++ b/server/src/session/mod.rs @@ -1,4 +1,4 @@ -mod session; mod message; +mod session; -pub use session::Session; \ No newline at end of file +pub use session::Session; diff --git a/server/src/session/session.rs b/server/src/session/session.rs index d01dd00..8b48521 100644 --- a/server/src/session/session.rs +++ b/server/src/session/session.rs @@ -3,8 +3,8 @@ use std::time; use quiche; use quiche::h3::webtransport; -use crate::{media,transport}; use super::message; +use crate::{media, transport}; #[derive(Default)] pub struct Session { @@ -16,7 +16,11 @@ pub struct Session { impl transport::App for Session { // Process any updates to a session. - fn poll(&mut self, conn: &mut quiche::Connection, session: &mut webtransport::ServerSession) -> anyhow::Result<()> { + fn poll( + &mut self, + conn: &mut quiche::Connection, + session: &mut webtransport::ServerSession, + ) -> anyhow::Result<()> { loop { let event = match session.poll(conn) { Err(webtransport::Error::Done) => break, @@ -39,18 +43,16 @@ impl transport::App for Session { self.media = Some(media); session.accept_connect_request(conn, None).unwrap(); - }, + } webtransport::ServerEvent::StreamData(stream_id) => { let mut buf = vec![0; 10000]; - while let Ok(len) = - session.recv_stream_data(conn, stream_id, &mut buf) - { + while let Ok(len) = session.recv_stream_data(conn, stream_id, &mut buf) { let stream_data = &buf[0..len]; log::debug!("stream data {:?}", stream_data); } } - _ => {}, + _ => {} } } @@ -69,7 +71,11 @@ impl transport::App for Session { } impl Session { - fn poll_source(&mut self, conn: &mut quiche::Connection, session: &mut webtransport::ServerSession) -> anyhow::Result<()> { + fn poll_source( + &mut self, + conn: &mut quiche::Connection, + session: &mut webtransport::ServerSession, + ) -> anyhow::Result<()> { // Get the media source once the connection is established. let media = match &mut self.media { Some(m) => m, @@ -92,7 +98,7 @@ impl Session { // Encode a JSON header indicating this is the video track. let mut message = message::Message::new(); - message.segment = Some(message::Segment{ + message.segment = Some(message::Segment { init: "video".to_string(), }); @@ -105,13 +111,13 @@ impl Session { self.streams.send(conn, stream_id, &data, false)?; stream_id - }, + } None => { // This is the start of an init segment. // Create a JSON header. let mut message = message::Message::new(); - message.init = Some(message::Init{ + message.init = Some(message::Init { id: "video".to_string(), }); @@ -135,4 +141,4 @@ impl Session { Ok(()) } -} \ No newline at end of file +} diff --git a/server/src/transport/app.rs b/server/src/transport/app.rs index b55ce53..9024448 100644 --- a/server/src/transport/app.rs +++ b/server/src/transport/app.rs @@ -3,6 +3,10 @@ use std::time; use quiche::h3::webtransport; pub trait App: Default { - fn poll(&mut self, conn: &mut quiche::Connection, session: &mut webtransport::ServerSession) -> anyhow::Result<()>; + fn poll( + &mut self, + conn: &mut quiche::Connection, + session: &mut webtransport::ServerSession, + ) -> anyhow::Result<()>; fn timeout(&self) -> Option; } diff --git a/server/src/transport/connection.rs b/server/src/transport/connection.rs index 2fb12d9..e9766dc 100644 --- a/server/src/transport/connection.rs +++ b/server/src/transport/connection.rs @@ -12,4 +12,4 @@ pub struct Connection { pub quiche: quiche::Connection, pub session: Option, pub app: T, -} \ No newline at end of file +} diff --git a/server/src/transport/mod.rs b/server/src/transport/mod.rs index f8d86e8..60ef79d 100644 --- a/server/src/transport/mod.rs +++ b/server/src/transport/mod.rs @@ -1,8 +1,8 @@ -mod server; -mod connection; mod app; +mod connection; +mod server; mod streams; pub use app::App; pub use server::{Config, Server}; -pub use streams::Streams; \ No newline at end of file +pub use streams::Streams; diff --git a/server/src/transport/server.rs b/server/src/transport/server.rs index 6c3eebc..cc9cbe1 100644 --- a/server/src/transport/server.rs +++ b/server/src/transport/server.rs @@ -2,8 +2,8 @@ use std::io; use quiche::h3::webtransport; -use super::connection; use super::app; +use super::connection; const MAX_DATAGRAM_SIZE: usize = 1350; @@ -36,11 +36,9 @@ impl Server { let poll = mio::Poll::new().unwrap(); let events = mio::Events::with_capacity(1024); - poll.registry().register( - &mut socket, - mio::Token(0), - mio::Interest::READABLE, - ).unwrap(); + poll.registry() + .register(&mut socket, mio::Token(0), mio::Interest::READABLE) + .unwrap(); // Generate random values for connection IDs. let rng = ring::rand::SystemRandom::new(); @@ -50,7 +48,8 @@ impl Server { let mut quic = quiche::Config::new(quiche::PROTOCOL_VERSION).unwrap(); quic.load_cert_chain_from_pem_file(&config.cert).unwrap(); quic.load_priv_key_from_pem_file(&config.key).unwrap(); - quic.set_application_protos(quiche::h3::APPLICATION_PROTOCOL).unwrap(); + quic.set_application_protos(quiche::h3::APPLICATION_PROTOCOL) + .unwrap(); quic.set_max_idle_timeout(5000); quic.set_max_recv_udp_payload_size(MAX_DATAGRAM_SIZE); quic.set_max_send_udp_payload_size(MAX_DATAGRAM_SIZE); @@ -92,17 +91,21 @@ impl Server { // Find the shorter timeout from all the active connections. // // TODO: use event loop that properly supports timers - let timeout = self.conns.values().filter_map(|c| { - let timeout = c.quiche.timeout(); - let expires = c.app.timeout(); + let timeout = self + .conns + .values() + .filter_map(|c| { + let timeout = c.quiche.timeout(); + let expires = c.app.timeout(); - match (timeout, expires) { - (Some(a), Some(b)) => Some(a.min(b)), - (Some(a), None) => Some(a), - (None, Some(b)) => Some(b), - (None, None) => None, - } - }).min(); + match (timeout, expires) { + (Some(a), Some(b)) => Some(a.min(b)), + (Some(a), None) => Some(a), + (None, Some(b)) => Some(b), + (None, None) => None, + } + }) + .min(); self.poll.poll(&mut self.events, timeout).unwrap(); @@ -120,7 +123,7 @@ impl Server { // Reads packets from the socket, updating any internal connection state. fn receive(&mut self) -> anyhow::Result<()> { - let mut src= [0; MAX_DATAGRAM_SIZE]; + let mut src = [0; MAX_DATAGRAM_SIZE]; // Try reading any data currently available on the socket. loop { @@ -150,20 +153,24 @@ impl Server { conn.quiche.recv(src, info)?; if conn.session.is_none() && conn.quiche.is_established() { - conn.session = Some(webtransport::ServerSession::with_transport(&mut conn.quiche)?) + conn.session = Some(webtransport::ServerSession::with_transport( + &mut conn.quiche, + )?) } - continue + continue; } else if let Some(conn) = self.conns.get_mut(&conn_id) { // 1-RTT traffic. conn.quiche.recv(src, info)?; // TODO is this needed here? if conn.session.is_none() && conn.quiche.is_established() { - conn.session = Some(webtransport::ServerSession::with_transport(&mut conn.quiche)?) + conn.session = Some(webtransport::ServerSession::with_transport( + &mut conn.quiche, + )?) } - continue + continue; } if hdr.ty != quiche::Type::Initial { @@ -174,10 +181,10 @@ impl Server { if !quiche::version_is_supported(hdr.version) { let len = quiche::negotiate_version(&hdr.scid, &hdr.dcid, &mut dst).unwrap(); - let dst= &dst[..len]; + let dst = &dst[..len]; self.socket.send_to(dst, from).unwrap(); - continue + continue; } let mut scid = [0; quiche::MAX_CONN_ID_LEN]; @@ -202,10 +209,10 @@ impl Server { ) .unwrap(); - let dst= &dst[..len]; + let dst = &dst[..len]; self.socket.send_to(dst, from).unwrap(); - continue + continue; } let odcid = validate_token(&from, token); @@ -222,21 +229,23 @@ impl Server { // Reuse the source connection ID we sent in the Retry packet, // instead of changing it again. - let conn_id= hdr.dcid.clone(); + let conn_id = hdr.dcid.clone(); let local_addr = self.socket.local_addr().unwrap(); - let mut conn = quiche::accept(&conn_id, odcid.as_ref(), local_addr, from, &mut self.quic)?; + let mut conn = + quiche::accept(&conn_id, odcid.as_ref(), local_addr, from, &mut self.quic)?; // Process potentially coalesced packets. conn.recv(src, info)?; - let user = connection::Connection{ + let user = connection::Connection { quiche: conn, session: None, app: T::default(), }; - self.conns.insert(user.quiche.source_id().into_owned(), user); + self.conns + .insert(user.quiche.source_id().into_owned(), user); } } @@ -262,7 +271,7 @@ impl Server { for conn in self.conns.values_mut() { loop { - let (size , info) = match conn.quiche.send(&mut pkt) { + let (size, info) = match conn.quiche.send(&mut pkt) { Ok(v) => v, Err(quiche::Error::Done) => return Ok(()), Err(e) => return Err(e.into()), @@ -283,7 +292,7 @@ impl Server { pub fn cleanup(&mut self) { // Garbage collect closed connections. - self.conns.retain(|_, ref mut c| !c.quiche.is_closed() ); + self.conns.retain(|_, ref mut c| !c.quiche.is_closed()); } } @@ -319,7 +328,8 @@ fn mint_token(hdr: &quiche::Header, src: &std::net::SocketAddr) -> Vec { /// Note that this function is only an example and doesn't do any cryptographic /// authenticate of the token. *It should not be used in production system*. fn validate_token<'a>( - src: &std::net::SocketAddr, token: &'a [u8], + src: &std::net::SocketAddr, + token: &'a [u8], ) -> Option> { if token.len() < 6 { return None; @@ -341,4 +351,4 @@ fn validate_token<'a>( } Some(quiche::ConnectionId::from_ref(&token[addr.len()..])) -} \ No newline at end of file +} diff --git a/server/src/transport/streams.rs b/server/src/transport/streams.rs index 6b78233..a69717c 100644 --- a/server/src/transport/streams.rs +++ b/server/src/transport/streams.rs @@ -1,8 +1,8 @@ use std::collections::hash_map as hmap; use std::collections::VecDeque; -use quiche; use anyhow; +use quiche; #[derive(Default)] pub struct Streams { @@ -16,14 +16,20 @@ struct State { } impl Streams { - pub fn send(&mut self, conn: &mut quiche::Connection, id: u64, buf: &[u8], fin: bool) -> anyhow::Result<()> { + pub fn send( + &mut self, + conn: &mut quiche::Connection, + id: u64, + buf: &[u8], + fin: bool, + ) -> anyhow::Result<()> { match self.lookup.entry(id) { hmap::Entry::Occupied(mut entry) => { // Add to the existing buffer. let state = entry.get_mut(); state.buffer.extend(buf); state.fin |= fin; - }, + } hmap::Entry::Vacant(entry) => { let size = conn.stream_send(id, buf, fin)?; @@ -32,9 +38,9 @@ impl Streams { let mut buffer = VecDeque::with_capacity(buf.len()); buffer.extend(&buf[size..]); - entry.insert(State{buffer, fin}); + entry.insert(State { buffer, fin }); } - }, + } }; Ok(()) @@ -58,7 +64,7 @@ impl Streams { let size = conn.stream_send(id, parts.0, false)?; if size == 0 { // No more space available for this stream. - continue 'outer + continue 'outer; } // Remove the bytes that were written. @@ -76,4 +82,4 @@ impl Streams { Ok(()) } -} \ No newline at end of file +} From e9663accc6b3e88c34928ffda8cddf50ceccc2ce Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 2 May 2023 11:09:36 -0700 Subject: [PATCH 09/23] cargo clippy --- server/src/main.rs | 1 - server/src/media/source.rs | 4 +- server/src/session/mod.rs | 145 +++++++++++++++++++++++++++++++- server/src/session/session.rs | 143 ------------------------------- server/src/transport/server.rs | 4 +- server/src/transport/streams.rs | 2 +- 6 files changed, 148 insertions(+), 151 deletions(-) diff --git a/server/src/main.rs b/server/src/main.rs index a6a1119..3d4d422 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,7 +1,6 @@ use warp::{session, transport}; use clap::Parser; -use env_logger; /// Search for a pattern in a file and display the lines that contain it. #[derive(Parser)] diff --git a/server/src/media/source.rs b/server/src/media/source.rs index 6ce9dcb..38cfb97 100644 --- a/server/src/media/source.rs +++ b/server/src/media/source.rs @@ -58,7 +58,7 @@ impl Source { }) } - pub fn next(&mut self) -> anyhow::Result> { + pub fn get(&mut self) -> anyhow::Result> { if self.fragments.is_empty() { self.parse()?; }; @@ -116,7 +116,7 @@ impl Source { toov.write_box(&mut toov_data)?; let mut file = std::fs::File::create(format!("track{}.mp4", track_id))?; - file.write_all(toov_data.as_slice()); + file.write_all(toov_data.as_slice())?; self.fragments.push_back(Fragment { track: track_id, diff --git a/server/src/session/mod.rs b/server/src/session/mod.rs index 592f525..47e0074 100644 --- a/server/src/session/mod.rs +++ b/server/src/session/mod.rs @@ -1,4 +1,145 @@ mod message; -mod session; -pub use session::Session; +use std::time; + +use quiche; +use quiche::h3::webtransport; + +use crate::{media, transport}; + +#[derive(Default)] +pub struct Session { + media: Option, + stream_id: Option, // stream ID of the current segment + + streams: transport::Streams, // An easy way of buffering stream data. +} + +impl transport::App for Session { + // Process any updates to a session. + fn poll( + &mut self, + conn: &mut quiche::Connection, + session: &mut webtransport::ServerSession, + ) -> anyhow::Result<()> { + loop { + let event = match session.poll(conn) { + Err(webtransport::Error::Done) => break, + Err(e) => return Err(e.into()), + Ok(e) => e, + }; + + log::debug!("webtransport event: {:?}", event); + + match event { + webtransport::ServerEvent::ConnectRequest(req) => { + log::debug!("new connect {:?}", req); + // you can handle request with + // req.authority() + // req.path() + // and you can validate this request with req.origin() + + // TODO + let media = media::Source::new("../media/fragmented.mp4")?; + self.media = Some(media); + + session.accept_connect_request(conn, None).unwrap(); + } + webtransport::ServerEvent::StreamData(stream_id) => { + let mut buf = vec![0; 10000]; + while let Ok(len) = session.recv_stream_data(conn, stream_id, &mut buf) { + let stream_data = &buf[0..len]; + log::debug!("stream data {:?}", stream_data); + } + } + + _ => {} + } + } + + // Send any pending stream data. + self.streams.poll(conn)?; + + // Fetch the next media fragment, possibly queuing up stream data. + self.poll_source(conn, session)?; + + Ok(()) + } + + fn timeout(&self) -> Option { + self.media.as_ref().and_then(|m| m.timeout()) + } +} + +impl Session { + fn poll_source( + &mut self, + conn: &mut quiche::Connection, + session: &mut webtransport::ServerSession, + ) -> anyhow::Result<()> { + // Get the media source once the connection is established. + let media = match &mut self.media { + Some(m) => m, + None => return Ok(()), + }; + + // Get the next media fragment. + let fragment = match media.get()? { + Some(f) => f, + None => return Ok(()), + }; + + // Check if we have already created a stream for this fragment. + let stream_id = match self.stream_id { + Some(old_stream_id) if fragment.keyframe => { + // This is the start of a new segment. + + // Close the prior stream. + self.streams.send(conn, old_stream_id, &[], true)?; + + // Encode a JSON header indicating this is the video track. + let mut message = message::Message::new(); + message.segment = Some(message::Segment { + init: "video".to_string(), + }); + + // Open a new stream. + let stream_id = session.open_stream(conn, false)?; + // TODO: conn.stream_priority(stream_id, urgency, incremental) + + // Write the header. + let data = message.serialize()?; + self.streams.send(conn, stream_id, &data, false)?; + + stream_id + } + None => { + // This is the start of an init segment. + + // Create a JSON header. + let mut message = message::Message::new(); + message.init = Some(message::Init { + id: "video".to_string(), + }); + + let data = message.serialize()?; + + // Create a new stream and write the header. + let stream_id = session.open_stream(conn, false)?; + self.streams.send(conn, stream_id, data.as_slice(), false)?; + + stream_id + } + Some(stream_id) => stream_id, // Continuation of init or segment + }; + + // Write the current fragment. + let data = fragment.data.as_slice(); + self.streams.send(conn, stream_id, data, false)?; + + // Save the stream ID for the next fragment. + self.stream_id = Some(stream_id); + + Ok(()) + } +} diff --git a/server/src/session/session.rs b/server/src/session/session.rs index 8b48521..8b13789 100644 --- a/server/src/session/session.rs +++ b/server/src/session/session.rs @@ -1,144 +1 @@ -use std::time; -use quiche; -use quiche::h3::webtransport; - -use super::message; -use crate::{media, transport}; - -#[derive(Default)] -pub struct Session { - media: Option, - stream_id: Option, // stream ID of the current segment - - streams: transport::Streams, // An easy way of buffering stream data. -} - -impl transport::App for Session { - // Process any updates to a session. - fn poll( - &mut self, - conn: &mut quiche::Connection, - session: &mut webtransport::ServerSession, - ) -> anyhow::Result<()> { - loop { - let event = match session.poll(conn) { - Err(webtransport::Error::Done) => break, - Err(e) => return Err(e.into()), - Ok(e) => e, - }; - - log::debug!("webtransport event: {:?}", event); - - match event { - webtransport::ServerEvent::ConnectRequest(req) => { - log::debug!("new connect {:?}", req); - // you can handle request with - // req.authority() - // req.path() - // and you can validate this request with req.origin() - - // TODO - let media = media::Source::new("../media/fragmented.mp4")?; - self.media = Some(media); - - session.accept_connect_request(conn, None).unwrap(); - } - webtransport::ServerEvent::StreamData(stream_id) => { - let mut buf = vec![0; 10000]; - while let Ok(len) = session.recv_stream_data(conn, stream_id, &mut buf) { - let stream_data = &buf[0..len]; - log::debug!("stream data {:?}", stream_data); - } - } - - _ => {} - } - } - - // Send any pending stream data. - self.streams.poll(conn)?; - - // Fetch the next media fragment, possibly queuing up stream data. - self.poll_source(conn, session)?; - - Ok(()) - } - - fn timeout(&self) -> Option { - self.media.as_ref().and_then(|m| m.timeout()) - } -} - -impl Session { - fn poll_source( - &mut self, - conn: &mut quiche::Connection, - session: &mut webtransport::ServerSession, - ) -> anyhow::Result<()> { - // Get the media source once the connection is established. - let media = match &mut self.media { - Some(m) => m, - None => return Ok(()), - }; - - // Get the next media fragment. - let fragment = match media.next()? { - Some(f) => f, - None => return Ok(()), - }; - - // Check if we have already created a stream for this fragment. - let stream_id = match self.stream_id { - Some(old_stream_id) if fragment.keyframe => { - // This is the start of a new segment. - - // Close the prior stream. - self.streams.send(conn, old_stream_id, &[], true)?; - - // Encode a JSON header indicating this is the video track. - let mut message = message::Message::new(); - message.segment = Some(message::Segment { - init: "video".to_string(), - }); - - // Open a new stream. - let stream_id = session.open_stream(conn, false)?; - // TODO: conn.stream_priority(stream_id, urgency, incremental) - - // Write the header. - let data = message.serialize()?; - self.streams.send(conn, stream_id, &data, false)?; - - stream_id - } - None => { - // This is the start of an init segment. - - // Create a JSON header. - let mut message = message::Message::new(); - message.init = Some(message::Init { - id: "video".to_string(), - }); - - let data = message.serialize()?; - - // Create a new stream and write the header. - let stream_id = session.open_stream(conn, false)?; - self.streams.send(conn, stream_id, data.as_slice(), false)?; - - stream_id - } - Some(stream_id) => stream_id, // Continuation of init or segment - }; - - // Write the current fragment. - let data = fragment.data.as_slice(); - self.streams.send(conn, stream_id, data, false)?; - - // Save the stream ID for the next fragment. - self.stream_id = Some(stream_id); - - Ok(()) - } -} diff --git a/server/src/transport/server.rs b/server/src/transport/server.rs index cc9cbe1..202d526 100644 --- a/server/src/transport/server.rs +++ b/server/src/transport/server.rs @@ -250,7 +250,7 @@ impl Server { } pub fn app(&mut self) -> anyhow::Result<()> { - for (_, conn) in &mut self.conns { + for conn in self.conns.values_mut() { if let Some(session) = &mut conn.session { if let Err(e) = conn.app.poll(&mut conn.quiche, session) { // Close the connection on any application error @@ -279,7 +279,7 @@ impl Server { let pkt = &pkt[..size]; - match self.socket.send_to(&pkt, info.to) { + match self.socket.send_to(pkt, info.to) { Err(err) if err.kind() == io::ErrorKind::WouldBlock => break, Err(err) => return Err(err.into()), Ok(_) => (), diff --git a/server/src/transport/streams.rs b/server/src/transport/streams.rs index a69717c..215731b 100644 --- a/server/src/transport/streams.rs +++ b/server/src/transport/streams.rs @@ -57,7 +57,7 @@ impl Streams { let state = entry.get_mut(); // Keep reading from the buffer until it's empty. - while state.buffer.len() > 0 { + while !state.buffer.is_empty() { // VecDeque is a ring buffer, so we can't write the whole thing at once. let parts = state.buffer.as_slices(); From d7237c4926db49d3588d87bf26a08eba57938e56 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 4 May 2023 19:43:43 -0700 Subject: [PATCH 10/23] Send INIT as a single message. Much simpler on both the client and server side. --- player/src/audio/decoder.ts | 121 ------------------- player/src/audio/index.ts | 4 +- player/src/audio/renderer.ts | 85 ------------- player/src/audio/worker.ts | 26 ---- player/src/media/decoder.ts | 160 +++++++++++++++++++++++++ player/src/media/index.ts | 82 +++++++++++++ player/src/{audio => media}/message.ts | 27 +++-- player/src/media/renderer.ts | 138 +++++++++++++++++++++ player/src/{audio => media}/ring.ts | 0 player/src/{video => media}/worker.ts | 4 +- player/src/{audio => media}/worklet.ts | 2 +- player/src/mp4/index.ts | 21 ++-- player/src/mp4/init.ts | 14 +-- player/src/mp4/mp4box.d.ts | 12 +- player/src/mp4/rename.ts | 12 ++ player/src/player/index.ts | 22 +--- player/src/transport/index.ts | 96 ++------------- player/src/video/decoder.ts | 127 -------------------- player/src/video/index.ts | 27 ----- player/src/video/message.ts | 17 --- player/src/video/renderer.ts | 91 -------------- server/Cargo.lock | 1 - server/Cargo.toml | 2 +- server/src/media/mod.rs | 2 +- server/src/media/source.rs | 149 +++++++++-------------- server/src/session/message.rs | 6 +- server/src/session/mod.rs | 79 ++++++------ server/src/session/session.rs | 1 - 28 files changed, 546 insertions(+), 782 deletions(-) delete mode 100644 player/src/audio/decoder.ts delete mode 100644 player/src/audio/renderer.ts delete mode 100644 player/src/audio/worker.ts create mode 100644 player/src/media/decoder.ts create mode 100644 player/src/media/index.ts rename player/src/{audio => media}/message.ts (50%) create mode 100644 player/src/media/renderer.ts rename player/src/{audio => media}/ring.ts (100%) rename player/src/{video => media}/worker.ts (86%) rename player/src/{audio => media}/worklet.ts (93%) create mode 100644 player/src/mp4/rename.ts delete mode 100644 player/src/video/decoder.ts delete mode 100644 player/src/video/index.ts delete mode 100644 player/src/video/message.ts delete mode 100644 player/src/video/renderer.ts delete mode 100644 server/src/session/session.rs diff --git a/player/src/audio/decoder.ts b/player/src/audio/decoder.ts deleted file mode 100644 index d23bdc3..0000000 --- a/player/src/audio/decoder.ts +++ /dev/null @@ -1,121 +0,0 @@ -import * as Message from "./message"; -import * as MP4 from "../mp4" -import * as Stream from "../stream" -import * as Util from "../util" - -import Renderer from "./renderer" - -export default class Decoder { - // Store the init message for each track - tracks: Map>; - decoder: AudioDecoder; // TODO one per track - sync: Message.Sync; - - constructor(config: Message.Config, renderer: Renderer) { - this.tracks = new Map(); - - this.decoder = new AudioDecoder({ - output: renderer.emit.bind(renderer), - error: console.warn, - }); - } - - init(msg: Message.Init) { - let defer = this.tracks.get(msg.track); - if (!defer) { - defer = new Util.Deferred() - this.tracks.set(msg.track, defer) - } - - if (msg.info.audioTracks.length != 1 || msg.info.videoTracks.length != 0) { - throw new Error("Expected a single audio track") - } - - const track = msg.info.audioTracks[0] - const audio = track.audio - - defer.resolve(msg) - } - - async decode(msg: Message.Segment) { - let track = this.tracks.get(msg.track); - if (!track) { - track = new Util.Deferred() - this.tracks.set(msg.track, track) - } - - // Wait for the init segment to be fully received and parsed - const init = await track.promise; - const audio = init.info.audioTracks[0] - - if (this.decoder.state == "unconfigured") { - this.decoder.configure({ - codec: audio.codec, - numberOfChannels: audio.audio.channel_count, - sampleRate: audio.audio.sample_rate, - }) - } - - const input = MP4.New(); - - input.onSamples = (id: number, user: any, samples: MP4.Sample[]) => { - for (let sample of samples) { - // Convert to microseconds - const timestamp = 1000 * 1000 * sample.dts / sample.timescale - const duration = 1000 * 1000 * sample.duration / sample.timescale - - // This assumes that timescale == sample rate - this.decoder.decode(new EncodedAudioChunk({ - type: sample.is_sync ? "key" : "delta", - data: sample.data, - duration: duration, - timestamp: timestamp, - })) - } - } - - input.onReady = (info: any) => { - input.setExtractionOptions(info.tracks[0].id, {}, { nbSamples: 1 }); - input.start(); - } - - // MP4box requires us to reparse the init segment unfortunately - let offset = 0; - - for (let raw of init.raw) { - raw.fileStart = offset - input.appendBuffer(raw) - } - - const stream = new Stream.Reader(msg.reader, msg.buffer) - - /* TODO I'm not actually sure why this code doesn't work; something trips up the MP4 parser - while (1) { - const data = await stream.read() - if (!data) break - - input.appendBuffer(data) - input.flush() - } - */ - - // One day I'll figure it out; until then read one top-level atom at a time - while (!await stream.done()) { - const raw = await stream.peek(4) - const size = new DataView(raw.buffer, raw.byteOffset, raw.byteLength).getUint32(0) - const atom = await stream.bytes(size) - - // Make a copy of the atom because mp4box only accepts an ArrayBuffer unfortunately - let box = new Uint8Array(atom.byteLength); - box.set(atom) - - // and for some reason we need to modify the underlying ArrayBuffer with offset - let buffer = box.buffer as MP4.ArrayBuffer - buffer.fileStart = offset - - // Parse the data - offset = input.appendBuffer(buffer) - input.flush() - } - } -} \ No newline at end of file diff --git a/player/src/audio/index.ts b/player/src/audio/index.ts index 725bc11..7076cd3 100644 --- a/player/src/audio/index.ts +++ b/player/src/audio/index.ts @@ -1,8 +1,8 @@ import * as Message from "./message" -import Renderer from "./renderer" +import Renderer from "../media/audio" import Decoder from "./decoder" -import { RingInit } from "./ring" +import { RingInit } from "../media/ring" // Abstracts the Worker and Worklet into a simpler API // This class must be created on the main thread due to AudioContext. diff --git a/player/src/audio/renderer.ts b/player/src/audio/renderer.ts deleted file mode 100644 index 7c5ec9e..0000000 --- a/player/src/audio/renderer.ts +++ /dev/null @@ -1,85 +0,0 @@ -import * as Message from "./message" -import { Ring } from "./ring" - -export default class Renderer { - ring: Ring; - queue: Array; - sync?: DOMHighResTimeStamp - running: number; - - constructor(config: Message.Config) { - this.ring = new Ring(config.ring) - this.queue = []; - this.running = 0 - } - - emit(frame: AudioData) { - if (!this.sync) { - // Save the frame as the sync point - this.sync = 1000 * performance.now() - frame.timestamp - } - - // Insert the frame into the queue sorted by timestamp. - if (this.queue.length > 0 && this.queue[this.queue.length-1].timestamp <= frame.timestamp) { - // Fast path because we normally append to the end. - this.queue.push(frame) - } else { - // Do a full binary search - let low = 0 - let high = this.queue.length; - - while (low < high) { - var mid = (low + high) >>> 1; - if (this.queue[mid].timestamp < frame.timestamp) low = mid + 1; - else high = mid; - } - - this.queue.splice(low, 0, frame) - } - - if (!this.running) { - // Wait for the next animation frame - this.running = self.requestAnimationFrame(this.render.bind(this)) - } - } - - render() { - // Determine the target timestamp. - const target = 1000 * performance.now() - this.sync! - - // Check if we should skip some frames - while (this.queue.length) { - const next = this.queue[0] - if (next.timestamp >= target) { - break - } - - console.warn("dropping audio") - - this.queue.shift() - next.close() - } - - // Push as many as we can to the ring buffer. - while (this.queue.length) { - let frame = this.queue[0] - let ok = this.ring.write(frame) - if (!ok) { - break - } - - frame.close() - this.queue.shift() - } - - if (this.queue.length) { - this.running = self.requestAnimationFrame(this.render.bind(this)) - } else { - this.running = 0 - } - } - - play(play: Message.Play) { - this.ring.reset() - } -} \ No newline at end of file diff --git a/player/src/audio/worker.ts b/player/src/audio/worker.ts deleted file mode 100644 index 7ed9003..0000000 --- a/player/src/audio/worker.ts +++ /dev/null @@ -1,26 +0,0 @@ -import Decoder from "./decoder" -import Renderer from "./renderer" - -import * as Message from "./message" - -let decoder: Decoder -let renderer: Renderer; - -self.addEventListener('message', (e: MessageEvent) => { - if (e.data.config) { - renderer = new Renderer(e.data.config) - decoder = new Decoder(e.data.config, renderer) - } - - if (e.data.init) { - decoder.init(e.data.init) - } - - if (e.data.segment) { - decoder.decode(e.data.segment) - } - - if (e.data.play) { - renderer.play(e.data.play) - } -}) \ No newline at end of file diff --git a/player/src/media/decoder.ts b/player/src/media/decoder.ts new file mode 100644 index 0000000..ca2f639 --- /dev/null +++ b/player/src/media/decoder.ts @@ -0,0 +1,160 @@ +import * as Message from "./message"; +import * as MP4 from "../mp4" +import * as Stream from "../stream" +import * as Util from "../util" + +import Renderer from "./renderer" + +export default class Decoder { + init: MP4.InitParser; + decoders: Map; + renderer: Renderer; + + constructor(renderer: Renderer) { + this.init = new MP4.InitParser(); + this.decoders = new Map(); + this.renderer = renderer; + } + + async receiveInit(msg: Message.Init) { + let stream = new Stream.Reader(msg.reader, msg.buffer); + while (1) { + const data = await stream.read() + if (!data) break + + this.init.push(data) + } + + // TODO make sure the init segment is fully received + } + + async receiveSegment(msg: Message.Segment) { + // Wait for the init segment to be fully received and parsed + const info = await this.init.info + const input = MP4.New(); + + input.onSamples = this.onSamples.bind(this); + input.onReady = (info: any) => { + // Extract all of the tracks, because we don't know if it's audio or video. + for (let track of info.tracks) { + input.setExtractionOptions(track.id, track, { nbSamples: 1 }); + } + + input.start(); + } + + // MP4box requires us to reparse the init segment unfortunately + let offset = 0; + + for (let raw of this.init.raw) { + raw.fileStart = offset + offset = input.appendBuffer(raw) + } + + const stream = new Stream.Reader(msg.reader, msg.buffer) + + // For whatever reason, mp4box doesn't work until you read an atom at a time. + while (!await stream.done()) { + const raw = await stream.peek(4) + + // TODO this doesn't support when size = 0 (until EOF) or size = 1 (extended size) + const size = new DataView(raw.buffer, raw.byteOffset, raw.byteLength).getUint32(0) + const atom = await stream.bytes(size) + + // Make a copy of the atom because mp4box only accepts an ArrayBuffer unfortunately + let box = new Uint8Array(atom.byteLength); + box.set(atom) + + // and for some reason we need to modify the underlying ArrayBuffer with offset + let buffer = box.buffer as MP4.ArrayBuffer + buffer.fileStart = offset + + // Parse the data + offset = input.appendBuffer(buffer) + input.flush() + } + } + + onSamples(track_id: number, track: MP4.Track, samples: MP4.Sample[]) { + let decoder = this.decoders.get(track_id); + + if (!decoder) { + // We need a sample to initalize the video decoder, because of mp4box limitations. + let sample = samples[0]; + + if (MP4.isVideoTrack(track)) { + // Configure the decoder using the AVC box for H.264 + // TODO it should be easy to support other codecs, just need to know the right boxes. + const avcc = sample.description.avcC; + if (!avcc) throw new Error("TODO only h264 is supported"); + + const description = new MP4.Stream(new Uint8Array(avcc.size), 0, false) + avcc.write(description) + + const videoDecoder = new VideoDecoder({ + output: this.renderer.push.bind(this.renderer), + error: console.warn, + }); + + videoDecoder.configure({ + codec: track.codec, + codedHeight: track.video.height, + codedWidth: track.video.width, + description: description.buffer?.slice(8), + // optimizeForLatency: true + }) + + decoder = videoDecoder + } else if (MP4.isAudioTrack(track)) { + const audioDecoder = new AudioDecoder({ + output: this.renderer.push.bind(this.renderer), + error: console.warn, + }); + + audioDecoder.configure({ + codec: track.codec, + numberOfChannels: track.audio.channel_count, + sampleRate: track.audio.sample_rate, + }) + + decoder = audioDecoder + } else { + throw new Error("unknown track type") + } + + this.decoders.set(track_id, decoder) + } + + for (let sample of samples) { + // Convert to microseconds + const timestamp = 1000 * 1000 * sample.dts / sample.timescale + const duration = 1000 * 1000 * sample.duration / sample.timescale + + if (isAudioDecoder(decoder)) { + decoder.decode(new EncodedAudioChunk({ + type: sample.is_sync ? "key" : "delta", + data: sample.data, + duration: duration, + timestamp: timestamp, + })) + } else if (isVideoDecoder(decoder)) { + decoder.decode(new EncodedVideoChunk({ + type: sample.is_sync ? "key" : "delta", + data: sample.data, + duration: duration, + timestamp: timestamp, + })) + } else { + throw new Error("unknown decoder type") + } + } + } +} + +function isAudioDecoder(decoder: AudioDecoder | VideoDecoder): decoder is AudioDecoder { + return decoder instanceof AudioDecoder +} + +function isVideoDecoder(decoder: AudioDecoder | VideoDecoder): decoder is VideoDecoder { + return decoder instanceof VideoDecoder +} \ No newline at end of file diff --git a/player/src/media/index.ts b/player/src/media/index.ts new file mode 100644 index 0000000..fd42ed9 --- /dev/null +++ b/player/src/media/index.ts @@ -0,0 +1,82 @@ +import * as Message from "./message" +import { RingInit } from "./ring" + +// Abstracts the Worker and Worklet into a simpler API +// This class must be created on the main thread due to AudioContext. +export default class Media { + context: AudioContext; + worker: Worker; + worklet: Promise; + + constructor(videoConfig: Message.VideoConfig) { + // Assume 44.1kHz and two audio channels + const audioConfig = { + sampleRate: 44100, + ring: new RingInit(2, 4410), // 100ms at 44.1khz + } + + const config = { + audio: audioConfig, + video: videoConfig, + } + + this.context = new AudioContext({ + latencyHint: "interactive", + sampleRate: config.audio.sampleRate, + }) + + + this.worker = this.setupWorker(config) + this.worklet = this.setupWorklet(config) + } + + init(init: Message.Init) { + this.worker.postMessage({ init }, [ init.buffer.buffer, init.reader ]) + } + + segment(segment: Message.Segment) { + this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ]) + } + + play(play: Message.Play) { + this.context.resume() + //this.worker.postMessage({ play }) + } + + private setupWorker(config: Message.Config): Worker { + const url = new URL('worker.ts', import.meta.url) + + const worker = new Worker(url, { + type: "module", + name: "media", + }) + + worker.postMessage({ config }, [ config.video.canvas ]) + + return worker + } + + private async setupWorklet(config: Message.Config): Promise { + // Load the worklet source code. + const url = new URL('worklet.ts', import.meta.url) + await this.context.audioWorklet.addModule(url) + + const volume = this.context.createGain() + volume.gain.value = 2.0; + + // Create a worklet + const worklet = new AudioWorkletNode(this.context, 'renderer'); + worklet.onprocessorerror = (e: Event) => { + console.error("Audio worklet error:", e) + }; + + worklet.port.postMessage({ config }) + + // Connect the worklet to the volume node and then to the speakers + worklet.connect(volume) + volume.connect(this.context.destination) + + return worklet + } + +} \ No newline at end of file diff --git a/player/src/audio/message.ts b/player/src/media/message.ts similarity index 50% rename from player/src/audio/message.ts rename to player/src/media/message.ts index 73d4f1c..a457200 100644 --- a/player/src/audio/message.ts +++ b/player/src/media/message.ts @@ -1,28 +1,29 @@ import * as MP4 from "../mp4" -import { RingInit } from "./ring" +import { RingInit } from "../media/ring" export interface Config { + audio: AudioConfig; + video: VideoConfig; +} + +export interface VideoConfig { + canvas: OffscreenCanvas; +} + +export interface AudioConfig { + // audio stuff sampleRate: number; ring: RingInit; } export interface Init { - track: string; - info: MP4.Info; - raw: MP4.ArrayBuffer[]; -} - -export interface Segment { - track: string; buffer: Uint8Array; // unread buffered data reader: ReadableStream; // unread unbuffered data } -// Audio tells video when the given timestamp should be rendered. -export interface Sync { - origin: number; - clock: DOMHighResTimeStamp; - timestamp: number; +export interface Segment { + buffer: Uint8Array; // unread buffered data + reader: ReadableStream; // unread unbuffered data } export interface Play { diff --git a/player/src/media/renderer.ts b/player/src/media/renderer.ts new file mode 100644 index 0000000..af56e2a --- /dev/null +++ b/player/src/media/renderer.ts @@ -0,0 +1,138 @@ +import * as Message from "./message"; +import { Ring } from "./ring" + +export default class Renderer { + audioRing: Ring; + audioQueue: Array; + + videoCanvas: OffscreenCanvas; + videoQueue: Array; + + render: number; // non-zero if requestAnimationFrame has been called + sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0, in microseconds + last?: number; // the timestamp of the last rendered frame, in microseconds + + constructor(config: Message.Config) { + this.audioRing = new Ring(config.audio.ring); + this.audioQueue = []; + + this.videoCanvas = config.video.canvas; + this.videoQueue = []; + + this.render = 0; + } + + push(frame: AudioData | VideoFrame) { + if (!this.sync) { + // Save the frame as the sync point + this.sync = 1000 * performance.now() - frame.timestamp + } + + // Drop any old frames + if (this.last && frame.timestamp <= this.last) { + frame.close() + return + } + + let queue + if (isAudioData(frame)) { + queue = this.audioQueue; + } else if (isVideoFrame(frame)) { + queue = this.videoQueue; + } else { + throw new Error("unknown frame type") + } + + // Insert the frame into the queue sorted by timestamp. + if (queue.length > 0 && queue[queue.length-1].timestamp <= frame.timestamp) { + // Fast path because we normally append to the end. + queue.push(frame as any) + } else { + // Do a full binary search + let low = 0 + let high = queue.length; + + while (low < high) { + var mid = (low + high) >>> 1; + if (queue[mid].timestamp < frame.timestamp) low = mid + 1; + else high = mid; + } + + queue.splice(low, 0, frame as any) + } + + // Queue up to render the next frame. + if (!this.render) { + this.render = self.requestAnimationFrame(this.draw.bind(this)) + } + } + + draw(now: DOMHighResTimeStamp) { + // Determine the target timestamp. + const target = 1000 * now - this.sync! + + this.drawAudio(now, target) + this.drawVideo(now, target) + + if (this.audioQueue.length || this.videoQueue.length) { + this.render = self.requestAnimationFrame(this.draw.bind(this)) + } else { + this.render = 0 + } + } + + drawAudio(now: DOMHighResTimeStamp, target: DOMHighResTimeStamp) { + // Check if we should skip some frames + while (this.audioQueue.length) { + const next = this.audioQueue[0] + if (next.timestamp >= target) { + let ok = this.audioRing.write(next) + if (!ok) { + // No more space in the ring + break + } + } else { + console.warn("dropping audio") + } + + next.close() + this.audioQueue.shift() + } + } + + drawVideo(now: DOMHighResTimeStamp, target: DOMHighResTimeStamp) { + if (this.videoQueue.length == 0) return; + + let frame = this.videoQueue[0]; + if (frame.timestamp >= target) { + // nothing to render yet, wait for the next animation frame + this.render = self.requestAnimationFrame(this.draw.bind(this)) + return + } + + this.videoQueue.shift(); + + // Check if we should skip some frames + while (this.videoQueue.length) { + const next = this.videoQueue[0] + if (next.timestamp > target) break + + frame.close() + frame = this.videoQueue.shift()!; + } + + const ctx = this.videoCanvas.getContext("2d"); + ctx!.drawImage(frame, 0, 0, this.videoCanvas.width, this.videoCanvas.height) // TODO aspect ratio + + this.last = frame.timestamp; + frame.close() + } +} + +function isAudioData(frame: AudioData | VideoFrame): frame is AudioData { + return frame instanceof AudioData +} + +function isVideoFrame(frame: AudioData | VideoFrame): frame is VideoFrame { + return frame instanceof VideoFrame +} \ No newline at end of file diff --git a/player/src/audio/ring.ts b/player/src/media/ring.ts similarity index 100% rename from player/src/audio/ring.ts rename to player/src/media/ring.ts diff --git a/player/src/video/worker.ts b/player/src/media/worker.ts similarity index 86% rename from player/src/video/worker.ts rename to player/src/media/worker.ts index 8eeede8..4597c29 100644 --- a/player/src/video/worker.ts +++ b/player/src/media/worker.ts @@ -13,10 +13,10 @@ self.addEventListener('message', async (e: MessageEvent) => { decoder = new Decoder(renderer) } else if (e.data.init) { const init = e.data.init as Message.Init - await decoder.init(init) + await decoder.receiveInit(init) } else if (e.data.segment) { const segment = e.data.segment as Message.Segment - await decoder.decode(segment) + await decoder.receiveSegment(segment) } }) diff --git a/player/src/audio/worklet.ts b/player/src/media/worklet.ts similarity index 93% rename from player/src/audio/worklet.ts rename to player/src/media/worklet.ts index 401eec9..5961c32 100644 --- a/player/src/audio/worklet.ts +++ b/player/src/media/worklet.ts @@ -25,7 +25,7 @@ class Renderer extends AudioWorkletProcessor { } config(config: Message.Config) { - this.ring = new Ring(config.ring) + this.ring = new Ring(config.audio.ring) } // Inputs and outputs in groups of 128 samples. diff --git a/player/src/mp4/index.ts b/player/src/mp4/index.ts index e05fbce..ca1a0bd 100644 --- a/player/src/mp4/index.ts +++ b/player/src/mp4/index.ts @@ -1,11 +1,12 @@ -// Rename some stuff so it's on brand. -export { - createFile as New, - MP4File as File, - MP4ArrayBuffer as ArrayBuffer, - MP4Info as Info, - DataStream as Stream, - Sample, -} from "mp4box" +import * as MP4 from "./rename" +export * from "./rename" -export { Init, InitParser } from "./init" \ No newline at end of file +export { Init, InitParser } from "./init" + +export function isAudioTrack(track: MP4.Track): track is MP4.AudioTrack { + return (track as MP4.AudioTrack).audio !== undefined; +} + +export function isVideoTrack(track: MP4.Track): track is MP4.VideoTrack { + return (track as MP4.VideoTrack).video !== undefined; +} \ No newline at end of file diff --git a/player/src/mp4/init.ts b/player/src/mp4/init.ts index d36c25b..45e7c42 100644 --- a/player/src/mp4/init.ts +++ b/player/src/mp4/init.ts @@ -20,19 +20,7 @@ export class InitParser { // Create a promise that gets resolved once the init segment has been parsed. this.info = new Promise((resolve, reject) => { this.mp4box.onError = reject - - // https://github.com/gpac/mp4box.js#onreadyinfo - this.mp4box.onReady = (info: MP4.Info) => { - if (!info.isFragmented) { - reject("expected a fragmented mp4") - } - - if (info.tracks.length != 1) { - reject("expected a single track") - } - - resolve(info) - } + this.mp4box.onReady = resolve }) } diff --git a/player/src/mp4/mp4box.d.ts b/player/src/mp4/mp4box.d.ts index 7ce91f7..018f185 100644 --- a/player/src/mp4/mp4box.d.ts +++ b/player/src/mp4/mp4box.d.ts @@ -1,7 +1,7 @@ // https://github.com/gpac/mp4box.js/issues/233 declare module "mp4box" { - interface MP4MediaTrack { + export interface MP4MediaTrack { id: number; created: Date; modified: Date; @@ -19,26 +19,26 @@ declare module "mp4box" { nb_samples: number; } - interface MP4VideoData { + export interface MP4VideoData { width: number; height: number; } - interface MP4VideoTrack extends MP4MediaTrack { + export interface MP4VideoTrack extends MP4MediaTrack { video: MP4VideoData; } - interface MP4AudioData { + export interface MP4AudioData { sample_rate: number; channel_count: number; sample_size: number; } - interface MP4AudioTrack extends MP4MediaTrack { + export interface MP4AudioTrack extends MP4MediaTrack { audio: MP4AudioData; } - type MP4Track = MP4VideoTrack | MP4AudioTrack; + export type MP4Track = MP4VideoTrack | MP4AudioTrack; export interface MP4Info { duration: number; diff --git a/player/src/mp4/rename.ts b/player/src/mp4/rename.ts new file mode 100644 index 0000000..d45682b --- /dev/null +++ b/player/src/mp4/rename.ts @@ -0,0 +1,12 @@ +// Rename some stuff so it's on brand. +export { + createFile as New, + MP4File as File, + MP4ArrayBuffer as ArrayBuffer, + MP4Info as Info, + MP4Track as Track, + MP4AudioTrack as AudioTrack, + MP4VideoTrack as VideoTrack, + DataStream as Stream, + Sample, +} from "mp4box" \ No newline at end of file diff --git a/player/src/player/index.ts b/player/src/player/index.ts index b18a625..7fafccb 100644 --- a/player/src/player/index.ts +++ b/player/src/player/index.ts @@ -1,6 +1,5 @@ -import Audio from "../audio" import Transport from "../transport" -import Video from "../video" +import Media from "../media" export interface PlayerInit { url: string; @@ -9,22 +8,18 @@ export interface PlayerInit { } export default class Player { - audio: Audio; - video: Video; + media: Media; transport: Transport; constructor(props: PlayerInit) { - this.audio = new Audio() - this.video = new Video({ + this.media = new Media({ canvas: props.canvas.transferControlToOffscreen(), }) this.transport = new Transport({ url: props.url, fingerprint: props.fingerprint, - - audio: this.audio, - video: this.video, + media: this.media, }) } @@ -33,13 +28,6 @@ export default class Player { } play() { - this.audio.play({}) - //this.video.play() + //this.media.play() } - - onMessage(msg: any) { - if (msg.sync) { - msg.sync - } - } } \ No newline at end of file diff --git a/player/src/transport/index.ts b/player/src/transport/index.ts index ee58ed1..e588a8f 100644 --- a/player/src/transport/index.ts +++ b/player/src/transport/index.ts @@ -2,30 +2,22 @@ import * as Message from "./message" import * as Stream from "../stream" import * as MP4 from "../mp4" -import Audio from "../audio" -import Video from "../video" +import Media from "../media" export interface TransportInit { url: string; fingerprint?: WebTransportHash; // the certificate fingerprint, temporarily needed for local development - - audio: Audio; - video: Video; + media: Media; } export default class Transport { quic: Promise; api: Promise; - tracks: Map - audio: Audio; - video: Video; + media: Media; constructor(props: TransportInit) { - this.tracks = new Map(); - - this.audio = props.audio; - this.video = props.video; + this.media = props.media; this.quic = this.connect(props) @@ -94,82 +86,18 @@ export default class Transport { const msg = JSON.parse(payload) if (msg.init) { - return this.handleInit(r, msg.init as Message.Init) + return this.media.init({ + buffer: r.buffer, + reader: r.reader, + }) } else if (msg.segment) { - return this.handleSegment(r, msg.segment as Message.Segment) + return this.media.segment({ + buffer: r.buffer, + reader: r.reader, + }) } else { console.warn("unknown message", msg); } } } - - async handleInit(stream: Stream.Reader, msg: Message.Init) { - console.log("handle init", msg); - - let track = this.tracks.get(msg.id); - if (!track) { - track = new MP4.InitParser() - this.tracks.set(msg.id, track) - } - - while (1) { - const data = await stream.read() - if (!data) break - - track.push(data) - } - - const info = await track.info - - console.log(info); - - if (info.audioTracks.length + info.videoTracks.length != 1) { - throw new Error("expected a single track") - } - - if (info.audioTracks.length) { - this.audio.init({ - track: msg.id, - info: info, - raw: track.raw, - }) - } else if (info.videoTracks.length) { - this.video.init({ - track: msg.id, - info: info, - raw: track.raw, - }) - } else { - throw new Error("init is neither audio nor video") - } - } - - async handleSegment(stream: Stream.Reader, msg: Message.Segment) { - console.log("handle segment", msg); - - let track = this.tracks.get(msg.init); - if (!track) { - track = new MP4.InitParser() - this.tracks.set(msg.init, track) - } - - // Wait until we learn if this is an audio or video track - const info = await track.info - - if (info.audioTracks.length) { - this.audio.segment({ - track: msg.init, - buffer: stream.buffer, - reader: stream.reader, - }) - } else if (info.videoTracks.length) { - this.video.segment({ - track: msg.init, - buffer: stream.buffer, - reader: stream.reader, - }) - } else { - throw new Error("segment is neither audio nor video") - } - } } \ No newline at end of file diff --git a/player/src/video/decoder.ts b/player/src/video/decoder.ts deleted file mode 100644 index 582dcd2..0000000 --- a/player/src/video/decoder.ts +++ /dev/null @@ -1,127 +0,0 @@ -import * as Message from "./message"; -import * as MP4 from "../mp4" -import * as Stream from "../stream" -import * as Util from "../util" - -import Renderer from "./renderer" - -export default class Decoder { - // Store the init message for each track - tracks: Map> - renderer: Renderer; - - constructor(renderer: Renderer) { - this.tracks = new Map(); - this.renderer = renderer; - } - - async init(msg: Message.Init) { - let track = this.tracks.get(msg.track); - if (!track) { - track = new Util.Deferred() - this.tracks.set(msg.track, track) - } - - if (msg.info.videoTracks.length != 1 || msg.info.audioTracks.length != 0) { - throw new Error("Expected a single video track") - } - - track.resolve(msg) - } - - async decode(msg: Message.Segment) { - let track = this.tracks.get(msg.track); - if (!track) { - track = new Util.Deferred() - this.tracks.set(msg.track, track) - } - - // Wait for the init segment to be fully received and parsed - const init = await track.promise; - const info = init.info; - const video = info.videoTracks[0] - - const decoder = new VideoDecoder({ - output: (frame: VideoFrame) => { - this.renderer.emit(frame) - }, - error: (err: Error) => { - console.warn(err) - } - }); - - const input = MP4.New(); - - input.onSamples = (id: number, user: any, samples: MP4.Sample[]) => { - for (let sample of samples) { - const timestamp = 1000 * sample.dts / sample.timescale // milliseconds - - if (sample.is_sync) { - // Configure the decoder using the AVC box for H.264 - const avcc = sample.description.avcC; - const description = new MP4.Stream(new Uint8Array(avcc.size), 0, false) - avcc.write(description) - - decoder.configure({ - codec: video.codec, - codedHeight: video.track_height, - codedWidth: video.track_width, - description: description.buffer?.slice(8), - // optimizeForLatency: true - }) - } - - decoder.decode(new EncodedVideoChunk({ - data: sample.data, - duration: sample.duration, - timestamp: timestamp, - type: sample.is_sync ? "key" : "delta", - })) - } - } - - input.onReady = (info: any) => { - input.setExtractionOptions(info.tracks[0].id, {}, { nbSamples: 1 }); - input.start(); - } - - // MP4box requires us to reparse the init segment unfortunately - let offset = 0; - - for (let raw of init.raw) { - raw.fileStart = offset - offset = input.appendBuffer(raw) - } - - const stream = new Stream.Reader(msg.reader, msg.buffer) - - /* TODO I'm not actually sure why this code doesn't work; something trips up the MP4 parser - while (1) { - const data = await stream.read() - if (!data) break - - input.appendBuffer(data) - input.flush() - } - */ - - // One day I'll figure it out; until then read one top-level atom at a time - while (!await stream.done()) { - const raw = await stream.peek(4) - const size = new DataView(raw.buffer, raw.byteOffset, raw.byteLength).getUint32(0) - const atom = await stream.bytes(size) - - // Make a copy of the atom because mp4box only accepts an ArrayBuffer unfortunately - let box = new Uint8Array(atom.byteLength); - box.set(atom) - - // and for some reason we need to modify the underlying ArrayBuffer with offset - let buffer = box.buffer as MP4.ArrayBuffer - buffer.fileStart = offset - - // Parse the data - offset = input.appendBuffer(buffer) - input.flush() - } - } -} \ No newline at end of file diff --git a/player/src/video/index.ts b/player/src/video/index.ts deleted file mode 100644 index f447072..0000000 --- a/player/src/video/index.ts +++ /dev/null @@ -1,27 +0,0 @@ -import * as Message from "./message" - -// Wrapper around the WebWorker API -export default class Video { - worker: Worker; - - constructor(config: Message.Config) { - const url = new URL('worker.ts', import.meta.url) - this.worker = new Worker(url, { - type: "module", - name: "video", - }) - this.worker.postMessage({ config }, [ config.canvas ]) - } - - init(init: Message.Init) { - this.worker.postMessage({ init }) // note: we copy the raw init bytes each time - } - - segment(segment: Message.Segment) { - this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ]) - } - - play() { - // TODO - } -} \ No newline at end of file diff --git a/player/src/video/message.ts b/player/src/video/message.ts deleted file mode 100644 index 61e4b6d..0000000 --- a/player/src/video/message.ts +++ /dev/null @@ -1,17 +0,0 @@ -import * as MP4 from "../mp4" - -export interface Config { - canvas: OffscreenCanvas; -} - -export interface Init { - track: string; - info: MP4.Info; - raw: MP4.ArrayBuffer[]; -} - -export interface Segment { - track: string; - buffer: Uint8Array; // unread buffered data - reader: ReadableStream; // unread unbuffered data -} \ No newline at end of file diff --git a/player/src/video/renderer.ts b/player/src/video/renderer.ts deleted file mode 100644 index 98fd3b6..0000000 --- a/player/src/video/renderer.ts +++ /dev/null @@ -1,91 +0,0 @@ -import * as Message from "./message"; - -export default class Renderer { - canvas: OffscreenCanvas; - queue: Array; - render: number; // non-zero if requestAnimationFrame has been called - sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0 - last?: number; // the timestamp of the last rendered frame - - constructor(config: Message.Config) { - this.canvas = config.canvas; - this.queue = []; - this.render = 0; - } - - emit(frame: VideoFrame) { - if (!this.sync) { - // Save the frame as the sync point - this.sync = performance.now() - frame.timestamp - } - - // Drop any old frames - if (this.last && frame.timestamp <= this.last) { - frame.close() - return - } - - // Insert the frame into the queue sorted by timestamp. - if (this.queue.length > 0 && this.queue[this.queue.length-1].timestamp <= frame.timestamp) { - // Fast path because we normally append to the end. - this.queue.push(frame) - } else { - // Do a full binary search - let low = 0 - let high = this.queue.length; - - while (low < high) { - var mid = (low + high) >>> 1; - if (this.queue[mid].timestamp < frame.timestamp) low = mid + 1; - else high = mid; - } - - this.queue.splice(low, 0, frame) - } - - // Queue up to render the next frame. - if (!this.render) { - this.render = self.requestAnimationFrame(this.draw.bind(this)) - } - } - - draw(now: DOMHighResTimeStamp) { - // Determine the target timestamp. - const target = now - this.sync! - - let frame = this.queue[0] - if (frame.timestamp >= target) { - // nothing to render yet, wait for the next animation frame - this.render = self.requestAnimationFrame(this.draw.bind(this)) - return - } - - this.queue.shift() - - // Check if we should skip some frames - while (this.queue.length) { - const next = this.queue[0] - if (next.timestamp > target) { - break - } - - frame.close() - - this.queue.shift() - frame = next - } - - const ctx = this.canvas.getContext("2d"); - ctx!.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height) // TODO aspect ratio - - this.last = frame.timestamp; - frame.close() - - if (this.queue.length > 0) { - this.render = self.requestAnimationFrame(this.draw.bind(this)) - } else { - // Break the loop for now - this.render = 0 - } - } -} \ No newline at end of file diff --git a/server/Cargo.lock b/server/Cargo.lock index b470484..0ae81ac 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -329,7 +329,6 @@ dependencies = [ [[package]] name = "mp4" version = "0.13.0" -source = "git+https://github.com/kixelated/mp4-rust.git?branch=trexs#efefcc47353f477518bff01493785ae0daa8efd4" dependencies = [ "byteorder", "bytes", diff --git a/server/Cargo.toml b/server/Cargo.toml index 660ab93..af201c6 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -13,6 +13,6 @@ mio = { version = "0.8", features = ["net", "os-poll"] } env_logger = "0.9.3" ring = "0.16" anyhow = "1.0.70" -mp4 = { git = "https://github.com/kixelated/mp4-rust.git", branch = "trexs" } +mp4 = { path = "../../mp4-rust" } # { git = "https://github.com/kixelated/mp4-rust.git", branch = "trexs" } serde = "1.0.160" serde_json = "1.0" \ No newline at end of file diff --git a/server/src/media/mod.rs b/server/src/media/mod.rs index e7ec5eb..f73c588 100644 --- a/server/src/media/mod.rs +++ b/server/src/media/mod.rs @@ -1,3 +1,3 @@ mod source; -pub use source::{Fragment, Source}; +pub use source::{Fragment, Source}; \ No newline at end of file diff --git a/server/src/media/source.rs b/server/src/media/source.rs index 38cfb97..e639ce6 100644 --- a/server/src/media/source.rs +++ b/server/src/media/source.rs @@ -1,37 +1,32 @@ -use io::Read; -use std::collections::VecDeque; use std::{fs, io, time}; - -use std::io::Write; +use std::collections::{HashMap,VecDeque}; +use std::io::Read; use anyhow; -use mp4; -use mp4::{ReadBox, WriteBox}; +use mp4; +use mp4::ReadBox; pub struct Source { // We read the file once, in order, and don't seek backwards. reader: io::BufReader, - // Any fragments parsed and ready to be returned by next(). - fragments: VecDeque, - // The timestamp when the broadcast "started", so we can sleep to simulate a live stream. start: time::Instant, - // The raw ftyp box, which we need duplicate for each track, but we don't know how many tracks exist yet. - ftyp: Vec, + // The initialization payload; ftyp + moov boxes. + pub init: Vec, - // The parsed moov box, so we can look up track information later. - moov: Option, + // The timescale used for each track. + timescale: HashMap, + + // Any fragments parsed and ready to be returned by next(). + fragments: VecDeque, } pub struct Fragment { // The track ID for the fragment. - pub track: u32, - - // The type of the fragment. - pub typ: mp4::BoxType, + pub track_id: u32, // The data of the fragment. pub data: Vec, @@ -44,21 +39,42 @@ pub struct Fragment { } impl Source { - pub fn new(path: &str) -> io::Result { + pub fn new(path: &str) -> anyhow::Result { let f = fs::File::open(path)?; - let reader = io::BufReader::new(f); + let mut reader = io::BufReader::new(f); let start = time::Instant::now(); - Ok(Self { + let ftyp = read_atom(&mut reader)?; + anyhow::ensure!(&ftyp[4..8] == b"ftyp", "expected ftyp atom"); + + let moov = read_atom(&mut reader)?; + anyhow::ensure!(&moov[4..8] == b"moov", "expected moov atom"); + + let mut init = ftyp; + init.extend(&moov); + + // We're going to parse the moov box. + // We have to read the moov box header to correctly advance the cursor for the mp4 crate. + let mut moov_reader = io::Cursor::new(&moov); + let moov_header = mp4::BoxHeader::read(&mut moov_reader)?; + + // Parse the moov box so we can detect the timescales for each track. + let moov = mp4::MoovBox::read_box(&mut moov_reader, moov_header.size)?; + let timescale = moov.traks + .iter() + .map(|trak| (trak.tkhd.track_id, trak.mdia.mdhd.timescale)) + .collect(); + + Ok(Self{ reader, start, + init, + timescale, fragments: VecDeque::new(), - ftyp: Vec::new(), - moov: None, }) } - pub fn get(&mut self) -> anyhow::Result> { + pub fn fragment(&mut self) -> anyhow::Result> { if self.fragments.is_empty() { self.parse()?; }; @@ -72,63 +88,13 @@ impl Source { fn parse(&mut self) -> anyhow::Result<()> { loop { - // Read the next full atom. - let atom = read_box(&mut self.reader)?; + let atom = read_atom(&mut self.reader)?; - // Before we return it, let's do some simple parsing. let mut reader = io::Cursor::new(&atom); let header = mp4::BoxHeader::read(&mut reader)?; match header.name { - mp4::BoxType::FtypBox => { - // Don't return anything until we know the total number of tracks. - // To be honest, I didn't expect the borrow checker to allow this, but it does! - self.ftyp = atom; - } - mp4::BoxType::MoovBox => { - // We need to split the moov based on the tracks. - let moov = mp4::MoovBox::read_box(&mut reader, header.size)?; - - for trak in &moov.traks { - let track_id = trak.tkhd.track_id; - - // Push the styp atom for each track. - self.fragments.push_back(Fragment { - track: track_id, - typ: mp4::BoxType::FtypBox, - data: self.ftyp.clone(), - keyframe: false, - timestamp: None, - }); - - // Unfortunately, we need to create a brand new moov atom for each track. - // We remove every box for other track IDs. - let mut toov = moov.clone(); - toov.traks.retain(|t| t.tkhd.track_id == track_id); - toov.mvex - .as_mut() - .expect("missing mvex") - .trexs - .retain(|f| f.track_id == track_id); - - // Marshal the box. - let mut toov_data = Vec::new(); - toov.write_box(&mut toov_data)?; - - let mut file = std::fs::File::create(format!("track{}.mp4", track_id))?; - file.write_all(toov_data.as_slice())?; - - self.fragments.push_back(Fragment { - track: track_id, - typ: mp4::BoxType::MoovBox, - data: toov_data, - keyframe: false, - timestamp: None, - }); - } - - self.moov = Some(moov); - } + mp4::BoxType::FtypBox | mp4::BoxType::MoovBox => anyhow::bail!("must call init first"), mp4::BoxType::MoofBox => { let moof = mp4::MoofBox::read_box(&mut reader, header.size)?; @@ -138,8 +104,7 @@ impl Source { } self.fragments.push_back(Fragment { - track: moof.trafs[0].tfhd.track_id, - typ: mp4::BoxType::MoofBox, + track_id: moof.trafs[0].tfhd.track_id, data: atom, keyframe: has_keyframe(&moof), timestamp: first_timestamp(&moof), @@ -147,11 +112,9 @@ impl Source { } mp4::BoxType::MdatBox => { let moof = self.fragments.back().expect("no atom before mdat"); - assert!(moof.typ == mp4::BoxType::MoofBox, "no moof before mdat"); self.fragments.push_back(Fragment { - track: moof.track, - typ: mp4::BoxType::MoofBox, + track_id: moof.track_id, data: atom, keyframe: false, timestamp: None, @@ -160,7 +123,9 @@ impl Source { // We have some media data, return so we can start sending it. return Ok(()); } - _ => anyhow::bail!("unknown top-level atom: {:?}", header.name), + _ => { + // Skip unknown atoms + } } } } @@ -171,15 +136,9 @@ impl Source { let timestamp = next.timestamp?; // Find the timescale for the track. - let track = self - .moov - .as_ref()? - .traks - .iter() - .find(|t| t.tkhd.track_id == next.track)?; - let timescale = track.mdia.mdhd.timescale as u64; + let timescale = self.timescale.get(&next.track_id).unwrap(); - let delay = time::Duration::from_millis(1000 * timestamp / timescale); + let delay = time::Duration::from_millis(1000 * timestamp / *timescale as u64); let elapsed = self.start.elapsed(); delay.checked_sub(elapsed) @@ -187,14 +146,16 @@ impl Source { } // Read a full MP4 atom into a vector. -fn read_box(reader: &mut R) -> anyhow::Result> { +pub fn read_atom(reader: &mut R) -> anyhow::Result> { // Read the 8 bytes for the size + type let mut buf = [0u8; 8]; reader.read_exact(&mut buf)?; // Convert the first 4 bytes into the size. let size = u32::from_be_bytes(buf[0..4].try_into()?) as u64; - let mut out = buf.to_vec(); + //let typ = &buf[4..8].try_into().ok().unwrap(); + + let mut raw = buf.to_vec(); let mut limit = match size { // Runs until the end of the file. @@ -222,9 +183,9 @@ fn read_box(reader: &mut R) -> anyhow::Result> { }; // Append to the vector and return it. - limit.read_to_end(&mut out)?; + limit.read_to_end(&mut raw)?; - Ok(out) + Ok(raw) } fn has_keyframe(moof: &mp4::MoofBox) -> bool { @@ -261,4 +222,4 @@ fn has_keyframe(moof: &mp4::MoofBox) -> bool { fn first_timestamp(moof: &mp4::MoofBox) -> Option { Some(moof.trafs.first()?.tfdt.as_ref()?.base_media_decode_time) -} +} \ No newline at end of file diff --git a/server/src/session/message.rs b/server/src/session/message.rs index 74243e8..0849e2e 100644 --- a/server/src/session/message.rs +++ b/server/src/session/message.rs @@ -7,13 +7,11 @@ pub struct Message { } #[derive(Serialize, Deserialize)] -pub struct Init { - pub id: String, -} +pub struct Init {} #[derive(Serialize, Deserialize)] pub struct Segment { - pub init: String, + pub track_id: u32, } impl Message { diff --git a/server/src/session/mod.rs b/server/src/session/mod.rs index 47e0074..0d59e8e 100644 --- a/server/src/session/mod.rs +++ b/server/src/session/mod.rs @@ -1,6 +1,7 @@ mod message; use std::time; +use std::collections::hash_map as hmap; use quiche; use quiche::h3::webtransport; @@ -10,9 +11,8 @@ use crate::{media, transport}; #[derive(Default)] pub struct Session { media: Option, - stream_id: Option, // stream ID of the current segment - streams: transport::Streams, // An easy way of buffering stream data. + tracks: hmap::HashMap, // map from track_id to current stream_id } impl transport::App for Session { @@ -38,12 +38,23 @@ impl transport::App for Session { // req.authority() // req.path() // and you can validate this request with req.origin() + session.accept_connect_request(conn, None).unwrap(); // TODO let media = media::Source::new("../media/fragmented.mp4")?; - self.media = Some(media); + let init = &media.init; - session.accept_connect_request(conn, None).unwrap(); + // Create a JSON header. + let mut message = message::Message::new(); + message.init = Some(message::Init{}); + let data = message.serialize()?; + + // Create a new stream and write the header. + let stream_id = session.open_stream(conn, false)?; + self.streams.send(conn, stream_id, data.as_slice(), false)?; + self.streams.send(conn, stream_id, init.as_slice(), true)?; + + self.media = Some(media); } webtransport::ServerEvent::StreamData(stream_id) => { let mut buf = vec![0; 10000]; @@ -84,62 +95,54 @@ impl Session { }; // Get the next media fragment. - let fragment = match media.get()? { + let fragment = match media.fragment()? { Some(f) => f, None => return Ok(()), }; - // Check if we have already created a stream for this fragment. - let stream_id = match self.stream_id { - Some(old_stream_id) if fragment.keyframe => { - // This is the start of a new segment. + let stream_id = match self.tracks.get(&fragment.track_id) { + // Close the old stream. + Some(stream_id) if fragment.keyframe => { + self.streams.send(conn, *stream_id, &[], true)?; + None + }, - // Close the prior stream. - self.streams.send(conn, old_stream_id, &[], true)?; + // Use the existing stream + Some(stream_id) => Some(*stream_id), - // Encode a JSON header indicating this is the video track. - let mut message = message::Message::new(); - message.segment = Some(message::Segment { - init: "video".to_string(), - }); + // No existing stream. + _ => None, + }; - // Open a new stream. + let stream_id = match stream_id { + // Use the existing stream, + Some(stream_id) => stream_id, + + // Open a new stream. + None => { let stream_id = session.open_stream(conn, false)?; // TODO: conn.stream_priority(stream_id, urgency, incremental) + // Encode a JSON header indicating this is a new track. + let mut message = message::Message::new(); + message.segment = Some(message::Segment { + track_id: fragment.track_id, + }); + // Write the header. let data = message.serialize()?; self.streams.send(conn, stream_id, &data, false)?; - stream_id - } - None => { - // This is the start of an init segment. - - // Create a JSON header. - let mut message = message::Message::new(); - message.init = Some(message::Init { - id: "video".to_string(), - }); - - let data = message.serialize()?; - - // Create a new stream and write the header. - let stream_id = session.open_stream(conn, false)?; - self.streams.send(conn, stream_id, data.as_slice(), false)?; + self.tracks.insert(fragment.track_id, stream_id); stream_id - } - Some(stream_id) => stream_id, // Continuation of init or segment + }, }; // Write the current fragment. let data = fragment.data.as_slice(); self.streams.send(conn, stream_id, data, false)?; - // Save the stream ID for the next fragment. - self.stream_id = Some(stream_id); - Ok(()) } } diff --git a/server/src/session/session.rs b/server/src/session/session.rs deleted file mode 100644 index 8b13789..0000000 --- a/server/src/session/session.rs +++ /dev/null @@ -1 +0,0 @@ - From e6791b872d26d8f9aed4de8e446afcd9873a089f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 8 May 2023 09:20:51 -0600 Subject: [PATCH 11/23] Finish merging audio with video. --- player/src/audio/index.ts | 77 -------------------------------------- player/src/media/index.ts | 25 ++++++------- player/src/player/index.ts | 2 +- 3 files changed, 13 insertions(+), 91 deletions(-) delete mode 100644 player/src/audio/index.ts diff --git a/player/src/audio/index.ts b/player/src/audio/index.ts deleted file mode 100644 index 7076cd3..0000000 --- a/player/src/audio/index.ts +++ /dev/null @@ -1,77 +0,0 @@ -import * as Message from "./message" -import Renderer from "../media/audio" -import Decoder from "./decoder" - -import { RingInit } from "../media/ring" - -// Abstracts the Worker and Worklet into a simpler API -// This class must be created on the main thread due to AudioContext. -export default class Audio { - context: AudioContext; - worker: Worker; - worklet: Promise; - - constructor() { - // Assume 44.1kHz and two audio channels - const config = { - sampleRate: 44100, - ring: new RingInit(2, 4410), // 100ms at 44.1khz - } - - this.context = new AudioContext({ - latencyHint: "interactive", - sampleRate: config.sampleRate, - }) - - this.worker = this.setupWorker(config) - this.worklet = this.setupWorklet(config) - } - - private setupWorker(config: Message.Config): Worker { - const url = new URL('worker.ts', import.meta.url) - const worker = new Worker(url, { - name: "audio", - type: "module", - }) - - worker.postMessage({ config }) - - return worker - } - - private async setupWorklet(config: Message.Config): Promise { - // Load the worklet source code. - const url = new URL('worklet.ts', import.meta.url) - await this.context.audioWorklet.addModule(url) - - const volume = this.context.createGain() - volume.gain.value = 2.0; - - // Create a worklet - const worklet = new AudioWorkletNode(this.context, 'renderer'); - worklet.onprocessorerror = (e: Event) => { - console.error("Audio worklet error:", e) - }; - - worklet.port.postMessage({ config }) - - // Connect the worklet to the volume node and then to the speakers - worklet.connect(volume) - volume.connect(this.context.destination) - - return worklet - } - - init(init: Message.Init) { - this.worker.postMessage({ init }) - } - - segment(segment: Message.Segment) { - this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ]) - } - - play(play: Message.Play) { - this.context.resume() - //this.worker.postMessage({ play }) - } -} \ No newline at end of file diff --git a/player/src/media/index.ts b/player/src/media/index.ts index fd42ed9..376052a 100644 --- a/player/src/media/index.ts +++ b/player/src/media/index.ts @@ -30,19 +30,6 @@ export default class Media { this.worklet = this.setupWorklet(config) } - init(init: Message.Init) { - this.worker.postMessage({ init }, [ init.buffer.buffer, init.reader ]) - } - - segment(segment: Message.Segment) { - this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ]) - } - - play(play: Message.Play) { - this.context.resume() - //this.worker.postMessage({ play }) - } - private setupWorker(config: Message.Config): Worker { const url = new URL('worker.ts', import.meta.url) @@ -79,4 +66,16 @@ export default class Media { return worklet } + init(init: Message.Init) { + this.worker.postMessage({ init }, [ init.buffer.buffer, init.reader ]) + } + + segment(segment: Message.Segment) { + this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ]) + } + + play(play: Message.Play) { + this.context.resume() + //this.worker.postMessage({ play }) + } } \ No newline at end of file diff --git a/player/src/player/index.ts b/player/src/player/index.ts index 7fafccb..9e9c2a6 100644 --- a/player/src/player/index.ts +++ b/player/src/player/index.ts @@ -28,6 +28,6 @@ export default class Player { } play() { - //this.media.play() + this.media.play({}) } } \ No newline at end of file From 9f0c24b552df76fa72589faa7a2fcd94565631c8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 8 May 2023 10:30:32 -0600 Subject: [PATCH 12/23] Refactor the TS code a bit. --- media/generate | 7 +- player/src/index.ts | 11 ++- player/src/media/index.ts | 81 ------------------ player/src/mp4/index.ts | 26 +++--- player/src/mp4/mp4box.d.ts | 81 +++++++++++++++++- player/src/mp4/rename.ts | 12 --- player/src/{media => player}/decoder.ts | 13 ++- player/src/player/index.ts | 100 ++++++++++++++++++----- player/src/{media => player}/message.ts | 5 +- player/src/{media => player}/renderer.ts | 11 ++- player/src/{media => player}/ring.ts | 10 +-- player/src/{media => player}/worker.ts | 0 player/src/{media => player}/worklet.ts | 6 +- player/src/transport/index.ts | 29 +++---- player/src/transport/interface.ts | 14 ++++ player/src/transport/message.ts | 11 +-- 16 files changed, 238 insertions(+), 179 deletions(-) delete mode 100644 player/src/media/index.ts delete mode 100644 player/src/mp4/rename.ts rename player/src/{media => player}/decoder.ts (94%) rename player/src/{media => player}/message.ts (85%) rename player/src/{media => player}/renderer.ts (94%) rename player/src/{media => player}/ring.ts (95%) rename player/src/{media => player}/worker.ts (100%) rename player/src/{media => player}/worklet.ts (87%) create mode 100644 player/src/transport/interface.ts diff --git a/media/generate b/media/generate index 922d2bc..5105c06 100755 --- a/media/generate +++ b/media/generate @@ -6,8 +6,7 @@ cd "$(dirname "$0")" # separate_moof: Splits audio and video into separate moof flags. # omit_tfhd_offset: Removes absolute byte offsets so we can fragment. -ffmpeg -i source.mp4 \ +ffmpeg -i source.mp4 -y \ + -c copy \ -movflags empty_moov+frag_every_frame+separate_moof+omit_tfhd_offset \ - -c:v copy \ - -an \ - fragmented.mp4 + fragmented.mp4 2>&1 diff --git a/player/src/index.ts b/player/src/index.ts index 89e2e22..5882480 100644 --- a/player/src/index.ts +++ b/player/src/index.ts @@ -1,4 +1,5 @@ import Player from "./player" +import Transport from "./transport" // @ts-ignore embed the certificate fingerprint using bundler import fingerprintHex from 'bundle-text:../fingerprint.hex'; @@ -14,19 +15,23 @@ const params = new URLSearchParams(window.location.search) const url = params.get("url") || "https://127.0.0.1:4443/watch" const canvas = document.querySelector("canvas#video")! -const player = new Player({ +const transport = new Transport({ url: url, fingerprint: { // TODO remove when Chrome accepts the system CA "algorithm": "sha-256", "value": new Uint8Array(fingerprint), }, - canvas: canvas, +}) + +const player = new Player({ + transport, + canvas: canvas.transferControlToOffscreen(), }) const play = document.querySelector("#screen #play")! let playFunc = (e: Event) => { - player.play() + player.play({}) e.preventDefault() play.removeEventListener('click', playFunc) diff --git a/player/src/media/index.ts b/player/src/media/index.ts deleted file mode 100644 index 376052a..0000000 --- a/player/src/media/index.ts +++ /dev/null @@ -1,81 +0,0 @@ -import * as Message from "./message" -import { RingInit } from "./ring" - -// Abstracts the Worker and Worklet into a simpler API -// This class must be created on the main thread due to AudioContext. -export default class Media { - context: AudioContext; - worker: Worker; - worklet: Promise; - - constructor(videoConfig: Message.VideoConfig) { - // Assume 44.1kHz and two audio channels - const audioConfig = { - sampleRate: 44100, - ring: new RingInit(2, 4410), // 100ms at 44.1khz - } - - const config = { - audio: audioConfig, - video: videoConfig, - } - - this.context = new AudioContext({ - latencyHint: "interactive", - sampleRate: config.audio.sampleRate, - }) - - - this.worker = this.setupWorker(config) - this.worklet = this.setupWorklet(config) - } - - private setupWorker(config: Message.Config): Worker { - const url = new URL('worker.ts', import.meta.url) - - const worker = new Worker(url, { - type: "module", - name: "media", - }) - - worker.postMessage({ config }, [ config.video.canvas ]) - - return worker - } - - private async setupWorklet(config: Message.Config): Promise { - // Load the worklet source code. - const url = new URL('worklet.ts', import.meta.url) - await this.context.audioWorklet.addModule(url) - - const volume = this.context.createGain() - volume.gain.value = 2.0; - - // Create a worklet - const worklet = new AudioWorkletNode(this.context, 'renderer'); - worklet.onprocessorerror = (e: Event) => { - console.error("Audio worklet error:", e) - }; - - worklet.port.postMessage({ config }) - - // Connect the worklet to the volume node and then to the speakers - worklet.connect(volume) - volume.connect(this.context.destination) - - return worklet - } - - init(init: Message.Init) { - this.worker.postMessage({ init }, [ init.buffer.buffer, init.reader ]) - } - - segment(segment: Message.Segment) { - this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ]) - } - - play(play: Message.Play) { - this.context.resume() - //this.worker.postMessage({ play }) - } -} \ No newline at end of file diff --git a/player/src/mp4/index.ts b/player/src/mp4/index.ts index ca1a0bd..b375434 100644 --- a/player/src/mp4/index.ts +++ b/player/src/mp4/index.ts @@ -1,12 +1,16 @@ -import * as MP4 from "./rename" -export * from "./rename" +// Rename some stuff so it's on brand. +export { + createFile as New, + MP4File as File, + MP4ArrayBuffer as ArrayBuffer, + MP4Info as Info, + MP4Track as Track, + MP4AudioTrack as AudioTrack, + MP4VideoTrack as VideoTrack, + DataStream as Stream, + Box, + ISOFile, + Sample, +} from "mp4box" -export { Init, InitParser } from "./init" - -export function isAudioTrack(track: MP4.Track): track is MP4.AudioTrack { - return (track as MP4.AudioTrack).audio !== undefined; -} - -export function isVideoTrack(track: MP4.Track): track is MP4.VideoTrack { - return (track as MP4.VideoTrack).video !== undefined; -} \ No newline at end of file +export { Init, InitParser } from "./init" \ No newline at end of file diff --git a/player/src/mp4/mp4box.d.ts b/player/src/mp4/mp4box.d.ts index 018f185..2f95f94 100644 --- a/player/src/mp4/mp4box.d.ts +++ b/player/src/mp4/mp4box.d.ts @@ -82,7 +82,7 @@ declare module "mp4box" { description: any; data: ArrayBuffer; size: number; - alreadyRead: number; + alreadyRead?: number; duration: number; cts: number; dts: number; @@ -104,7 +104,7 @@ declare module "mp4box" { const LITTLE_ENDIAN: boolean; export class DataStream { - constructor(buffer: ArrayBuffer, byteOffset?: number, littleEndian?: boolean); + constructor(buffer?: ArrayBuffer, byteOffset?: number, littleEndian?: boolean); getPosition(): number; get byteLength(): number; @@ -144,5 +144,82 @@ declare module "mp4box" { // TODO I got bored porting the remaining functions } + export class Box { + write(stream: DataStream): void; + } + + export interface TrackOptions { + id?: number; + type?: string; + width?: number; + height?: number; + duration?: number; + layer?: number; + timescale?: number; + media_duration?: number; + language?: string; + hdlr?: string; + + // video + avcDecoderConfigRecord?: any; + + // audio + balance?: number; + channel_count?: number; + samplesize?: number; + samplerate?: number; + + //captions + namespace?: string; + schema_location?: string; + auxiliary_mime_types?: string; + + description?: any; + description_boxes?: Box[]; + + default_sample_description_index_id?: number; + default_sample_duration?: number; + default_sample_size?: number; + default_sample_flags?: number; + } + + export interface FileOptions { + brands?: string[]; + timescale?: number; + rate?: number; + duration?: number; + width?: number; + } + + export interface SampleOptions { + sample_description_index?: number; + duration?: number; + cts?: number; + dts?: number; + is_sync?: boolean; + is_leading?: number; + depends_on?: number; + is_depended_on?: number; + has_redundancy?: number; + degradation_priority?: number; + subsamples?: any; + } + + // TODO add the remaining functions + // TODO move to another module + export class ISOFile { + constructor(stream?: DataStream); + + init(options?: FileOptions): ISOFile; + addTrack(options?: TrackOptions): number; + addSample(track: number, data: ArrayBuffer, options?: SampleOptions): Sample; + + createSingleSampleMoof(sample: Sample): Box; + + // helpers + getTrackById(id: number): Box | undefined; + getTrexById(id: number): Box | undefined; + } + export { }; } \ No newline at end of file diff --git a/player/src/mp4/rename.ts b/player/src/mp4/rename.ts deleted file mode 100644 index d45682b..0000000 --- a/player/src/mp4/rename.ts +++ /dev/null @@ -1,12 +0,0 @@ -// Rename some stuff so it's on brand. -export { - createFile as New, - MP4File as File, - MP4ArrayBuffer as ArrayBuffer, - MP4Info as Info, - MP4Track as Track, - MP4AudioTrack as AudioTrack, - MP4VideoTrack as VideoTrack, - DataStream as Stream, - Sample, -} from "mp4box" \ No newline at end of file diff --git a/player/src/media/decoder.ts b/player/src/player/decoder.ts similarity index 94% rename from player/src/media/decoder.ts rename to player/src/player/decoder.ts index ca2f639..64c5683 100644 --- a/player/src/media/decoder.ts +++ b/player/src/player/decoder.ts @@ -1,7 +1,6 @@ import * as Message from "./message"; import * as MP4 from "../mp4" import * as Stream from "../stream" -import * as Util from "../util" import Renderer from "./renderer" @@ -82,7 +81,7 @@ export default class Decoder { // We need a sample to initalize the video decoder, because of mp4box limitations. let sample = samples[0]; - if (MP4.isVideoTrack(track)) { + if (isVideoTrack(track)) { // Configure the decoder using the AVC box for H.264 // TODO it should be easy to support other codecs, just need to know the right boxes. const avcc = sample.description.avcC; @@ -105,7 +104,7 @@ export default class Decoder { }) decoder = videoDecoder - } else if (MP4.isAudioTrack(track)) { + } else if (isAudioTrack(track)) { const audioDecoder = new AudioDecoder({ output: this.renderer.push.bind(this.renderer), error: console.warn, @@ -157,4 +156,12 @@ function isAudioDecoder(decoder: AudioDecoder | VideoDecoder): decoder is AudioD function isVideoDecoder(decoder: AudioDecoder | VideoDecoder): decoder is VideoDecoder { return decoder instanceof VideoDecoder +} + +function isAudioTrack(track: MP4.Track): track is MP4.AudioTrack { + return (track as MP4.AudioTrack).audio !== undefined; +} + +function isVideoTrack(track: MP4.Track): track is MP4.VideoTrack { + return (track as MP4.VideoTrack).video !== undefined; } \ No newline at end of file diff --git a/player/src/player/index.ts b/player/src/player/index.ts index 9e9c2a6..1ef73b2 100644 --- a/player/src/player/index.ts +++ b/player/src/player/index.ts @@ -1,33 +1,89 @@ +import * as Message from "./message" +import * as Ring from "./ring" import Transport from "../transport" -import Media from "../media" -export interface PlayerInit { - url: string; - fingerprint?: WebTransportHash; // the certificate fingerprint, temporarily needed for local development - canvas: HTMLCanvasElement; +export interface Config { + transport: Transport + canvas: OffscreenCanvas; } +// This class must be created on the main thread due to AudioContext. export default class Player { - media: Media; - transport: Transport; + context: AudioContext; + worker: Worker; + worklet: Promise; - constructor(props: PlayerInit) { - this.media = new Media({ - canvas: props.canvas.transferControlToOffscreen(), - }) + transport: Transport - this.transport = new Transport({ - url: props.url, - fingerprint: props.fingerprint, - media: this.media, + constructor(config: Config) { + this.transport = config.transport + this.transport.callback = this; + + const video = { + canvas: config.canvas, + }; + + // Assume 44.1kHz and two audio channels + const audio = { + sampleRate: 44100, + ring: new Ring.Buffer(2, 4410), // 100ms at 44.1khz + } + + this.context = new AudioContext({ + latencyHint: "interactive", + sampleRate: audio.sampleRate, }) - } - async close() { - this.transport.close() - } + this.worker = this.setupWorker({ audio, video }) + this.worklet = this.setupWorklet(audio) + } - play() { - this.media.play({}) - } + private setupWorker(config: Message.Config): Worker { + const url = new URL('worker.ts', import.meta.url) + + const worker = new Worker(url, { + type: "module", + name: "media", + }) + + worker.postMessage({ config }, [ config.video.canvas ]) + + return worker + } + + private async setupWorklet(config: Message.AudioConfig): Promise { + // Load the worklet source code. + const url = new URL('worklet.ts', import.meta.url) + await this.context.audioWorklet.addModule(url) + + const volume = this.context.createGain() + volume.gain.value = 2.0; + + // Create a worklet + const worklet = new AudioWorkletNode(this.context, 'renderer'); + worklet.onprocessorerror = (e: Event) => { + console.error("Audio worklet error:", e) + }; + + worklet.port.postMessage({ config }) + + // Connect the worklet to the volume node and then to the speakers + worklet.connect(volume) + volume.connect(this.context.destination) + + return worklet + } + + onInit(init: Message.Init) { + this.worker.postMessage({ init }, [ init.buffer.buffer, init.reader ]) + } + + onSegment(segment: Message.Segment) { + this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ]) + } + + play(play: Message.Play) { + this.context.resume() + //this.worker.postMessage({ play }) + } } \ No newline at end of file diff --git a/player/src/media/message.ts b/player/src/player/message.ts similarity index 85% rename from player/src/media/message.ts rename to player/src/player/message.ts index a457200..dcad2e0 100644 --- a/player/src/media/message.ts +++ b/player/src/player/message.ts @@ -1,5 +1,4 @@ -import * as MP4 from "../mp4" -import { RingInit } from "../media/ring" +import * as Ring from "./ring" export interface Config { audio: AudioConfig; @@ -13,7 +12,7 @@ export interface VideoConfig { export interface AudioConfig { // audio stuff sampleRate: number; - ring: RingInit; + ring: Ring.Buffer; } export interface Init { diff --git a/player/src/media/renderer.ts b/player/src/player/renderer.ts similarity index 94% rename from player/src/media/renderer.ts rename to player/src/player/renderer.ts index af56e2a..d2d6c86 100644 --- a/player/src/media/renderer.ts +++ b/player/src/player/renderer.ts @@ -68,8 +68,11 @@ export default class Renderer { } draw(now: DOMHighResTimeStamp) { + // Convert to microseconds + now *= 1000; + // Determine the target timestamp. - const target = 1000 * now - this.sync! + const target = now - this.sync! this.drawAudio(now, target) this.drawVideo(now, target) @@ -85,9 +88,11 @@ export default class Renderer { // Check if we should skip some frames while (this.audioQueue.length) { const next = this.audioQueue[0] - if (next.timestamp >= target) { + + if (next.timestamp > target) { let ok = this.audioRing.write(next) if (!ok) { + console.warn("ring buffer is full") // No more space in the ring break } @@ -101,7 +106,7 @@ export default class Renderer { } drawVideo(now: DOMHighResTimeStamp, target: DOMHighResTimeStamp) { - if (this.videoQueue.length == 0) return; + if (!this.videoQueue.length) return; let frame = this.videoQueue[0]; if (frame.timestamp >= target) { diff --git a/player/src/media/ring.ts b/player/src/player/ring.ts similarity index 95% rename from player/src/media/ring.ts rename to player/src/player/ring.ts index cd796ee..ffc1c6b 100644 --- a/player/src/media/ring.ts +++ b/player/src/player/ring.ts @@ -11,15 +11,15 @@ export class Ring { channels: Float32Array[]; capacity: number; - constructor(init: RingInit) { - this.state = new Int32Array(init.state) + constructor(buf: Buffer) { + this.state = new Int32Array(buf.state) this.channels = [] - for (let channel of init.channels) { + for (let channel of buf.channels) { this.channels.push(new Float32Array(channel)) } - this.capacity = init.capacity + this.capacity = buf.capacity } // Add the samples for single audio frame @@ -121,7 +121,7 @@ export class Ring { } // No prototype to make this easier to send via postMessage -export class RingInit { +export class Buffer { state: SharedArrayBuffer; channels: SharedArrayBuffer[]; diff --git a/player/src/media/worker.ts b/player/src/player/worker.ts similarity index 100% rename from player/src/media/worker.ts rename to player/src/player/worker.ts diff --git a/player/src/media/worklet.ts b/player/src/player/worklet.ts similarity index 87% rename from player/src/media/worklet.ts rename to player/src/player/worklet.ts index 5961c32..4946bd8 100644 --- a/player/src/media/worklet.ts +++ b/player/src/player/worklet.ts @@ -20,12 +20,12 @@ class Renderer extends AudioWorkletProcessor { onMessage(e: MessageEvent) { if (e.data.config) { - this.config(e.data.config) + this.onConfig(e.data.config) } } - config(config: Message.Config) { - this.ring = new Ring(config.audio.ring) + onConfig(config: Message.AudioConfig) { + this.ring = new Ring(config.ring) } // Inputs and outputs in groups of 128 samples. diff --git a/player/src/transport/index.ts b/player/src/transport/index.ts index e588a8f..72eb4fd 100644 --- a/player/src/transport/index.ts +++ b/player/src/transport/index.ts @@ -1,25 +1,18 @@ -import * as Message from "./message" import * as Stream from "../stream" -import * as MP4 from "../mp4" +import * as Interface from "./interface" -import Media from "../media" - -export interface TransportInit { +export interface Config { url: string; fingerprint?: WebTransportHash; // the certificate fingerprint, temporarily needed for local development - media: Media; } export default class Transport { quic: Promise; api: Promise; + callback?: Interface.Callback; - media: Media; - - constructor(props: TransportInit) { - this.media = props.media; - - this.quic = this.connect(props) + constructor(config: Config) { + this.quic = this.connect(config) // Create a unidirectional stream for all of our messages this.api = this.quic.then((q) => { @@ -35,13 +28,13 @@ export default class Transport { } // Helper function to make creating a promise easier - private async connect(props: TransportInit): Promise { + private async connect(config: Config): Promise { let options: WebTransportOptions = {}; - if (props.fingerprint) { - options.serverCertificateHashes = [ props.fingerprint ] + if (config.fingerprint) { + options.serverCertificateHashes = [ config.fingerprint ] } - const quic = new WebTransport(props.url, options) + const quic = new WebTransport(config.url, options) await quic.ready return quic } @@ -86,12 +79,12 @@ export default class Transport { const msg = JSON.parse(payload) if (msg.init) { - return this.media.init({ + return this.callback?.onInit({ buffer: r.buffer, reader: r.reader, }) } else if (msg.segment) { - return this.media.segment({ + return this.callback?.onSegment({ buffer: r.buffer, reader: r.reader, }) diff --git a/player/src/transport/interface.ts b/player/src/transport/interface.ts new file mode 100644 index 0000000..cc5e3aa --- /dev/null +++ b/player/src/transport/interface.ts @@ -0,0 +1,14 @@ +export interface Callback { + onInit(init: Init): any + onSegment(segment: Segment): any +} + +export interface Init { + buffer: Uint8Array; // unread buffered data + reader: ReadableStream; // unread unbuffered data +} + +export interface Segment { + buffer: Uint8Array; // unread buffered data + reader: ReadableStream; // unread unbuffered data +} \ No newline at end of file diff --git a/player/src/transport/message.ts b/player/src/transport/message.ts index d151538..7cb511f 100644 --- a/player/src/transport/message.ts +++ b/player/src/transport/message.ts @@ -1,12 +1,5 @@ -export interface Init { - id: string -} - -export interface Segment { - init: string // id of the init segment - timestamp: number // presentation timestamp in milliseconds of the first sample - // TODO track would be nice -} +export interface Init {} +export interface Segment {} export interface Debug { max_bitrate: number From 29921ba46d4c67f08efaf7cecf64a25d100f79e2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 9 May 2023 09:06:29 -0600 Subject: [PATCH 13/23] Rename player folder and initial broadcaster code. --- README.md | 2 +- cert/generate | 2 +- {player => web}/.gitignore | 0 {player => web}/.proxyrc.js | 0 {player => web}/package.json | 0 web/src/broadcaster/encoder.ts | 104 ++++++++++++++++++ web/src/broadcaster/index.ts | 4 + {player => web}/src/index.css | 0 {player => web}/src/index.html | 0 {player => web}/src/index.ts | 0 {player => web}/src/mp4/index.ts | 0 {player => web}/src/mp4/init.ts | 0 {player => web}/src/mp4/mp4box.d.ts | 0 {player => web}/src/player/decoder.ts | 0 {player => web}/src/player/index.ts | 0 {player => web}/src/player/message.ts | 0 {player => web}/src/player/renderer.ts | 0 {player => web}/src/player/ring.ts | 0 {player => web}/src/player/worker.ts | 0 {player => web}/src/player/worklet.ts | 0 {player => web}/src/stream/index.ts | 0 {player => web}/src/stream/reader.ts | 0 {player => web}/src/stream/writer.ts | 0 {player => web}/src/transport/index.ts | 0 {player => web}/src/transport/interface.ts | 0 {player => web}/src/transport/message.ts | 0 .../src/transport/webtransport.d.ts | 0 {player => web}/src/util/deferred.ts | 0 {player => web}/src/util/index.ts | 0 {player => web}/tsconfig.json | 0 {player => web}/yarn.lock | 0 31 files changed, 110 insertions(+), 2 deletions(-) rename {player => web}/.gitignore (100%) rename {player => web}/.proxyrc.js (100%) rename {player => web}/package.json (100%) create mode 100644 web/src/broadcaster/encoder.ts create mode 100644 web/src/broadcaster/index.ts rename {player => web}/src/index.css (100%) rename {player => web}/src/index.html (100%) rename {player => web}/src/index.ts (100%) rename {player => web}/src/mp4/index.ts (100%) rename {player => web}/src/mp4/init.ts (100%) rename {player => web}/src/mp4/mp4box.d.ts (100%) rename {player => web}/src/player/decoder.ts (100%) rename {player => web}/src/player/index.ts (100%) rename {player => web}/src/player/message.ts (100%) rename {player => web}/src/player/renderer.ts (100%) rename {player => web}/src/player/ring.ts (100%) rename {player => web}/src/player/worker.ts (100%) rename {player => web}/src/player/worklet.ts (100%) rename {player => web}/src/stream/index.ts (100%) rename {player => web}/src/stream/reader.ts (100%) rename {player => web}/src/stream/writer.ts (100%) rename {player => web}/src/transport/index.ts (100%) rename {player => web}/src/transport/interface.ts (100%) rename {player => web}/src/transport/message.ts (100%) rename {player => web}/src/transport/webtransport.d.ts (100%) rename {player => web}/src/util/deferred.ts (100%) rename {player => web}/src/util/index.ts (100%) rename {player => web}/tsconfig.json (100%) rename {player => web}/yarn.lock (100%) diff --git a/README.md b/README.md index 22e9373..7af173b 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ This can be accessed via WebTransport on `https://localhost:4443` by default. The web assets need to be hosted with a HTTPS server. If you're using a self-signed certificate, you may need to ignore the security warning in Chrome (Advanced -> proceed to localhost). ``` -cd player +cd web yarn install yarn serve ``` diff --git a/cert/generate b/cert/generate index d89953d..bcbf9c9 100755 --- a/cert/generate +++ b/cert/generate @@ -17,4 +17,4 @@ go run filippo.io/mkcert -ecdsa -install go run filippo.io/mkcert -ecdsa -days 10 -cert-file "$CRT" -key-file "$KEY" localhost 127.0.0.1 ::1 # Compute the sha256 fingerprint of the certificate for WebTransport -openssl x509 -in "$CRT" -outform der | openssl dgst -sha256 > ../player/fingerprint.hex +openssl x509 -in "$CRT" -outform der | openssl dgst -sha256 > ../web/fingerprint.hex diff --git a/player/.gitignore b/web/.gitignore similarity index 100% rename from player/.gitignore rename to web/.gitignore diff --git a/player/.proxyrc.js b/web/.proxyrc.js similarity index 100% rename from player/.proxyrc.js rename to web/.proxyrc.js diff --git a/player/package.json b/web/package.json similarity index 100% rename from player/package.json rename to web/package.json diff --git a/web/src/broadcaster/encoder.ts b/web/src/broadcaster/encoder.ts new file mode 100644 index 0000000..91b96b6 --- /dev/null +++ b/web/src/broadcaster/encoder.ts @@ -0,0 +1,104 @@ +import * as MP4 from "../mp4" + +export class Encoder { + container: MP4.ISOFile + audio: AudioEncoder + video: VideoEncoder + + constructor() { + this.container = new MP4.ISOFile(); + + this.audio = new AudioEncoder({ + output: this.onAudio.bind(this), + error: console.warn, + }); + + this.video = new VideoEncoder({ + output: this.onVideo.bind(this), + error: console.warn, + }); + + this.container.init(); + + this.audio.configure({ + codec: "mp4a.40.2", + numberOfChannels: 2, + sampleRate: 44100, + + // TODO bitrate + }) + + this.video.configure({ + codec: "avc1.42002A", // TODO h.264 baseline + avc: { format: "avc" }, // or annexb + width: 1280, + height: 720, + + // TODO bitrate + // TODO bitrateMode + // TODO framerate + // TODO latencyMode + }) + } + + onAudio(frame: EncodedAudioChunk, metadata: EncodedAudioChunkMetadata) { + const config = metadata.decoderConfig! + const track_id = 1; + + if (!this.container.getTrackById(track_id)) { + this.container.addTrack({ + id: track_id, + type: "mp4a", // TODO wrong + timescale: 1000, // TODO verify + + channel_count: config.numberOfChannels, + samplerate: config.sampleRate, + + description: config.description, // TODO verify + // TODO description_boxes?: Box[]; + }); + } + + const buffer = new Uint8Array(frame.byteLength); + frame.copyTo(buffer); + + // TODO cts? + const sample = this.container.addSample(track_id, buffer, { + is_sync: frame.type == "key", + duration: frame.duration!, + dts: frame.timestamp, + }); + + const stream = this.container.createSingleSampleMoof(sample); + } + + onVideo(frame: EncodedVideoChunk, metadata?: EncodedVideoChunkMetadata) { + const config = metadata!.decoderConfig! + const track_id = 2; + + if (!this.container.getTrackById(track_id)) { + this.container.addTrack({ + id: 2, + type: "avc1", + width: config.codedWidth, + height: config.codedHeight, + timescale: 1000, // TODO verify + + description: config.description, // TODO verify + // TODO description_boxes?: Box[]; + }); + } + + const buffer = new Uint8Array(frame.byteLength); + frame.copyTo(buffer); + + // TODO cts? + const sample = this.container.addSample(track_id, buffer, { + is_sync: frame.type == "key", + duration: frame.duration!, + dts: frame.timestamp, + }); + + const stream = this.container.createSingleSampleMoof(sample); + } +} \ No newline at end of file diff --git a/web/src/broadcaster/index.ts b/web/src/broadcaster/index.ts new file mode 100644 index 0000000..39d50e6 --- /dev/null +++ b/web/src/broadcaster/index.ts @@ -0,0 +1,4 @@ +export default class Broadcaster { + constructor() { + } +} \ No newline at end of file diff --git a/player/src/index.css b/web/src/index.css similarity index 100% rename from player/src/index.css rename to web/src/index.css diff --git a/player/src/index.html b/web/src/index.html similarity index 100% rename from player/src/index.html rename to web/src/index.html diff --git a/player/src/index.ts b/web/src/index.ts similarity index 100% rename from player/src/index.ts rename to web/src/index.ts diff --git a/player/src/mp4/index.ts b/web/src/mp4/index.ts similarity index 100% rename from player/src/mp4/index.ts rename to web/src/mp4/index.ts diff --git a/player/src/mp4/init.ts b/web/src/mp4/init.ts similarity index 100% rename from player/src/mp4/init.ts rename to web/src/mp4/init.ts diff --git a/player/src/mp4/mp4box.d.ts b/web/src/mp4/mp4box.d.ts similarity index 100% rename from player/src/mp4/mp4box.d.ts rename to web/src/mp4/mp4box.d.ts diff --git a/player/src/player/decoder.ts b/web/src/player/decoder.ts similarity index 100% rename from player/src/player/decoder.ts rename to web/src/player/decoder.ts diff --git a/player/src/player/index.ts b/web/src/player/index.ts similarity index 100% rename from player/src/player/index.ts rename to web/src/player/index.ts diff --git a/player/src/player/message.ts b/web/src/player/message.ts similarity index 100% rename from player/src/player/message.ts rename to web/src/player/message.ts diff --git a/player/src/player/renderer.ts b/web/src/player/renderer.ts similarity index 100% rename from player/src/player/renderer.ts rename to web/src/player/renderer.ts diff --git a/player/src/player/ring.ts b/web/src/player/ring.ts similarity index 100% rename from player/src/player/ring.ts rename to web/src/player/ring.ts diff --git a/player/src/player/worker.ts b/web/src/player/worker.ts similarity index 100% rename from player/src/player/worker.ts rename to web/src/player/worker.ts diff --git a/player/src/player/worklet.ts b/web/src/player/worklet.ts similarity index 100% rename from player/src/player/worklet.ts rename to web/src/player/worklet.ts diff --git a/player/src/stream/index.ts b/web/src/stream/index.ts similarity index 100% rename from player/src/stream/index.ts rename to web/src/stream/index.ts diff --git a/player/src/stream/reader.ts b/web/src/stream/reader.ts similarity index 100% rename from player/src/stream/reader.ts rename to web/src/stream/reader.ts diff --git a/player/src/stream/writer.ts b/web/src/stream/writer.ts similarity index 100% rename from player/src/stream/writer.ts rename to web/src/stream/writer.ts diff --git a/player/src/transport/index.ts b/web/src/transport/index.ts similarity index 100% rename from player/src/transport/index.ts rename to web/src/transport/index.ts diff --git a/player/src/transport/interface.ts b/web/src/transport/interface.ts similarity index 100% rename from player/src/transport/interface.ts rename to web/src/transport/interface.ts diff --git a/player/src/transport/message.ts b/web/src/transport/message.ts similarity index 100% rename from player/src/transport/message.ts rename to web/src/transport/message.ts diff --git a/player/src/transport/webtransport.d.ts b/web/src/transport/webtransport.d.ts similarity index 100% rename from player/src/transport/webtransport.d.ts rename to web/src/transport/webtransport.d.ts diff --git a/player/src/util/deferred.ts b/web/src/util/deferred.ts similarity index 100% rename from player/src/util/deferred.ts rename to web/src/util/deferred.ts diff --git a/player/src/util/index.ts b/web/src/util/index.ts similarity index 100% rename from player/src/util/index.ts rename to web/src/util/index.ts diff --git a/player/tsconfig.json b/web/tsconfig.json similarity index 100% rename from player/tsconfig.json rename to web/tsconfig.json diff --git a/player/yarn.lock b/web/yarn.lock similarity index 100% rename from player/yarn.lock rename to web/yarn.lock From 28f5b973083cad5b42def2f78c1a1c4402bb6f42 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 9 May 2023 10:29:39 -0600 Subject: [PATCH 14/23] Implement prioritization. Not tested. --- server/src/media/source.rs | 8 +- server/src/session/mod.rs | 11 ++- server/src/transport/streams.rs | 128 ++++++++++++++++++++++---------- 3 files changed, 103 insertions(+), 44 deletions(-) diff --git a/server/src/media/source.rs b/server/src/media/source.rs index e639ce6..b8579bc 100644 --- a/server/src/media/source.rs +++ b/server/src/media/source.rs @@ -35,7 +35,7 @@ pub struct Fragment { pub keyframe: bool, // The timestamp of the fragment, in milliseconds, to simulate a live stream. - pub timestamp: Option, + pub timestamp: u64, } impl Source { @@ -107,7 +107,7 @@ impl Source { track_id: moof.trafs[0].tfhd.track_id, data: atom, keyframe: has_keyframe(&moof), - timestamp: first_timestamp(&moof), + timestamp: first_timestamp(&moof).expect("couldn't find timestamp"), }) } mp4::BoxType::MdatBox => { @@ -117,7 +117,7 @@ impl Source { track_id: moof.track_id, data: atom, keyframe: false, - timestamp: None, + timestamp: moof.timestamp, }); // We have some media data, return so we can start sending it. @@ -133,7 +133,7 @@ impl Source { // Simulate a live stream by sleeping until the next timestamp in the media. pub fn timeout(&self) -> Option { let next = self.fragments.front()?; - let timestamp = next.timestamp?; + let timestamp = next.timestamp; // Find the timescale for the track. let timescale = self.timescale.get(&next.track_id).unwrap(); diff --git a/server/src/session/mod.rs b/server/src/session/mod.rs index 0d59e8e..88360b7 100644 --- a/server/src/session/mod.rs +++ b/server/src/session/mod.rs @@ -120,11 +120,17 @@ impl Session { // Open a new stream. None => { + // Create a new unidirectional stream. let stream_id = session.open_stream(conn, false)?; - // TODO: conn.stream_priority(stream_id, urgency, incremental) + + // Set the stream priority to be equal to the timestamp. + // We subtract from u64::MAX so newer media is sent important. + // TODO prioritize audio + let order = u64::MAX - fragment.timestamp; + self.streams.send_order(conn, stream_id, order); // Encode a JSON header indicating this is a new track. - let mut message = message::Message::new(); + let mut message: message::Message = message::Message::new(); message.segment = Some(message::Segment { track_id: fragment.track_id, }); @@ -133,6 +139,7 @@ impl Session { let data = message.serialize()?; self.streams.send(conn, stream_id, &data, false)?; + // Keep a mapping from the track id to the current stream id. self.tracks.insert(fragment.track_id, stream_id); stream_id diff --git a/server/src/transport/streams.rs b/server/src/transport/streams.rs index 215731b..ccb929b 100644 --- a/server/src/transport/streams.rs +++ b/server/src/transport/streams.rs @@ -1,4 +1,3 @@ -use std::collections::hash_map as hmap; use std::collections::VecDeque; use anyhow; @@ -6,16 +5,19 @@ use quiche; #[derive(Default)] pub struct Streams { - lookup: hmap::HashMap, + ordered: Vec, } -#[derive(Default)] -struct State { +struct Stream { + id: u64, + order: u64, + buffer: VecDeque, fin: bool, } impl Streams { + // Write the data to the given stream, buffering it if needed. pub fn send( &mut self, conn: &mut quiche::Connection, @@ -23,63 +25,113 @@ impl Streams { buf: &[u8], fin: bool, ) -> anyhow::Result<()> { - match self.lookup.entry(id) { - hmap::Entry::Occupied(mut entry) => { - // Add to the existing buffer. - let state = entry.get_mut(); - state.buffer.extend(buf); - state.fin |= fin; - } - hmap::Entry::Vacant(entry) => { - let size = conn.stream_send(id, buf, fin)?; + if buf.is_empty() && !fin { + return Ok(()) + } - if size < buf.len() { - // Short write, save the rest for later. - let mut buffer = VecDeque::with_capacity(buf.len()); - buffer.extend(&buf[size..]); + // Get the index of the stream, or add it to the list of streams. + let pos = self.ordered.iter().position(|s| s.id == id).unwrap_or_else(|| { + // Create a new stream + let stream = Stream{ + id, + buffer: VecDeque::new(), + fin: false, + order: 0, // Default to highest priority until send_order is called. + }; - entry.insert(State { buffer, fin }); - } - } + self.insert(conn, stream) + }); + + let stream = &mut self.ordered[pos]; + + // Check if we've already closed the stream, just in case. + if stream.fin && !buf.is_empty() { + anyhow::bail!("stream is already finished"); + } + + // If there's no data buffered, try to write it immediately. + let size = if stream.buffer.is_empty() { + conn.stream_send(id, buf, fin)? + } else { + 0 }; + if size < buf.len() { + // Short write, save the rest for later. + stream.buffer.extend(&buf[size..]); + } + + stream.fin |= fin; + Ok(()) } + // Flush any pending stream data. pub fn poll(&mut self, conn: &mut quiche::Connection) -> anyhow::Result<()> { - 'outer: for id in conn.writable() { - // Check if there's any buffered data for this stream. - let mut entry = match self.lookup.entry(id) { - hmap::Entry::Occupied(entry) => entry, - hmap::Entry::Vacant(_) => continue, - }; - - let state = entry.get_mut(); - + // Loop over stream in order order. + 'outer: for stream in self.ordered.iter_mut() { // Keep reading from the buffer until it's empty. - while !state.buffer.is_empty() { + while !stream.buffer.is_empty() { // VecDeque is a ring buffer, so we can't write the whole thing at once. - let parts = state.buffer.as_slices(); + let parts = stream.buffer.as_slices(); - let size = conn.stream_send(id, parts.0, false)?; + let size = conn.stream_send(stream.id, parts.0, false)?; if size == 0 { // No more space available for this stream. continue 'outer; } // Remove the bytes that were written. - state.buffer.drain(..size); + stream.buffer.drain(..size); } - if state.fin { + if stream.fin { // Write the stream done signal. - conn.stream_send(id, &[], true)?; + conn.stream_send(stream.id, &[], true)?; } - - // We can remove the value from the lookup once we've flushed everything. - entry.remove(); } + // Remove streams that are done. + // No need to reprioritize, since the streams are still in order order. + self.ordered.retain(|stream| !stream.buffer.is_empty() || !stream.fin); + Ok(()) } + + // Set the send order of the stream. + pub fn send_order(&mut self, conn: &mut quiche::Connection, id: u64, order: u64) { + let mut stream = match self.ordered.iter().position(|s| s.id == id) { + // Remove the stream from the existing list. + Some(pos) => self.ordered.remove(pos), + + // This is a new stream, insert it into the list. + None => Stream{ + id, + buffer: VecDeque::new(), + fin: false, + order, + }, + }; + + stream.order = order; + + self.insert(conn, stream); + } + + fn insert(&mut self, conn: &mut quiche::Connection, stream: Stream) -> usize { + // Look for the position to insert the stream. + let pos = match self.ordered.binary_search_by_key(&stream.order, |s| s.order) { + Ok(pos) | Err(pos) => pos, + }; + + self.ordered.insert(pos, stream); + + // Reprioritize all later streams. + // TODO we can avoid this if stream_priorty takes a u64 + for (i, stream) in self.ordered[pos..].iter().enumerate() { + _ = conn.stream_priority(stream.id, (pos+i) as u8, true); + } + + pos + } } From 0f4d823d399d632ee1274bd66a2a163bf5fad589 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 9 May 2023 14:24:14 -0600 Subject: [PATCH 15/23] cargo fmt --- server/src/media/mod.rs | 2 +- server/src/media/source.rs | 29 ++++++++++++++----------- server/src/session/mod.rs | 16 ++++++-------- server/src/transport/streams.rs | 38 ++++++++++++++++++++------------- 4 files changed, 47 insertions(+), 38 deletions(-) diff --git a/server/src/media/mod.rs b/server/src/media/mod.rs index f73c588..e7ec5eb 100644 --- a/server/src/media/mod.rs +++ b/server/src/media/mod.rs @@ -1,3 +1,3 @@ mod source; -pub use source::{Fragment, Source}; \ No newline at end of file +pub use source::{Fragment, Source}; diff --git a/server/src/media/source.rs b/server/src/media/source.rs index b8579bc..ebbb5e4 100644 --- a/server/src/media/source.rs +++ b/server/src/media/source.rs @@ -1,6 +1,6 @@ -use std::{fs, io, time}; -use std::collections::{HashMap,VecDeque}; +use std::collections::{HashMap, VecDeque}; use std::io::Read; +use std::{fs, io, time}; use anyhow; @@ -18,7 +18,7 @@ pub struct Source { pub init: Vec, // The timescale used for each track. - timescale: HashMap, + timescales: HashMap, // Any fragments parsed and ready to be returned by next(). fragments: VecDeque, @@ -60,16 +60,12 @@ impl Source { // Parse the moov box so we can detect the timescales for each track. let moov = mp4::MoovBox::read_box(&mut moov_reader, moov_header.size)?; - let timescale = moov.traks - .iter() - .map(|trak| (trak.tkhd.track_id, trak.mdia.mdhd.timescale)) - .collect(); - Ok(Self{ + Ok(Self { reader, start, init, - timescale, + timescales: timescales(&moov), fragments: VecDeque::new(), }) } @@ -94,7 +90,9 @@ impl Source { let header = mp4::BoxHeader::read(&mut reader)?; match header.name { - mp4::BoxType::FtypBox | mp4::BoxType::MoovBox => anyhow::bail!("must call init first"), + mp4::BoxType::FtypBox | mp4::BoxType::MoovBox => { + anyhow::bail!("must call init first") + } mp4::BoxType::MoofBox => { let moof = mp4::MoofBox::read_box(&mut reader, header.size)?; @@ -136,7 +134,7 @@ impl Source { let timestamp = next.timestamp; // Find the timescale for the track. - let timescale = self.timescale.get(&next.track_id).unwrap(); + let timescale = self.timescales.get(&next.track_id).unwrap(); let delay = time::Duration::from_millis(1000 * timestamp / *timescale as u64); let elapsed = self.start.elapsed(); @@ -222,4 +220,11 @@ fn has_keyframe(moof: &mp4::MoofBox) -> bool { fn first_timestamp(moof: &mp4::MoofBox) -> Option { Some(moof.trafs.first()?.tfdt.as_ref()?.base_media_decode_time) -} \ No newline at end of file +} + +fn timescales(moov: &mp4::MoovBox) -> HashMap { + moov.traks + .iter() + .map(|trak| (trak.tkhd.track_id, trak.mdia.mdhd.timescale)) + .collect() +} diff --git a/server/src/session/mod.rs b/server/src/session/mod.rs index 88360b7..b3b38e7 100644 --- a/server/src/session/mod.rs +++ b/server/src/session/mod.rs @@ -1,7 +1,7 @@ mod message; -use std::time; use std::collections::hash_map as hmap; +use std::time; use quiche; use quiche::h3::webtransport; @@ -29,11 +29,8 @@ impl transport::App for Session { Ok(e) => e, }; - log::debug!("webtransport event: {:?}", event); - match event { - webtransport::ServerEvent::ConnectRequest(req) => { - log::debug!("new connect {:?}", req); + webtransport::ServerEvent::ConnectRequest(_req) => { // you can handle request with // req.authority() // req.path() @@ -46,7 +43,7 @@ impl transport::App for Session { // Create a JSON header. let mut message = message::Message::new(); - message.init = Some(message::Init{}); + message.init = Some(message::Init {}); let data = message.serialize()?; // Create a new stream and write the header. @@ -59,8 +56,7 @@ impl transport::App for Session { webtransport::ServerEvent::StreamData(stream_id) => { let mut buf = vec![0; 10000]; while let Ok(len) = session.recv_stream_data(conn, stream_id, &mut buf) { - let stream_data = &buf[0..len]; - log::debug!("stream data {:?}", stream_data); + let _stream_data = &buf[0..len]; } } @@ -105,7 +101,7 @@ impl Session { Some(stream_id) if fragment.keyframe => { self.streams.send(conn, *stream_id, &[], true)?; None - }, + } // Use the existing stream Some(stream_id) => Some(*stream_id), @@ -143,7 +139,7 @@ impl Session { self.tracks.insert(fragment.track_id, stream_id); stream_id - }, + } }; // Write the current fragment. diff --git a/server/src/transport/streams.rs b/server/src/transport/streams.rs index ccb929b..149e5de 100644 --- a/server/src/transport/streams.rs +++ b/server/src/transport/streams.rs @@ -26,21 +26,25 @@ impl Streams { fin: bool, ) -> anyhow::Result<()> { if buf.is_empty() && !fin { - return Ok(()) + return Ok(()); } // Get the index of the stream, or add it to the list of streams. - let pos = self.ordered.iter().position(|s| s.id == id).unwrap_or_else(|| { - // Create a new stream - let stream = Stream{ - id, - buffer: VecDeque::new(), - fin: false, - order: 0, // Default to highest priority until send_order is called. - }; + let pos = self + .ordered + .iter() + .position(|s| s.id == id) + .unwrap_or_else(|| { + // Create a new stream + let stream = Stream { + id, + buffer: VecDeque::new(), + fin: false, + order: 0, // Default to highest priority until send_order is called. + }; - self.insert(conn, stream) - }); + self.insert(conn, stream) + }); let stream = &mut self.ordered[pos]; @@ -93,7 +97,8 @@ impl Streams { // Remove streams that are done. // No need to reprioritize, since the streams are still in order order. - self.ordered.retain(|stream| !stream.buffer.is_empty() || !stream.fin); + self.ordered + .retain(|stream| !stream.buffer.is_empty() || !stream.fin); Ok(()) } @@ -105,7 +110,7 @@ impl Streams { Some(pos) => self.ordered.remove(pos), // This is a new stream, insert it into the list. - None => Stream{ + None => Stream { id, buffer: VecDeque::new(), fin: false, @@ -120,7 +125,10 @@ impl Streams { fn insert(&mut self, conn: &mut quiche::Connection, stream: Stream) -> usize { // Look for the position to insert the stream. - let pos = match self.ordered.binary_search_by_key(&stream.order, |s| s.order) { + let pos = match self + .ordered + .binary_search_by_key(&stream.order, |s| s.order) + { Ok(pos) | Err(pos) => pos, }; @@ -129,7 +137,7 @@ impl Streams { // Reprioritize all later streams. // TODO we can avoid this if stream_priorty takes a u64 for (i, stream) in self.ordered[pos..].iter().enumerate() { - _ = conn.stream_priority(stream.id, (pos+i) as u8, true); + _ = conn.stream_priority(stream.id, (pos + i) as u8, true); } pos From 4675c27179428a737b73ede8b44fb57a876ab0da Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 16 May 2023 10:23:50 -0700 Subject: [PATCH 16/23] Make a docker-compose ez mode. --- README.md | 69 ++++++++----------------- cert/.dockerignore | 3 ++ cert/.gitignore | 1 + cert/Dockerfile | 20 ++++++++ cert/generate | 2 +- docker-compose.yml | 45 ++++++++++++++++ media/.dockerignore | 1 + media/Dockerfile | 20 ++++++++ media/{generate => fragment} | 0 server/.dockerignore | 1 + server/Cargo.lock | 6 ++- server/Cargo.toml | 6 +-- server/Dockerfile | 42 +++++++++++++++ server/src/main.rs | 2 +- server/src/session/mod.rs | 9 ++-- server/src/transport/server.rs | 4 ++ server/src/transport/streams.rs | 66 +++++++++++++----------- web/.dockerignore | 4 ++ web/.eslintrc.cjs | 13 +++++ web/Dockerfile | 26 ++++++++++ web/package.json | 5 +- web/src/index.ts | 6 +-- web/src/player/audio.ts | 75 +++++++++++++++++++++++++++ web/src/player/video.ts | 91 +++++++++++++++++++++++++++++++++ web/src/transport/index.ts | 4 +- 25 files changed, 425 insertions(+), 96 deletions(-) create mode 100644 cert/.dockerignore create mode 100644 cert/Dockerfile create mode 100644 docker-compose.yml create mode 100644 media/.dockerignore create mode 100644 media/Dockerfile rename media/{generate => fragment} (100%) create mode 100644 server/.dockerignore create mode 100644 server/Dockerfile create mode 100644 web/.dockerignore create mode 100644 web/.eslintrc.cjs create mode 100644 web/Dockerfile create mode 100644 web/src/player/audio.ts create mode 100644 web/src/player/video.ts diff --git a/README.md b/README.md index 7af173b..43e6dc6 100644 --- a/README.md +++ b/README.md @@ -1,64 +1,39 @@ # Warp -Segmented live media delivery protocol utilizing QUIC streams. See the [Warp draft](https://datatracker.ietf.org/doc/draft-lcurley-warp/). +Live media delivery protocol utilizing QUIC streams. See the [Warp draft](https://datatracker.ietf.org/doc/draft-lcurley-warp/). -Warp works by delivering each audio and video segment as a separate QUIC stream. These streams are assigned a priority such that old video will arrive last and can be dropped. This avoids buffering in many cases, offering the viewer a potentially better experience. +Warp works by delivering media over independent QUIC stream. These streams are assigned a priority such that old video will arrive last and can be dropped. This avoids buffering in many cases, offering the viewer a potentially better experience. -# Limitations -## Browser Support -This demo currently only works on Chrome for two reasons: +This demo requires WebTransport and WebCodecs, which currently (May 2023) only works on Chrome. -1. WebTransport support. -2. [Media underflow behavior](https://github.com/whatwg/html/issues/6359). +# Development +## Easy Mode +Requires Docker *only*. -The ability to skip video abuses the fact that Chrome can play audio without video for up to 3 seconds (hardcoded!) when using MSE. It is possible to use something like WebCodecs instead... but that's still Chrome only at the moment. +``` +docker-compose up --build +``` -## Streaming -This demo works by reading pre-encoded media and sleeping based on media timestamps. Obviously this is not a live stream; you should plug in your own encoder or source. +Then open [https://localhost:4444/](https://localhost:4444) in a browser. You'll have to click past the TLS error, but that's the price you pay for being lazy. Follow the more in-depth instructions if you want a better development experience. -The media is encoded on disk as a LL-DASH playlist. There's a crude parser and I haven't used DASH before so don't expect it to work with arbitrary inputs. - -## QUIC Implementation -This demo uses a fork of [quic-go](https://github.com/lucas-clemente/quic-go). There are two critical features missing upstream: - -1. ~~[WebTransport](https://github.com/lucas-clemente/quic-go/issues/3191)~~ -2. [Prioritization](https://github.com/lucas-clemente/quic-go/pull/3442) - -## Congestion Control -This demo uses a single rendition. A production implementation will want to: - -1. Change the rendition bitrate to match the estimated bitrate. -2. Switch renditions at segment boundaries based on the estimated bitrate. -3. or both! - -Also, quic-go ships with the default New Reno congestion control. Something like [BBRv2](https://github.com/lucas-clemente/quic-go/issues/341) will work much better for live video as it limits RTT growth. - - -# Setup ## Requirements * Go +* Rust * ffmpeg * openssl -* Chrome Canary +* Chrome ## Media This demo simulates a live stream by reading a file from disk and sleeping based on media timestamps. Obviously you should hook this up to a real live stream to do anything useful. -Download your favorite media file: +Download your favorite media file and convert it to fragmented MP4: ``` wget http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4 -O media/source.mp4 +./media/fragment ``` -Use ffmpeg to create a LL-DASH playlist. This creates a segment every 2s and MP4 fragment every 10ms. -``` -./media/generate -``` - -You can increase the `frag_duration` (microseconds) to slightly reduce the file size in exchange for higher latency. - -## TLS +## Certificates Unfortunately, QUIC mandates TLS and makes local development difficult. - -If you have a valid certificate you can use it instead of self-signing. The go binaries take a `-tls-cert` and `-tls-key` argument. Skip the remaining steps in this section and use your hostname instead. +If you have a valid certificate you can use it instead of self-signing. Otherwise, we use [mkcert](https://github.com/FiloSottile/mkcert) to install a self-signed CA: ``` @@ -72,13 +47,13 @@ The Warp server supports WebTransport, pushing media over streams once a connect ``` cd server -go run main.go +cargo run ``` -This can be accessed via WebTransport on `https://localhost:4443` by default. +This listens for WebTransport connections (not HTTP) on `https://localhost:4443` by default. -## Web Player -The web assets need to be hosted with a HTTPS server. If you're using a self-signed certificate, you may need to ignore the security warning in Chrome (Advanced -> proceed to localhost). +## Web +The web assets need to be hosted with a HTTPS server. ``` cd web @@ -86,6 +61,4 @@ yarn install yarn serve ``` -These can be accessed on `https://localhost:4444` by default. - -If you use a custom domain for the Warp server, make sure to override the server URL with the `url` query string parameter, e.g. `https://localhost:4444/?url=https://warp.demo`. +These can be accessed on `https://localhost:4444` by default. \ No newline at end of file diff --git a/cert/.dockerignore b/cert/.dockerignore new file mode 100644 index 0000000..9879661 --- /dev/null +++ b/cert/.dockerignore @@ -0,0 +1,3 @@ +*.crt +*.key +*.hex \ No newline at end of file diff --git a/cert/.gitignore b/cert/.gitignore index be870b4..9879661 100644 --- a/cert/.gitignore +++ b/cert/.gitignore @@ -1,2 +1,3 @@ *.crt *.key +*.hex \ No newline at end of file diff --git a/cert/Dockerfile b/cert/Dockerfile new file mode 100644 index 0000000..84cb44c --- /dev/null +++ b/cert/Dockerfile @@ -0,0 +1,20 @@ +# Use ubuntu because it's ez +FROM ubuntu:latest + +# Use openssl and golang to generate certificates +RUN apt-get update && \ + apt-get install -y ca-certificates openssl golang + +# Save the certificates to a volume +VOLUME /cert +WORKDIR /cert + +# Download the go modules +COPY go.mod go.sum ./ +RUN go mod download + +# Copy over the remaining files. +COPY . . + +# TODO support an output directory +CMD ./generate \ No newline at end of file diff --git a/cert/generate b/cert/generate index bcbf9c9..c86b309 100755 --- a/cert/generate +++ b/cert/generate @@ -17,4 +17,4 @@ go run filippo.io/mkcert -ecdsa -install go run filippo.io/mkcert -ecdsa -days 10 -cert-file "$CRT" -key-file "$KEY" localhost 127.0.0.1 ::1 # Compute the sha256 fingerprint of the certificate for WebTransport -openssl x509 -in "$CRT" -outform der | openssl dgst -sha256 > ../web/fingerprint.hex +openssl x509 -in "$CRT" -outform der | openssl dgst -sha256 > localhost.hex \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..c2ad05d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,45 @@ +version: '3' + +services: + # Generate certificates only valid for 14 days. + cert: + build: ./cert + volumes: + - cert:/cert + + # Generate a fragmented MP4 file for testing. + media: + build: ./media + volumes: + - media:/media + + # Serve the web code once we have certificates. + web: + build: ./web + ports: + - "4444:4444" + volumes: + - cert:/cert + depends_on: + cert: + condition: service_completed_successfully + + # Run the server once we have certificates and media. + server: + build: ./server + environment: + - RUST_LOG=debug + ports: + - "4443:4443/udp" + volumes: + - cert:/cert + - media:/media + depends_on: + cert: + condition: service_completed_successfully + media: + condition: service_completed_successfully + +volumes: + cert: + media: diff --git a/media/.dockerignore b/media/.dockerignore new file mode 100644 index 0000000..fa48555 --- /dev/null +++ b/media/.dockerignore @@ -0,0 +1 @@ +fragmented.mp4 diff --git a/media/Dockerfile b/media/Dockerfile new file mode 100644 index 0000000..f8f36e4 --- /dev/null +++ b/media/Dockerfile @@ -0,0 +1,20 @@ +# Create a build image +FROM ubuntu:latest + +# Install necessary packages +RUN apt-get update && \ + apt-get install -y \ + ca-certificates \ + wget \ + ffmpeg + +# Create a media volume +VOLUME /media +WORKDIR /media + +# Download a file from the internet, in this case my boy big buck bunny +RUN wget http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4 -O source.mp4 + +# Copy an run a script to create a fragmented mp4 (more overhead, easier to split) +COPY fragment . +CMD ./fragment diff --git a/media/generate b/media/fragment similarity index 100% rename from media/generate rename to media/fragment diff --git a/server/.dockerignore b/server/.dockerignore new file mode 100644 index 0000000..eb5a316 --- /dev/null +++ b/server/.dockerignore @@ -0,0 +1 @@ +target diff --git a/server/Cargo.lock b/server/Cargo.lock index 0ae81ac..88d44d5 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -329,6 +329,8 @@ dependencies = [ [[package]] name = "mp4" version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "509348cba250e7b852a875100a2ddce7a36ee3abf881a681c756670c1774264d" dependencies = [ "byteorder", "bytes", @@ -384,7 +386,7 @@ dependencies = [ [[package]] name = "octets" version = "0.2.0" -source = "git+https://github.com/n8o/quiche.git?branch=master#0137dc3ca6f4f31e3175d0a0868acb9c64b46cc7" +source = "git+https://github.com/kixelated/quiche.git?branch=master#007a25b35b9509d673466fed8ddc73fd8d9b4184" [[package]] name = "once_cell" @@ -404,7 +406,7 @@ dependencies = [ [[package]] name = "quiche" version = "0.17.1" -source = "git+https://github.com/n8o/quiche.git?branch=master#0137dc3ca6f4f31e3175d0a0868acb9c64b46cc7" +source = "git+https://github.com/kixelated/quiche.git?branch=master#007a25b35b9509d673466fed8ddc73fd8d9b4184" dependencies = [ "cmake", "lazy_static", diff --git a/server/Cargo.toml b/server/Cargo.toml index af201c6..1d04ec0 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -6,13 +6,13 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -quiche = { git = "https://github.com/n8o/quiche.git", branch = "master" } # WebTransport fork +quiche = { git = "https://github.com/kixelated/quiche.git", branch = "master" } # WebTransport fork clap = { version = "4.0", features = [ "derive" ] } log = { version = "0.4", features = ["std"] } mio = { version = "0.8", features = ["net", "os-poll"] } env_logger = "0.9.3" ring = "0.16" anyhow = "1.0.70" -mp4 = { path = "../../mp4-rust" } # { git = "https://github.com/kixelated/mp4-rust.git", branch = "trexs" } +mp4 = "0.13.0" serde = "1.0.160" -serde_json = "1.0" \ No newline at end of file +serde_json = "1.0" diff --git a/server/Dockerfile b/server/Dockerfile new file mode 100644 index 0000000..b903dc9 --- /dev/null +++ b/server/Dockerfile @@ -0,0 +1,42 @@ +# Use the official Rust image as the base image +FROM rust:latest as build + +# Quiche requires docker +RUN apt-get update && \ + apt-get install -y cmake + +# Set the build directory +WORKDIR /warp + +# Create an empty project +RUN cargo init --bin + +# Copy the Cargo.toml and Cargo.lock files to the container +COPY Cargo.toml Cargo.lock ./ + +# Build the empty project so we download/cache dependencies +RUN cargo build --release + +# Copy the entire project to the container +COPY . . + +# Build the project +RUN cargo build --release + +# Make a new image to run the binary +FROM ubuntu:latest + +# Use a volume to access certificates +VOLUME /cert + +# Use another volume to access the media +VOLUME /media + +# Expose port 4443 for the server +EXPOSE 4443/udp + +# Copy the built binary +COPY --from=build /warp/target/release/warp /bin + +# Set the startup command to run the binary +CMD warp --cert /cert/localhost.crt --key /cert/localhost.key --media /media/fragmented.mp4 \ No newline at end of file diff --git a/server/src/main.rs b/server/src/main.rs index 3d4d422..56441d7 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -6,7 +6,7 @@ use clap::Parser; #[derive(Parser)] struct Cli { /// Listen on this address - #[arg(short, long, default_value = "127.0.0.1:4443")] + #[arg(short, long, default_value = "0.0.0.0:4443")] addr: String, /// Use the certificate file at this path diff --git a/server/src/session/mod.rs b/server/src/session/mod.rs index b3b38e7..0b4af65 100644 --- a/server/src/session/mod.rs +++ b/server/src/session/mod.rs @@ -29,13 +29,15 @@ impl transport::App for Session { Ok(e) => e, }; + log::debug!("webtransport event {:?}", event); + match event { webtransport::ServerEvent::ConnectRequest(_req) => { // you can handle request with // req.authority() // req.path() // and you can validate this request with req.origin() - session.accept_connect_request(conn, None).unwrap(); + session.accept_connect_request(conn, None)?; // TODO let media = media::Source::new("../media/fragmented.mp4")?; @@ -65,10 +67,11 @@ impl transport::App for Session { } // Send any pending stream data. - self.streams.poll(conn)?; + // NOTE: This doesn't return an error because it's async, and would be confusing. + self.streams.poll(conn); // Fetch the next media fragment, possibly queuing up stream data. - self.poll_source(conn, session)?; + self.poll_source(conn, session).expect("poll_source"); Ok(()) } diff --git a/server/src/transport/server.rs b/server/src/transport/server.rs index 202d526..d0ab1f7 100644 --- a/server/src/transport/server.rs +++ b/server/src/transport/server.rs @@ -78,6 +78,8 @@ impl Server { } pub fn run(&mut self) -> anyhow::Result<()> { + log::info!("listening on {}", self.socket.local_addr()?); + loop { self.wait()?; self.receive()?; @@ -253,6 +255,8 @@ impl Server { for conn in self.conns.values_mut() { if let Some(session) = &mut conn.session { if let Err(e) = conn.app.poll(&mut conn.quiche, session) { + log::debug!("app error: {:?}", e); + // Close the connection on any application error let reason = format!("app error: {:?}", e); conn.quiche.close(true, 0xff, reason.as_bytes()).ok(); diff --git a/server/src/transport/streams.rs b/server/src/transport/streams.rs index 149e5de..de9bdd9 100644 --- a/server/src/transport/streams.rs +++ b/server/src/transport/streams.rs @@ -55,7 +55,11 @@ impl Streams { // If there's no data buffered, try to write it immediately. let size = if stream.buffer.is_empty() { - conn.stream_send(id, buf, fin)? + match conn.stream_send(id, buf, fin) { + Ok(size) => size, + Err(quiche::Error::Done) => 0, + Err(e) => anyhow::bail!(e), + } } else { 0 }; @@ -71,36 +75,8 @@ impl Streams { } // Flush any pending stream data. - pub fn poll(&mut self, conn: &mut quiche::Connection) -> anyhow::Result<()> { - // Loop over stream in order order. - 'outer: for stream in self.ordered.iter_mut() { - // Keep reading from the buffer until it's empty. - while !stream.buffer.is_empty() { - // VecDeque is a ring buffer, so we can't write the whole thing at once. - let parts = stream.buffer.as_slices(); - - let size = conn.stream_send(stream.id, parts.0, false)?; - if size == 0 { - // No more space available for this stream. - continue 'outer; - } - - // Remove the bytes that were written. - stream.buffer.drain(..size); - } - - if stream.fin { - // Write the stream done signal. - conn.stream_send(stream.id, &[], true)?; - } - } - - // Remove streams that are done. - // No need to reprioritize, since the streams are still in order order. - self.ordered - .retain(|stream| !stream.buffer.is_empty() || !stream.fin); - - Ok(()) + pub fn poll(&mut self, conn: &mut quiche::Connection) { + self.ordered.retain_mut(|s| s.poll(conn).is_ok()); } // Set the send order of the stream. @@ -143,3 +119,31 @@ impl Streams { pos } } + +impl Stream { + fn poll(&mut self, conn: &mut quiche::Connection) -> quiche::Result<()> { + // Keep reading from the buffer until it's empty. + while !self.buffer.is_empty() { + // VecDeque is a ring buffer, so we can't write the whole thing at once. + let parts = self.buffer.as_slices(); + + let size = conn.stream_send(self.id, parts.0, false)?; + if size == 0 { + // No more space available for this stream. + return Ok(()); + } + + // Remove the bytes that were written. + self.buffer.drain(..size); + } + + if self.fin { + // Write the stream done signal. + conn.stream_send(self.id, &[], true)?; + + Err(quiche::Error::Done) + } else { + Ok(()) + } + } +} diff --git a/web/.dockerignore b/web/.dockerignore new file mode 100644 index 0000000..93e58d7 --- /dev/null +++ b/web/.dockerignore @@ -0,0 +1,4 @@ +dist +.parcel-cache +node_modules +fingerprint.hex \ No newline at end of file diff --git a/web/.eslintrc.cjs b/web/.eslintrc.cjs new file mode 100644 index 0000000..278f910 --- /dev/null +++ b/web/.eslintrc.cjs @@ -0,0 +1,13 @@ +/* eslint-env node */ +module.exports = { + extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended'], + parser: '@typescript-eslint/parser', + plugins: ['@typescript-eslint'], + root: true, + ignorePatterns: [ 'dist', 'node_modules' ], + rules: { + "@typescript-eslint/ban-ts-comment": "off", + "@typescript-eslint/no-non-null-assertion": "off", + "@typescript-eslint/no-explicit-any": "off", + } +}; diff --git a/web/Dockerfile b/web/Dockerfile new file mode 100644 index 0000000..25c9320 --- /dev/null +++ b/web/Dockerfile @@ -0,0 +1,26 @@ +# Use the official Node.js image as the build image +FROM node:latest + +# Set the build directory +WORKDIR /build + +# Copy the package.json and yarn.lock files to the container +COPY package*.json yarn.lock ./ + +# Install dependencies +RUN yarn install + +# Copy the entire project to the container +COPY . . + +# Expose port 4444 for serving the project +EXPOSE 4444 + +# Copy the certificate hash before running +VOLUME /cert + +# Make a symlink to the certificate fingerprint +RUN ln -s /cert/localhost.hex fingerprint.hex + +# Copy the certificate fingerprint and start the web server +CMD yarn parcel serve --https --cert /cert/localhost.crt --key /cert/localhost.key --port 4444 \ No newline at end of file diff --git a/web/package.json b/web/package.json index 0fe1770..72148b6 100644 --- a/web/package.json +++ b/web/package.json @@ -1,7 +1,8 @@ { + "license": "Apache-2.0", "source": "src/index.html", "scripts": { - "serve": "parcel serve --https --cert ../cert/localhost.crt --key ../cert/localhost.key --host localhost --port 4444 --open", + "serve": "parcel serve --https --cert ../cert/localhost.crt --key ../cert/localhost.key --port 4444 --open", "build": "parcel build", "check": "tsc --noEmit" }, @@ -16,4 +17,4 @@ "dependencies": { "mp4box": "^0.5.2" } -} +} \ No newline at end of file diff --git a/web/src/index.ts b/web/src/index.ts index 5882480..2f83061 100644 --- a/web/src/index.ts +++ b/web/src/index.ts @@ -12,7 +12,7 @@ for (let c = 0; c < fingerprintHex.length-1; c += 2) { const params = new URLSearchParams(window.location.search) -const url = params.get("url") || "https://127.0.0.1:4443/watch" +const url = params.get("url") || "https://localhost:4443/watch" const canvas = document.querySelector("canvas#video")! const transport = new Transport({ @@ -30,7 +30,7 @@ const player = new Player({ const play = document.querySelector("#screen #play")! -let playFunc = (e: Event) => { +const playFunc = (e: Event) => { player.play({}) e.preventDefault() @@ -38,4 +38,4 @@ let playFunc = (e: Event) => { play.style.display = "none" } -play.addEventListener('click', playFunc) \ No newline at end of file +play.addEventListener('click', playFunc) diff --git a/web/src/player/audio.ts b/web/src/player/audio.ts new file mode 100644 index 0000000..361036d --- /dev/null +++ b/web/src/player/audio.ts @@ -0,0 +1,75 @@ +import * as Message from "./message"; +import { Ring } from "./ring" + +export default class Audio { + ring: Ring; + queue: Array; + + sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0, in microseconds + last?: number; // the timestamp of the last rendered frame, in microseconds + + constructor(config: Message.AudioConfig) { + this.ring = new Ring(config.ring); + this.queue = []; + } + + push(frame: AudioData) { + if (!this.sync) { + // Save the frame as the sync point + // TODO sync with video + this.sync = 1000 * performance.now() - frame.timestamp + } + + // Drop any old frames + if (this.last && frame.timestamp <= this.last) { + frame.close() + return + } + + // Insert the frame into the queue sorted by timestamp. + if (this.queue.length > 0 && this.queue[this.queue.length-1].timestamp <= frame.timestamp) { + // Fast path because we normally append to the end. + this.queue.push(frame) + } else { + // Do a full binary search + let low = 0 + let high = this.queue.length; + + while (low < high) { + const mid = (low + high) >>> 1; + if (this.queue[mid].timestamp < frame.timestamp) low = mid + 1; + else high = mid; + } + + this.queue.splice(low, 0, frame) + } + } + + + draw() { + // Convert to microseconds + const now = 1000 * performance.now(); + + // Determine the target timestamp. + const target = now - this.sync! + + // Check if we should skip some frames + while (this.queue.length) { + const next = this.queue[0] + + if (next.timestamp > target) { + const ok = this.ring.write(next) + if (!ok) { + console.warn("ring buffer is full") + // No more space in the ring + break + } + } else { + console.warn("dropping audio") + } + + next.close() + this.queue.shift() + } + } +} \ No newline at end of file diff --git a/web/src/player/video.ts b/web/src/player/video.ts new file mode 100644 index 0000000..d112150 --- /dev/null +++ b/web/src/player/video.ts @@ -0,0 +1,91 @@ +import * as Message from "./message"; + +export default class Video { + canvas: OffscreenCanvas; + queue: Array; + + render: number; // non-zero if requestAnimationFrame has been called + sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0, in microseconds + last?: number; // the timestamp of the last rendered frame, in microseconds + + constructor(config: Message.VideoConfig) { + this.canvas = config.canvas; + this.queue = []; + + this.render = 0; + } + + push(frame: VideoFrame) { + if (!this.sync) { + // Save the frame as the sync point + this.sync = 1000 * performance.now() - frame.timestamp + } + + // Drop any old frames + if (this.last && frame.timestamp <= this.last) { + frame.close() + return + } + + // Insert the frame into the queue sorted by timestamp. + if (this.queue.length > 0 && this.queue[this.queue.length-1].timestamp <= frame.timestamp) { + // Fast path because we normally append to the end. + this.queue.push(frame) + } else { + // Do a full binary search + let low = 0 + let high = this.queue.length; + + while (low < high) { + const mid = (low + high) >>> 1; + if (this.queue[mid].timestamp < frame.timestamp) low = mid + 1; + else high = mid; + } + + this.queue.splice(low, 0, frame) + } + + // Queue up to render the next frame. + if (!this.render) { + this.render = self.requestAnimationFrame(this.draw.bind(this)) + } + } + + draw(now: DOMHighResTimeStamp) { + // Convert to microseconds + now *= 1000; + + // Determine the target timestamp. + const target = now - this.sync! + + let frame = this.queue[0]; + if (frame.timestamp >= target) { + // nothing to render yet, wait for the next animation frame + this.render = self.requestAnimationFrame(this.draw.bind(this)) + return + } + + this.queue.shift(); + + // Check if we should skip some frames + while (this.queue.length) { + const next = this.queue[0] + if (next.timestamp > target) break + + frame.close() + frame = this.queue.shift()!; + } + + const ctx = this.canvas.getContext("2d"); + ctx!.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height) // TODO aspect ratio + + this.last = frame.timestamp; + frame.close() + + if (this.queue.length) { + this.render = self.requestAnimationFrame(this.draw.bind(this)) + } else { + this.render = 0 + } + } +} \ No newline at end of file diff --git a/web/src/transport/index.ts b/web/src/transport/index.ts index 72eb4fd..0532bc3 100644 --- a/web/src/transport/index.ts +++ b/web/src/transport/index.ts @@ -56,7 +56,7 @@ export default class Transport { const q = await this.quic const streams = q.incomingUnidirectionalStreams.getReader() - while (true) { + for (;;) { const result = await streams.read() if (result.done) break @@ -66,7 +66,7 @@ export default class Transport { } async handleStream(stream: ReadableStream) { - let r = new Stream.Reader(stream) + const r = new Stream.Reader(stream) while (!await r.done()) { const size = await r.uint32(); From 16abb2d6dcbfdf75702a30fa70ae28ae8e6e31b4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 18 May 2023 12:05:38 -0700 Subject: [PATCH 17/23] Fix docker-compose so it uses the right cert hex. --- cert/Dockerfile | 12 +++++++----- cert/generate | 2 +- media/Dockerfile | 15 ++++++++++----- server/src/main.rs | 2 +- web/src/index.ts | 4 ++-- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/cert/Dockerfile b/cert/Dockerfile index 84cb44c..df5be10 100644 --- a/cert/Dockerfile +++ b/cert/Dockerfile @@ -1,13 +1,12 @@ # Use ubuntu because it's ez FROM ubuntu:latest +WORKDIR /build + # Use openssl and golang to generate certificates RUN apt-get update && \ - apt-get install -y ca-certificates openssl golang + apt-get install -y ca-certificates openssl golang xxd -# Save the certificates to a volume -VOLUME /cert -WORKDIR /cert # Download the go modules COPY go.mod go.sum ./ @@ -16,5 +15,8 @@ RUN go mod download # Copy over the remaining files. COPY . . +# Save the certificates to a volume +VOLUME /cert + # TODO support an output directory -CMD ./generate \ No newline at end of file +CMD ./generate && cp localhost.* /cert \ No newline at end of file diff --git a/cert/generate b/cert/generate index c86b309..63bb335 100755 --- a/cert/generate +++ b/cert/generate @@ -17,4 +17,4 @@ go run filippo.io/mkcert -ecdsa -install go run filippo.io/mkcert -ecdsa -days 10 -cert-file "$CRT" -key-file "$KEY" localhost 127.0.0.1 ::1 # Compute the sha256 fingerprint of the certificate for WebTransport -openssl x509 -in "$CRT" -outform der | openssl dgst -sha256 > localhost.hex \ No newline at end of file +openssl x509 -in "$CRT" -outform der | openssl dgst -sha256 -binary | xxd -p -c 256 > localhost.hex \ No newline at end of file diff --git a/media/Dockerfile b/media/Dockerfile index f8f36e4..372db90 100644 --- a/media/Dockerfile +++ b/media/Dockerfile @@ -1,6 +1,9 @@ # Create a build image FROM ubuntu:latest +# Create the working directory. +WORKDIR /build + # Install necessary packages RUN apt-get update && \ apt-get install -y \ @@ -8,13 +11,15 @@ RUN apt-get update && \ wget \ ffmpeg -# Create a media volume -VOLUME /media -WORKDIR /media - # Download a file from the internet, in this case my boy big buck bunny RUN wget http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4 -O source.mp4 # Copy an run a script to create a fragmented mp4 (more overhead, easier to split) COPY fragment . -CMD ./fragment + +# Create a media volume +VOLUME /media + +# Fragment the media +# TODO support an output directory +CMD ./fragment && cp fragmented.mp4 /media \ No newline at end of file diff --git a/server/src/main.rs b/server/src/main.rs index 56441d7..373474c 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -6,7 +6,7 @@ use clap::Parser; #[derive(Parser)] struct Cli { /// Listen on this address - #[arg(short, long, default_value = "0.0.0.0:4443")] + #[arg(short, long, default_value = "[::]:4443")] addr: String, /// Use the certificate file at this path diff --git a/web/src/index.ts b/web/src/index.ts index 2f83061..5c46360 100644 --- a/web/src/index.ts +++ b/web/src/index.ts @@ -6,8 +6,8 @@ import fingerprintHex from 'bundle-text:../fingerprint.hex'; // Convert the hex to binary. let fingerprint = []; -for (let c = 0; c < fingerprintHex.length-1; c += 2) { - fingerprint.push(parseInt(fingerprintHex.substring(c, c+2), 16)); +for (let c = 0; c < fingerprintHex.length - 1; c += 2) { + fingerprint.push(parseInt(fingerprintHex.substring(c, c + 2), 16)); } const params = new URLSearchParams(window.location.search) From a9fd9186d2923e8afb79deca883613d07398196f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 18 May 2023 12:09:31 -0700 Subject: [PATCH 18/23] Add the fingerprint symlink back. --- web/.gitignore | 3 +-- web/fingerprint.hex | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) create mode 120000 web/fingerprint.hex diff --git a/web/.gitignore b/web/.gitignore index 10ec168..2610592 100644 --- a/web/.gitignore +++ b/web/.gitignore @@ -1,4 +1,3 @@ node_modules .parcel-cache -dist -fingerprint.hex +dist \ No newline at end of file diff --git a/web/fingerprint.hex b/web/fingerprint.hex new file mode 120000 index 0000000..387f554 --- /dev/null +++ b/web/fingerprint.hex @@ -0,0 +1 @@ +../cert/localhost.hex \ No newline at end of file From 3f6ea4238077c675278bdf86352a07e9f2624ecc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 22 May 2023 13:30:46 -0700 Subject: [PATCH 19/23] Fixed audio. --- web/src/index.html | 7 +- web/src/index.ts | 2 +- web/src/player/audio.ts | 68 ++++++----- web/src/player/index.ts | 43 ++++--- web/src/player/message.ts | 13 +- web/src/player/renderer.ts | 129 ++------------------ web/src/player/ring.ts | 241 +++++++++++++++++++------------------ web/src/player/video.ts | 47 +++++--- web/src/player/worker.ts | 3 + web/src/player/worklet.ts | 16 ++- 10 files changed, 239 insertions(+), 330 deletions(-) diff --git a/web/src/index.html b/web/src/index.html index aeda194..74179a8 100644 --- a/web/src/index.html +++ b/web/src/index.html @@ -2,7 +2,7 @@ - + WARP @@ -11,7 +11,7 @@
-
click for audio
+
click to play
@@ -31,4 +31,5 @@ - + + \ No newline at end of file diff --git a/web/src/index.ts b/web/src/index.ts index 5c46360..053f819 100644 --- a/web/src/index.ts +++ b/web/src/index.ts @@ -31,7 +31,7 @@ const player = new Player({ const play = document.querySelector("#screen #play")! const playFunc = (e: Event) => { - player.play({}) + player.play() e.preventDefault() play.removeEventListener('click', playFunc) diff --git a/web/src/player/audio.ts b/web/src/player/audio.ts index 361036d..cbfcd3e 100644 --- a/web/src/player/audio.ts +++ b/web/src/player/audio.ts @@ -2,24 +2,17 @@ import * as Message from "./message"; import { Ring } from "./ring" export default class Audio { - ring: Ring; + ring?: Ring; queue: Array; - sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0, in microseconds + render?: number; // non-zero if requestAnimationFrame has been called last?: number; // the timestamp of the last rendered frame, in microseconds - constructor(config: Message.AudioConfig) { - this.ring = new Ring(config.ring); - this.queue = []; + constructor(config: Message.Config) { + this.queue = [] } push(frame: AudioData) { - if (!this.sync) { - // Save the frame as the sync point - // TODO sync with video - this.sync = 1000 * performance.now() - frame.timestamp - } - // Drop any old frames if (this.last && frame.timestamp <= this.last) { frame.close() @@ -27,7 +20,7 @@ export default class Audio { } // Insert the frame into the queue sorted by timestamp. - if (this.queue.length > 0 && this.queue[this.queue.length-1].timestamp <= frame.timestamp) { + if (this.queue.length > 0 && this.queue[this.queue.length - 1].timestamp <= frame.timestamp) { // Fast path because we normally append to the end. this.queue.push(frame) } else { @@ -43,33 +36,44 @@ export default class Audio { this.queue.splice(low, 0, frame) } + + this.emit() } + emit() { + const ring = this.ring + if (!ring) { + return + } - draw() { - // Convert to microseconds - const now = 1000 * performance.now(); - - // Determine the target timestamp. - const target = now - this.sync! - - // Check if we should skip some frames while (this.queue.length) { - const next = this.queue[0] - - if (next.timestamp > target) { - const ok = this.ring.write(next) - if (!ok) { - console.warn("ring buffer is full") - // No more space in the ring - break - } - } else { - console.warn("dropping audio") + let frame = this.queue[0]; + if (ring.size() + frame.numberOfFrames > ring.capacity) { + // Buffer is full + break } - next.close() + const size = ring.write(frame) + if (size < frame.numberOfFrames) { + throw new Error("audio buffer is full") + } + + this.last = frame.timestamp + + frame.close() this.queue.shift() } } + + play(play: Message.Play) { + this.ring = new Ring(play.buffer) + + if (!this.render) { + const sampleRate = 44100 // TODO dynamic + + // Refresh every half buffer + const refresh = play.buffer.capacity / sampleRate * 1000 / 2 + this.render = setInterval(this.emit.bind(this), refresh) + } + } } \ No newline at end of file diff --git a/web/src/player/index.ts b/web/src/player/index.ts index 1ef73b2..a539167 100644 --- a/web/src/player/index.ts +++ b/web/src/player/index.ts @@ -19,26 +19,16 @@ export default class Player { this.transport = config.transport this.transport.callback = this; - const video = { - canvas: config.canvas, - }; - - // Assume 44.1kHz and two audio channels - const audio = { - sampleRate: 44100, - ring: new Ring.Buffer(2, 4410), // 100ms at 44.1khz - } - this.context = new AudioContext({ latencyHint: "interactive", - sampleRate: audio.sampleRate, + sampleRate: 44100, }) - this.worker = this.setupWorker({ audio, video }) - this.worklet = this.setupWorklet(audio) + this.worker = this.setupWorker(config) + this.worklet = this.setupWorklet(config) } - private setupWorker(config: Message.Config): Worker { + private setupWorker(config: Config): Worker { const url = new URL('worker.ts', import.meta.url) const worker = new Worker(url, { @@ -46,12 +36,16 @@ export default class Player { name: "media", }) - worker.postMessage({ config }, [ config.video.canvas ]) + const msg = { + canvas: config.canvas, + } + + worker.postMessage({ config: msg }, [msg.canvas]) return worker } - private async setupWorklet(config: Message.AudioConfig): Promise { + private async setupWorklet(config: Config): Promise { // Load the worklet source code. const url = new URL('worklet.ts', import.meta.url) await this.context.audioWorklet.addModule(url) @@ -65,8 +59,6 @@ export default class Player { console.error("Audio worklet error:", e) }; - worklet.port.postMessage({ config }) - // Connect the worklet to the volume node and then to the speakers worklet.connect(volume) volume.connect(this.context.destination) @@ -75,15 +67,22 @@ export default class Player { } onInit(init: Message.Init) { - this.worker.postMessage({ init }, [ init.buffer.buffer, init.reader ]) + this.worker.postMessage({ init }, [init.buffer.buffer, init.reader]) } onSegment(segment: Message.Segment) { - this.worker.postMessage({ segment }, [ segment.buffer.buffer, segment.reader ]) + this.worker.postMessage({ segment }, [segment.buffer.buffer, segment.reader]) } - play(play: Message.Play) { + async play() { this.context.resume() - //this.worker.postMessage({ play }) + + const play = { + buffer: new Ring.Buffer(2, 44100 / 10), // 100ms of audio + } + + const worklet = await this.worklet; + worklet.port.postMessage({ play }) + this.worker.postMessage({ play }) } } \ No newline at end of file diff --git a/web/src/player/message.ts b/web/src/player/message.ts index dcad2e0..0f4b00b 100644 --- a/web/src/player/message.ts +++ b/web/src/player/message.ts @@ -1,20 +1,10 @@ import * as Ring from "./ring" export interface Config { - audio: AudioConfig; - video: VideoConfig; -} - -export interface VideoConfig { + // video stuff canvas: OffscreenCanvas; } -export interface AudioConfig { - // audio stuff - sampleRate: number; - ring: Ring.Buffer; -} - export interface Init { buffer: Uint8Array; // unread buffered data reader: ReadableStream; // unread unbuffered data @@ -27,4 +17,5 @@ export interface Segment { export interface Play { timestamp?: number; + buffer: Ring.Buffer; } \ No newline at end of file diff --git a/web/src/player/renderer.ts b/web/src/player/renderer.ts index d2d6c86..3da2e85 100644 --- a/web/src/player/renderer.ts +++ b/web/src/player/renderer.ts @@ -1,136 +1,29 @@ import * as Message from "./message"; -import { Ring } from "./ring" +import Audio from "./audio" +import Video from "./video" export default class Renderer { - audioRing: Ring; - audioQueue: Array; - - videoCanvas: OffscreenCanvas; - videoQueue: Array; - - render: number; // non-zero if requestAnimationFrame has been called - sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0, in microseconds - last?: number; // the timestamp of the last rendered frame, in microseconds + audio: Audio; + video: Video; constructor(config: Message.Config) { - this.audioRing = new Ring(config.audio.ring); - this.audioQueue = []; - - this.videoCanvas = config.video.canvas; - this.videoQueue = []; - - this.render = 0; + this.audio = new Audio(config); + this.video = new Video(config); } push(frame: AudioData | VideoFrame) { - if (!this.sync) { - // Save the frame as the sync point - this.sync = 1000 * performance.now() - frame.timestamp - } - - // Drop any old frames - if (this.last && frame.timestamp <= this.last) { - frame.close() - return - } - - let queue if (isAudioData(frame)) { - queue = this.audioQueue; + this.audio.push(frame); } else if (isVideoFrame(frame)) { - queue = this.videoQueue; + this.video.push(frame); } else { throw new Error("unknown frame type") } - - // Insert the frame into the queue sorted by timestamp. - if (queue.length > 0 && queue[queue.length-1].timestamp <= frame.timestamp) { - // Fast path because we normally append to the end. - queue.push(frame as any) - } else { - // Do a full binary search - let low = 0 - let high = queue.length; - - while (low < high) { - var mid = (low + high) >>> 1; - if (queue[mid].timestamp < frame.timestamp) low = mid + 1; - else high = mid; - } - - queue.splice(low, 0, frame as any) - } - - // Queue up to render the next frame. - if (!this.render) { - this.render = self.requestAnimationFrame(this.draw.bind(this)) - } } - draw(now: DOMHighResTimeStamp) { - // Convert to microseconds - now *= 1000; - - // Determine the target timestamp. - const target = now - this.sync! - - this.drawAudio(now, target) - this.drawVideo(now, target) - - if (this.audioQueue.length || this.videoQueue.length) { - this.render = self.requestAnimationFrame(this.draw.bind(this)) - } else { - this.render = 0 - } - } - - drawAudio(now: DOMHighResTimeStamp, target: DOMHighResTimeStamp) { - // Check if we should skip some frames - while (this.audioQueue.length) { - const next = this.audioQueue[0] - - if (next.timestamp > target) { - let ok = this.audioRing.write(next) - if (!ok) { - console.warn("ring buffer is full") - // No more space in the ring - break - } - } else { - console.warn("dropping audio") - } - - next.close() - this.audioQueue.shift() - } - } - - drawVideo(now: DOMHighResTimeStamp, target: DOMHighResTimeStamp) { - if (!this.videoQueue.length) return; - - let frame = this.videoQueue[0]; - if (frame.timestamp >= target) { - // nothing to render yet, wait for the next animation frame - this.render = self.requestAnimationFrame(this.draw.bind(this)) - return - } - - this.videoQueue.shift(); - - // Check if we should skip some frames - while (this.videoQueue.length) { - const next = this.videoQueue[0] - if (next.timestamp > target) break - - frame.close() - frame = this.videoQueue.shift()!; - } - - const ctx = this.videoCanvas.getContext("2d"); - ctx!.drawImage(frame, 0, 0, this.videoCanvas.width, this.videoCanvas.height) // TODO aspect ratio - - this.last = frame.timestamp; - frame.close() + play(play: Message.Play) { + this.audio.play(play); + this.video.play(play); } } diff --git a/web/src/player/ring.ts b/web/src/player/ring.ts index ffc1c6b..ccf6f2a 100644 --- a/web/src/player/ring.ts +++ b/web/src/player/ring.ts @@ -1,123 +1,9 @@ // Ring buffer with audio samples. enum STATE { - READ_INDEX = 0, // Index of the current read position (mod capacity) - WRITE_INDEX, // Index of the current write position (mod capacity) - LENGTH // Clever way of saving the total number of enums values. -} - -export class Ring { - state: Int32Array; - channels: Float32Array[]; - capacity: number; - - constructor(buf: Buffer) { - this.state = new Int32Array(buf.state) - - this.channels = [] - for (let channel of buf.channels) { - this.channels.push(new Float32Array(channel)) - } - - this.capacity = buf.capacity - } - - // Add the samples for single audio frame - write(frame: AudioData): boolean { - let count = frame.numberOfFrames; - - let readIndex = Atomics.load(this.state, STATE.READ_INDEX) - let writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX) - let writeIndexNew = writeIndex + count; - - // There's not enough space in the ring buffer - if (writeIndexNew - readIndex > this.capacity) { - return false - } - - let startIndex = writeIndex % this.capacity; - let endIndex = writeIndexNew % this.capacity; - - // Loop over each channel - for (let i = 0; i < this.channels.length; i += 1) { - const channel = this.channels[i] - - if (startIndex < endIndex) { - // One continuous range to copy. - const full = channel.subarray(startIndex, endIndex) - - frame.copyTo(full, { - planeIndex: i, - frameCount: count, - }) - } else { - const first = channel.subarray(startIndex) - const second = channel.subarray(0, endIndex) - - frame.copyTo(first, { - planeIndex: i, - frameCount: first.length, - }) - - frame.copyTo(second, { - planeIndex: i, - frameOffset: first.length, - frameCount: second.length, - }) - } - } - - Atomics.store(this.state, STATE.WRITE_INDEX, writeIndexNew) - - return true - } - - read(dst: Float32Array[]) { - let readIndex = Atomics.load(this.state, STATE.READ_INDEX) - let writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX) - if (readIndex >= writeIndex) { - // nothing to read - return - } - - let readIndexNew = readIndex + dst[0].length - if (readIndexNew > writeIndex) { - // Partial read - readIndexNew = writeIndex - } - - let startIndex = readIndex % this.capacity; - let endIndex = readIndexNew % this.capacity; - - // Loop over each channel - for (let i = 0; i < dst.length; i += 1) { - if (i >= this.channels.length) { - // ignore excess channels - } - - const input = this.channels[i] - const output = dst[i] - - if (startIndex < endIndex) { - const full = input.subarray(startIndex, endIndex) - output.set(full) - } else { - const first = input.subarray(startIndex) - const second = input.subarray(0, endIndex) - - output.set(first) - output.set(second, first.length) - } - } - - Atomics.store(this.state, STATE.READ_INDEX, readIndexNew) - } - - // TODO not thread safe - clear() { - const writeIndex = Atomics.load(this.state, STATE.WRITE_INDEX) - Atomics.store(this.state, STATE.READ_INDEX, writeIndex) - } + READ_POS = 0, // The current read position + WRITE_POS, // The current write position + LENGTH // Clever way of saving the total number of enums values. } // No prototype to make this easier to send via postMessage @@ -140,4 +26,125 @@ export class Buffer { this.capacity = capacity } +} + +export class Ring { + state: Int32Array; + channels: Float32Array[]; + capacity: number; + + constructor(buffer: Buffer) { + this.state = new Int32Array(buffer.state) + + this.channels = [] + for (let channel of buffer.channels) { + this.channels.push(new Float32Array(channel)) + } + + this.capacity = buffer.capacity + } + + // Write samples for single audio frame, returning the total number written. + write(frame: AudioData): number { + let readPos = Atomics.load(this.state, STATE.READ_POS) + let writePos = Atomics.load(this.state, STATE.WRITE_POS) + + const startPos = writePos + let endPos = writePos + frame.numberOfFrames; + + if (endPos > readPos + this.capacity) { + endPos = readPos + this.capacity + if (endPos <= startPos) { + // No space to write + return 0 + } + } + + let startIndex = startPos % this.capacity; + let endIndex = endPos % this.capacity; + + // Loop over each channel + for (let i = 0; i < this.channels.length; i += 1) { + const channel = this.channels[i] + + if (startIndex < endIndex) { + // One continuous range to copy. + const full = channel.subarray(startIndex, endIndex) + + frame.copyTo(full, { + planeIndex: i, + frameCount: endIndex - startIndex, + }) + } else { + const first = channel.subarray(startIndex) + const second = channel.subarray(0, endIndex) + + frame.copyTo(first, { + planeIndex: i, + frameCount: first.length, + }) + + frame.copyTo(second, { + planeIndex: i, + frameOffset: first.length, + frameCount: second.length, + }) + } + } + + Atomics.store(this.state, STATE.WRITE_POS, endPos) + + return endPos - startPos + } + + read(dst: Float32Array[]): number { + let readPos = Atomics.load(this.state, STATE.READ_POS) + let writePos = Atomics.load(this.state, STATE.WRITE_POS) + + let startPos = readPos; + let endPos = startPos + dst[0].length; + + if (endPos > writePos) { + endPos = writePos + if (endPos <= startPos) { + // Nothing to read + return 0 + } + } + + let startIndex = startPos % this.capacity; + let endIndex = endPos % this.capacity; + + // Loop over each channel + for (let i = 0; i < dst.length; i += 1) { + if (i >= this.channels.length) { + // ignore excess channels + } + + const input = this.channels[i] + const output = dst[i] + + if (startIndex < endIndex) { + const full = input.subarray(startIndex, endIndex) + output.set(full) + } else { + const first = input.subarray(startIndex) + const second = input.subarray(0, endIndex) + + output.set(first) + output.set(second, first.length) + } + } + + Atomics.store(this.state, STATE.READ_POS, endPos) + + return endPos - startPos + } + + size() { + let readPos = Atomics.load(this.state, STATE.READ_POS) + let writePos = Atomics.load(this.state, STATE.WRITE_POS) + + return writePos - readPos + } } \ No newline at end of file diff --git a/web/src/player/video.ts b/web/src/player/video.ts index d112150..f725122 100644 --- a/web/src/player/video.ts +++ b/web/src/player/video.ts @@ -5,10 +5,10 @@ export default class Video { queue: Array; render: number; // non-zero if requestAnimationFrame has been called - sync?: DOMHighResTimeStamp; // the wall clock value for timestamp 0, in microseconds + sync?: number; // the wall clock value for timestamp 0, in microseconds last?: number; // the timestamp of the last rendered frame, in microseconds - constructor(config: Message.VideoConfig) { + constructor(config: Message.Config) { this.canvas = config.canvas; this.queue = []; @@ -16,11 +16,6 @@ export default class Video { } push(frame: VideoFrame) { - if (!this.sync) { - // Save the frame as the sync point - this.sync = 1000 * performance.now() - frame.timestamp - } - // Drop any old frames if (this.last && frame.timestamp <= this.last) { frame.close() @@ -28,7 +23,7 @@ export default class Video { } // Insert the frame into the queue sorted by timestamp. - if (this.queue.length > 0 && this.queue[this.queue.length-1].timestamp <= frame.timestamp) { + if (this.queue.length > 0 && this.queue[this.queue.length - 1].timestamp <= frame.timestamp) { // Fast path because we normally append to the end. this.queue.push(frame) } else { @@ -44,24 +39,35 @@ export default class Video { this.queue.splice(low, 0, frame) } - - // Queue up to render the next frame. - if (!this.render) { - this.render = self.requestAnimationFrame(this.draw.bind(this)) - } } - draw(now: DOMHighResTimeStamp) { + draw(now: number) { + // Draw and then queue up the next draw call. + this.drawOnce(now); + + // Queue up the new draw frame. + this.render = self.requestAnimationFrame(this.draw.bind(this)) + } + + drawOnce(now: number) { // Convert to microseconds now *= 1000; - // Determine the target timestamp. - const target = now - this.sync! + if (!this.queue.length) { + return + } let frame = this.queue[0]; + + if (!this.sync) { + this.sync = now - frame.timestamp; + } + + // Determine the target timestamp. + const target = now - this.sync + if (frame.timestamp >= target) { // nothing to render yet, wait for the next animation frame - this.render = self.requestAnimationFrame(this.draw.bind(this)) return } @@ -81,11 +87,12 @@ export default class Video { this.last = frame.timestamp; frame.close() + } - if (this.queue.length) { + play(play: Message.Play) { + // Queue up to render the next frame. + if (!this.render) { this.render = self.requestAnimationFrame(this.draw.bind(this)) - } else { - this.render = 0 } } } \ No newline at end of file diff --git a/web/src/player/worker.ts b/web/src/player/worker.ts index 4597c29..5a563f5 100644 --- a/web/src/player/worker.ts +++ b/web/src/player/worker.ts @@ -17,6 +17,9 @@ self.addEventListener('message', async (e: MessageEvent) => { } else if (e.data.segment) { const segment = e.data.segment as Message.Segment await decoder.receiveSegment(segment) + } else if (e.data.play) { + const play = e.data.play as Message.Play + await renderer.play(play) } }) diff --git a/web/src/player/worklet.ts b/web/src/player/worklet.ts index 4946bd8..fe3216e 100644 --- a/web/src/player/worklet.ts +++ b/web/src/player/worklet.ts @@ -19,19 +19,19 @@ class Renderer extends AudioWorkletProcessor { } onMessage(e: MessageEvent) { - if (e.data.config) { - this.onConfig(e.data.config) + if (e.data.play) { + this.onPlay(e.data.play) } } - onConfig(config: Message.AudioConfig) { - this.ring = new Ring(config.ring) + onPlay(play: Message.Play) { + this.ring = new Ring(play.buffer) } // Inputs and outputs in groups of 128 samples. process(inputs: Float32Array[][], outputs: Float32Array[][], parameters: Record): boolean { if (!this.ring) { - // Not initialized yet + // Paused return true } @@ -40,7 +40,11 @@ class Renderer extends AudioWorkletProcessor { } const output = outputs[0] - this.ring.read(output) + + const size = this.ring.read(output) + if (size < output.length) { + // TODO trigger rebuffering event + } return true; } From 4132d8db4dacac58c01ff2cac48ddd08d20ec5e1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 22 May 2023 13:49:02 -0700 Subject: [PATCH 20/23] Fix audio crashing after some time. --- web/src/player/ring.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/web/src/player/ring.ts b/web/src/player/ring.ts index ccf6f2a..4b1b7ae 100644 --- a/web/src/player/ring.ts +++ b/web/src/player/ring.ts @@ -60,6 +60,12 @@ export class Ring { } } + // capacity = 1024 + // read = 2048 + // write = 3072 + // startIndex = 0 + // readIndex = 0 + let startIndex = startPos % this.capacity; let endIndex = endPos % this.capacity; @@ -84,11 +90,13 @@ export class Ring { frameCount: first.length, }) - frame.copyTo(second, { - planeIndex: i, - frameOffset: first.length, - frameCount: second.length, - }) + if (second.length > 0) { + frame.copyTo(second, { + planeIndex: i, + frameOffset: first.length, + frameCount: second.length, + }) + } } } @@ -142,6 +150,7 @@ export class Ring { } size() { + // TODO is this thread safe? let readPos = Atomics.load(this.state, STATE.READ_POS) let writePos = Atomics.load(this.state, STATE.WRITE_POS) From 58a1aa85acc69e8feaea84f94925f9c3554a435f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 22 May 2023 14:09:11 -0700 Subject: [PATCH 21/23] Fix audio crashing after some time. --- web/src/player/ring.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/web/src/player/ring.ts b/web/src/player/ring.ts index 4b1b7ae..2483692 100644 --- a/web/src/player/ring.ts +++ b/web/src/player/ring.ts @@ -60,12 +60,6 @@ export class Ring { } } - // capacity = 1024 - // read = 2048 - // write = 3072 - // startIndex = 0 - // readIndex = 0 - let startIndex = startPos % this.capacity; let endIndex = endPos % this.capacity; @@ -90,7 +84,9 @@ export class Ring { frameCount: first.length, }) - if (second.length > 0) { + // We need this conditional when startIndex == 0 and endIndex == 0 + // When capacity=4410 and frameCount=1024, this was happening 52s into the audio. + if (second.length) { frame.copyTo(second, { planeIndex: i, frameOffset: first.length, From e15812ebedea4b329e7208465d193dff2a6097e3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 22 May 2023 15:22:52 -0700 Subject: [PATCH 22/23] Fix some crashes with the server. --- server/src/session/mod.rs | 2 +- server/src/transport/server.rs | 60 +++++++++++++++++++++------------- 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/server/src/session/mod.rs b/server/src/session/mod.rs index 0b4af65..e5c7cb0 100644 --- a/server/src/session/mod.rs +++ b/server/src/session/mod.rs @@ -71,7 +71,7 @@ impl transport::App for Session { self.streams.poll(conn); // Fetch the next media fragment, possibly queuing up stream data. - self.poll_source(conn, session).expect("poll_source"); + self.poll_source(conn, session)?; Ok(()) } diff --git a/server/src/transport/server.rs b/server/src/transport/server.rs index d0ab1f7..19031af 100644 --- a/server/src/transport/server.rs +++ b/server/src/transport/server.rs @@ -151,7 +151,6 @@ impl Server { // Check if it's an existing connection. if let Some(conn) = self.conns.get_mut(&hdr.dcid) { - // initial or handshake traffic. conn.quiche.recv(src, info)?; if conn.session.is_none() && conn.quiche.is_established() { @@ -162,7 +161,6 @@ impl Server { continue; } else if let Some(conn) = self.conns.get_mut(&conn_id) { - // 1-RTT traffic. conn.quiche.recv(src, info)?; // TODO is this needed here? @@ -176,7 +174,8 @@ impl Server { } if hdr.ty != quiche::Type::Initial { - anyhow::bail!("unknown connection ID"); + log::warn!("unknown connection ID"); + continue; } let mut dst = [0; MAX_DATAGRAM_SIZE]; @@ -222,11 +221,13 @@ impl Server { // The token was not valid, meaning the retry failed, so // drop the packet. if odcid.is_none() { - anyhow::bail!("invalid token"); + log::warn!("invalid token"); + continue; } if scid.len() != hdr.dcid.len() { - anyhow::bail!("invalid connection ID"); + log::warn!("invalid connection ID"); + continue; } // Reuse the source connection ID we sent in the Retry packet, @@ -234,6 +235,8 @@ impl Server { let conn_id = hdr.dcid.clone(); let local_addr = self.socket.local_addr().unwrap(); + log::debug!("new connection: dcid={:?} scid={:?}", hdr.dcid, scid); + let mut conn = quiche::accept(&conn_id, odcid.as_ref(), local_addr, from, &mut self.quic)?; @@ -246,13 +249,16 @@ impl Server { app: T::default(), }; - self.conns - .insert(user.quiche.source_id().into_owned(), user); + self.conns.insert(conn_id, user); } } pub fn app(&mut self) -> anyhow::Result<()> { for conn in self.conns.values_mut() { + if conn.quiche.is_closed() { + continue; + } + if let Some(session) = &mut conn.session { if let Err(e) = conn.app.poll(&mut conn.quiche, session) { log::debug!("app error: {:?}", e); @@ -271,23 +277,12 @@ impl Server { // them on the UDP socket, until quiche reports that there are no more // packets to be sent. pub fn send(&mut self) -> anyhow::Result<()> { - let mut pkt = [0; MAX_DATAGRAM_SIZE]; - for conn in self.conns.values_mut() { - loop { - let (size, info) = match conn.quiche.send(&mut pkt) { - Ok(v) => v, - Err(quiche::Error::Done) => return Ok(()), - Err(e) => return Err(e.into()), - }; + let conn = &mut conn.quiche; - let pkt = &pkt[..size]; - - match self.socket.send_to(pkt, info.to) { - Err(err) if err.kind() == io::ErrorKind::WouldBlock => break, - Err(err) => return Err(err.into()), - Ok(_) => (), - } + if let Err(e) = send_conn(&self.socket, conn) { + log::error!("{} send failed: {:?}", conn.trace_id(), e); + conn.close(false, 0x1, b"fail").ok(); } } @@ -300,6 +295,27 @@ impl Server { } } +// Send any pending packets for the connection over the socket. +fn send_conn(socket: &mio::net::UdpSocket, conn: &mut quiche::Connection) -> anyhow::Result<()> { + let mut pkt = [0; MAX_DATAGRAM_SIZE]; + + loop { + let (size, info) = match conn.send(&mut pkt) { + Ok(v) => v, + Err(quiche::Error::Done) => return Ok(()), + Err(e) => return Err(e.into()), + }; + + let pkt = &pkt[..size]; + + match socket.send_to(pkt, info.to) { + Err(e) if e.kind() == io::ErrorKind::WouldBlock => return Ok(()), + Err(e) => return Err(e.into()), + Ok(_) => (), + } + } +} + /// Generate a stateless retry token. /// /// The token includes the static string `"quiche"` followed by the IP address From 1137f3024cd3956bb00f9c4884dc72c7c42746ac Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 22 May 2023 15:23:15 -0700 Subject: [PATCH 23/23] Add qlog support and update dependencies. --- server/Cargo.lock | 338 ++++++++++++++++++++++++++------- server/Cargo.toml | 2 +- server/src/transport/server.rs | 24 +++ 3 files changed, 291 insertions(+), 73 deletions(-) diff --git a/server/Cargo.lock b/server/Cargo.lock index 88d44d5..ffb049a 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -4,18 +4,27 @@ version = 3 [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" dependencies = [ "memchr", ] [[package]] -name = "anstream" -version = "0.3.0" +name = "android_system_properties" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e579a7752471abc2a8268df8b20005e3eadd975f585398f17efcfd8d4927371" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" dependencies = [ "anstyle", "anstyle-parse", @@ -52,9 +61,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcd8291a340dd8ac70e18878bc4501dd7b4ff970cfa21c207d36ece51ea88fd" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" dependencies = [ "anstyle", "windows-sys 0.48.0", @@ -62,9 +71,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.70" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" [[package]] name = "atty" @@ -83,6 +92,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "bitflags" version = "1.3.2" @@ -91,9 +106,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "byteorder" @@ -120,10 +135,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] -name = "clap" -version = "4.2.2" +name = "chrono" +version = "0.4.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b802d85aaf3a1cdb02b224ba472ebdea62014fccfcb269b95a4d76443b5ee5a" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" +dependencies = [ + "iana-time-zone", + "num-integer", + "num-traits", + "serde", + "winapi", +] + +[[package]] +name = "clap" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93aae7a4192245f70fe75dd9157fc7b4a5bf53e88d30bd4396f7d8f9284d5acc" dependencies = [ "clap_builder", "clap_derive", @@ -132,9 +160,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.2.2" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14a1a858f532119338887a4b8e1af9c60de8249cd7bafd68036a489e261e37b6" +checksum = "4f423e341edefb78c9caba2d9c7f7687d0e72e89df3ce3394554754393ac3990" dependencies = [ "anstream", "anstyle", @@ -145,21 +173,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.2.0" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" +checksum = "191d9573962933b4027f932c600cd252ce27a8ad5979418fe78e43c07996f27b" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.15", + "syn", ] [[package]] name = "clap_lex" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" [[package]] name = "cmake" @@ -176,6 +204,47 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + +[[package]] +name = "darling" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0558d22a7b463ed0241e993f76f09f30b126687447751a8638587b864e4b3944" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8bfa2e259f8ee1ce5e97824a3c55ec4404a0d772ca7fa96bf19f0752a046eb" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "env_logger" version = "0.9.3" @@ -210,6 +279,18 @@ dependencies = [ "libc", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "heck" version = "0.4.1" @@ -231,12 +312,58 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "humantime" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "iana-time-zone" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown", + "serde", +] + [[package]] name = "io-lifetimes" version = "1.0.10" @@ -268,9 +395,9 @@ checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "js-sys" -version = "0.3.61" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" dependencies = [ "wasm-bindgen", ] @@ -283,21 +410,21 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.141" +version = "0.2.144" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" +checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" [[package]] name = "libm" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" +checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" [[package]] name = "linux-raw-sys" -version = "0.3.1" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "log" @@ -396,13 +523,25 @@ checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8" dependencies = [ "unicode-ident", ] +[[package]] +name = "qlog" +version = "0.9.0" +source = "git+https://github.com/kixelated/quiche.git?branch=master#007a25b35b9509d673466fed8ddc73fd8d9b4184" +dependencies = [ + "serde", + "serde_derive", + "serde_json", + "serde_with", + "smallvec", +] + [[package]] name = "quiche" version = "0.17.1" @@ -414,6 +553,7 @@ dependencies = [ "libm", "log", "octets", + "qlog", "ring", "slab", "smallvec", @@ -422,18 +562,18 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" dependencies = [ "proc-macro2", ] [[package]] name = "regex" -version = "1.7.3" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" dependencies = [ "aho-corasick", "memchr", @@ -442,9 +582,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.29" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" [[package]] name = "ring" @@ -463,9 +603,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.11" +version = "0.37.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" dependencies = [ "bitflags", "errno", @@ -483,22 +623,22 @@ checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "serde" -version = "1.0.160" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.160" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn", ] [[package]] @@ -507,11 +647,40 @@ version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ + "indexmap", "itoa", "ryu", "serde", ] +[[package]] +name = "serde_with" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ff71d2c147a7b57362cead5e22f772cd52f6ab31cfcd9edcd7f6aeb2a0afbe" +dependencies = [ + "base64", + "chrono", + "hex", + "indexmap", + "serde", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "881b6f881b17d13214e5d494c939ebab463d01264ce1811e9d4ac3a882e7695f" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "slab" version = "0.4.8" @@ -544,20 +713,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.109" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" dependencies = [ "proc-macro2", "quote", @@ -590,7 +748,34 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn", +] + +[[package]] +name = "time" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3403384eaacbca9923fa06940178ac13e4edb725486d70e8e15881d0c836cc" +dependencies = [ + "itoa", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" + +[[package]] +name = "time-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" +dependencies = [ + "time-core", ] [[package]] @@ -635,9 +820,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -645,24 +830,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -670,28 +855,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" [[package]] name = "web-sys" -version = "0.3.61" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +checksum = "3bdd9ef4e984da1187bf8110c5cf5b845fbc87a23602cdf912386a76fcd3a7c2" dependencies = [ "js-sys", "wasm-bindgen", @@ -728,6 +913,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.0", +] + [[package]] name = "windows-sys" version = "0.45.0" diff --git a/server/Cargo.toml b/server/Cargo.toml index 1d04ec0..cf51b5a 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -quiche = { git = "https://github.com/kixelated/quiche.git", branch = "master" } # WebTransport fork +quiche = { git = "https://github.com/kixelated/quiche.git", branch = "master", features = [ "qlog" ] } # WebTransport fork clap = { version = "4.0", features = [ "derive" ] } log = { version = "0.4", features = ["std"] } mio = { version = "0.8", features = ["net", "os-poll"] } diff --git a/server/src/transport/server.rs b/server/src/transport/server.rs index 19031af..ed2dee3 100644 --- a/server/src/transport/server.rs +++ b/server/src/transport/server.rs @@ -240,6 +240,30 @@ impl Server { let mut conn = quiche::accept(&conn_id, odcid.as_ref(), local_addr, from, &mut self.quic)?; + // Log each session with QLOG if the ENV var is set. + if let Some(dir) = std::env::var_os("QLOGDIR") { + let id = format!("{:?}", &scid); + + let mut path = std::path::PathBuf::from(dir); + let filename = format!("server-{id}.sqlog"); + path.push(filename); + + let writer = match std::fs::File::create(&path) { + Ok(f) => std::io::BufWriter::new(f), + + Err(e) => panic!( + "Error creating qlog file attempted path was {:?}: {}", + path, e + ), + }; + + conn.set_qlog( + std::boxed::Box::new(writer), + "warp-server qlog".to_string(), + format!("{} id={}", "warp-server qlog", id), + ); + } + // Process potentially coalesced packets. conn.recv(src, info)?;