From 41c11d5fcbfd235e5e135468ddf7e48bb7fa1cdf Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 9 Jan 2026 15:50:08 -0800 Subject: [PATCH 01/15] Adding local video publisher & subcriber examples --- Cargo.lock | 499 +++++++++++++-- Cargo.toml | 1 + examples/local_video/Cargo.toml | 40 ++ examples/local_video/README.md | 50 ++ examples/local_video/src/publisher.rs | 416 +++++++++++++ examples/local_video/src/subscriber.rs | 762 +++++++++++++++++++++++ examples/local_video/src/yuv_shader.wgsl | 63 ++ yuv-sys/Cargo.toml | 1 + yuv-sys/build.rs | 24 +- 9 files changed, 1802 insertions(+), 54 deletions(-) create mode 100644 examples/local_video/Cargo.toml create mode 100644 examples/local_video/README.md create mode 100644 examples/local_video/src/publisher.rs create mode 100644 examples/local_video/src/subscriber.rs create mode 100644 examples/local_video/src/yuv_shader.wgsl diff --git a/Cargo.lock b/Cargo.lock index d1782360e..c37392bb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,7 +49,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ - "cfg-if", + "cfg-if 1.0.3", "cipher", "cpufeatures", ] @@ -72,7 +72,7 @@ version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ - "cfg-if", + "cfg-if 1.0.3", "getrandom 0.3.4", "once_cell", "serde", @@ -335,7 +335,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" dependencies = [ "autocfg", - "cfg-if", + "cfg-if 1.0.3", "concurrent-queue", "futures-io", "futures-lite 2.6.1", @@ -491,12 +491,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" dependencies = [ "addr2line", - "cfg-if", + "cfg-if 1.0.3", "libc", "miniz_oxide", "object", "rustc-demangle", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -539,6 +539,29 @@ dependencies = [ "tokio", ] +[[package]] +name = "bindgen" +version = "0.65.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "log", + "peeking_take_while", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.106", + "which", +] + [[package]] name = "bindgen" version = "0.72.1" @@ -833,7 +856,7 @@ dependencies = [ "js-sys", "num-traits", "wasm-bindgen", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -906,6 +929,34 @@ dependencies = [ "error-code", ] +[[package]] +name = "cocoa" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c49e86fc36d5704151f5996b7b3795385f50ce09e3be0f47a0cfde869681cf8" +dependencies = [ + "bitflags 1.3.2", + "block", + "core-foundation 0.7.0", + "core-graphics 0.19.2", + "foreign-types 0.3.2", + "libc", + "objc", +] + +[[package]] +name = "cocoa-foundation" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81411967c50ee9a1fc11365f8c585f863a22a9697c89239c452292c40ba79b0d" +dependencies = [ + "bitflags 2.9.4", + "block", + "core-foundation 0.10.1", + "core-graphics-types 0.2.0", + "objc", +] + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -1021,13 +1072,23 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "core-foundation" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" +dependencies = [ + "core-foundation-sys 0.7.0", + "libc", +] + [[package]] name = "core-foundation" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.8.7", "libc", ] @@ -1037,16 +1098,34 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.8.7", "libc", ] +[[package]] +name = "core-foundation-sys" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" + [[package]] name = "core-foundation-sys" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core-graphics" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3889374e6ea6ab25dba90bb5d96202f61108058361f6dc72e8b03e6f8bbe923" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.7.0", + "foreign-types 0.3.2", + "libc", +] + [[package]] name = "core-graphics" version = "0.23.2" @@ -1055,7 +1134,7 @@ checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081" dependencies = [ "bitflags 1.3.2", "core-foundation 0.9.4", - "core-graphics-types", + "core-graphics-types 0.1.3", "foreign-types 0.5.0", "libc", ] @@ -1071,6 +1150,42 @@ dependencies = [ "libc", ] +[[package]] +name = "core-graphics-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb" +dependencies = [ + "bitflags 2.9.4", + "core-foundation 0.10.1", + "libc", +] + +[[package]] +name = "core-media-sys" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "273bf3fc5bf51fd06a7766a84788c1540b6527130a0bce39e00567d6ab9f31f1" +dependencies = [ + "cfg-if 0.1.10", + "core-foundation-sys 0.7.0", + "libc", +] + +[[package]] +name = "core-video-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ecad23610ad9757664d644e369246edde1803fcb43ed72876565098a5d3828" +dependencies = [ + "cfg-if 0.1.10", + "core-foundation-sys 0.7.0", + "core-graphics 0.19.2", + "libc", + "metal 0.18.0", + "objc", +] + [[package]] name = "coreaudio-rs" version = "0.11.3" @@ -1078,7 +1193,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace" dependencies = [ "bitflags 1.3.2", - "core-foundation-sys", + "core-foundation-sys 0.8.7", "coreaudio-sys", ] @@ -1088,7 +1203,7 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ceec7a6067e62d6f931a2baf6f3a751f4a892595bcec1461a3c94ef9949864b6" dependencies = [ - "bindgen", + "bindgen 0.72.1", ] [[package]] @@ -1098,7 +1213,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779" dependencies = [ "alsa", - "core-foundation-sys", + "core-foundation-sys 0.8.7", "coreaudio-rs", "dasp_sample", "jni", @@ -1129,7 +1244,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ - "cfg-if", + "cfg-if 1.0.3", ] [[package]] @@ -1321,7 +1436,7 @@ version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ - "cfg-if", + "cfg-if 1.0.3", "hashbrown 0.14.5", "lock_api", "once_cell", @@ -1545,7 +1660,7 @@ version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ - "cfg-if", + "cfg-if 1.0.3", ] [[package]] @@ -1768,6 +1883,18 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "nanorand", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -2002,7 +2129,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ "rustix 1.1.2", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -2011,7 +2138,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ - "cfg-if", + "cfg-if 1.0.3", "js-sys", "libc", "wasi", @@ -2024,7 +2151,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ - "cfg-if", + "cfg-if 1.0.3", "js-sys", "libc", "r-efi", @@ -2242,7 +2369,7 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ - "cfg-if", + "cfg-if 1.0.3", "crunchy", "num-traits", "zerocopy", @@ -2532,7 +2659,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" dependencies = [ "android_system_properties", - "core-foundation-sys", + "core-foundation-sys 0.8.7", "iana-time-zone-haiku", "js-sys", "log", @@ -2731,7 +2858,7 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ - "cfg-if", + "cfg-if 1.0.3", ] [[package]] @@ -2878,7 +3005,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" dependencies = [ "cesu8", - "cfg-if", + "cfg-if 1.0.3", "combine", "jni-sys", "log", @@ -2967,6 +3094,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "lebe" version = "0.5.3" @@ -2985,8 +3118,8 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ - "cfg-if", - "windows-link", + "cfg-if 1.0.3", + "windows-link 0.2.1", ] [[package]] @@ -3203,6 +3336,33 @@ dependencies = [ "tokio", ] +[[package]] +name = "local_video" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytemuck", + "clap", + "eframe", + "egui", + "egui-wgpu", + "env_logger 0.10.2", + "futures", + "image 0.24.9", + "libwebrtc", + "livekit", + "livekit-api", + "log", + "nokhwa", + "objc2 0.6.3", + "parking_lot", + "tokio", + "webrtc-sys", + "wgpu 25.0.2", + "winit", + "yuv-sys", +] + [[package]] name = "lock_api" version = "0.4.14" @@ -3275,6 +3435,21 @@ dependencies = [ "libc", ] +[[package]] +name = "metal" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e198a0ee42bdbe9ef2c09d0b9426f3b2b47d90d93a4a9b0395c4cea605e92dc0" +dependencies = [ + "bitflags 1.3.2", + "block", + "cocoa", + "core-graphics 0.19.2", + "foreign-types 0.3.2", + "log", + "objc", +] + [[package]] name = "metal" version = "0.31.0" @@ -3283,7 +3458,7 @@ checksum = "f569fb946490b5743ad69813cb19629130ce9374034abe31614a36402d18f99e" dependencies = [ "bitflags 2.10.0", "block", - "core-graphics-types", + "core-graphics-types 0.1.3", "foreign-types 0.5.0", "log", "objc", @@ -3399,6 +3574,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom 0.2.16", +] + [[package]] name = "native-tls" version = "0.2.14" @@ -3475,6 +3659,72 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" +[[package]] +name = "nokhwa" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c35ed9613f002f8095aafc97ad839e0bb6cebf79111c68265d8df212a5a294" +dependencies = [ + "flume", + "image 0.25.8", + "nokhwa-bindings-linux", + "nokhwa-bindings-macos", + "nokhwa-bindings-windows", + "nokhwa-core", + "parking_lot", + "paste", + "thiserror 2.0.17", +] + +[[package]] +name = "nokhwa-bindings-linux" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9201817bb00fa911c0aaf5ae7653b2f7a81a0492d119753ac85b74c2c5f177f" +dependencies = [ + "nokhwa-core", + "v4l", +] + +[[package]] +name = "nokhwa-bindings-macos" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de78eb4a2d47a68f490899aa0516070d7a972f853ec2bb374ab53be0bd39b60f" +dependencies = [ + "block", + "cocoa-foundation", + "core-foundation 0.10.1", + "core-media-sys", + "core-video-sys", + "flume", + "nokhwa-core", + "objc", + "once_cell", +] + +[[package]] +name = "nokhwa-bindings-windows" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2597245a984c92a9f2bcb239d85bbc62b34f8b277c2648f51f5c78b84b38da46" +dependencies = [ + "nokhwa-core", + "once_cell", + "windows 0.61.3", +] + +[[package]] +name = "nokhwa-core" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "903f3e0f406f7e9aad4fa0566c1d97cc7f88aab57847e1f919d1a34812dedee3" +dependencies = [ + "bytes", + "image 0.25.8", + "thiserror 2.0.17", +] + [[package]] name = "nom" version = "7.1.3" @@ -3550,6 +3800,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" dependencies = [ "malloc_buf", + "objc_exception", ] [[package]] @@ -3822,6 +4073,15 @@ dependencies = [ "objc2-foundation 0.2.2", ] +[[package]] +name = "objc_exception" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad970fb455818ad6cba4c122ad012fae53ae8b4795f86378bce65e4f6bab2ca4" +dependencies = [ + "cc", +] + [[package]] name = "object" version = "0.37.3" @@ -3970,12 +4230,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "backtrace", - "cfg-if", + "cfg-if 1.0.3", "libc", "petgraph 0.6.5", "redox_syscall 0.5.18", "smallvec", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -4044,6 +4304,12 @@ dependencies = [ "sha2", ] +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -4164,7 +4430,7 @@ checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" dependencies = [ "autocfg", "bitflags 1.3.2", - "cfg-if", + "cfg-if 1.0.3", "concurrent-queue", "libc", "log", @@ -4178,7 +4444,7 @@ version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" dependencies = [ - "cfg-if", + "cfg-if 1.0.3", "concurrent-queue", "hermit-abi", "pin-project-lite", @@ -4706,7 +4972,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.3", "getrandom 0.2.16", "libc", "untrusted", @@ -4973,7 +5239,7 @@ checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags 2.10.0", "core-foundation 0.9.4", - "core-foundation-sys", + "core-foundation-sys 0.8.7", "libc", "security-framework-sys", ] @@ -4986,7 +5252,7 @@ checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" dependencies = [ "bitflags 2.10.0", "core-foundation 0.10.1", - "core-foundation-sys", + "core-foundation-sys 0.8.7", "libc", "security-framework-sys", ] @@ -4997,7 +5263,7 @@ version = "2.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.8.7", "libc", ] @@ -5090,7 +5356,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ - "cfg-if", + "cfg-if 1.0.3", "cpufeatures", "digest", ] @@ -5101,7 +5367,7 @@ version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ - "cfg-if", + "cfg-if 1.0.3", "cpufeatures", "digest", ] @@ -5268,6 +5534,15 @@ dependencies = [ "hound", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + [[package]] name = "spirv" version = "0.3.0+sdk-1.3.268.0" @@ -5492,7 +5767,7 @@ version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ - "cfg-if", + "cfg-if 1.0.3", ] [[package]] @@ -5548,7 +5823,7 @@ dependencies = [ "arrayref", "arrayvec", "bytemuck", - "cfg-if", + "cfg-if 1.0.3", "log", "tiny-skia-path", ] @@ -6050,6 +6325,26 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "v4l" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8fbfea44a46799d62c55323f3c55d06df722fbe577851d848d328a1041c3403" +dependencies = [ + "bitflags 1.3.2", + "libc", + "v4l2-sys-mit", +] + +[[package]] +name = "v4l2-sys-mit" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6779878362b9bacadc7893eac76abe69612e8837ef746573c4a5239daf11990b" +dependencies = [ + "bindgen 0.65.1", +] + [[package]] name = "valuable" version = "0.1.1" @@ -6126,7 +6421,7 @@ version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60" dependencies = [ - "cfg-if", + "cfg-if 1.0.3", "once_cell", "rustversion", "wasm-bindgen-macro", @@ -6139,7 +6434,7 @@ version = "0.4.55" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "551f88106c6d5e7ccc7cd9a16f312dd3b5d36ea8b4954304657d5dfba115d4a0" dependencies = [ - "cfg-if", + "cfg-if 1.0.3", "js-sys", "once_cell", "wasm-bindgen", @@ -6556,7 +6851,7 @@ dependencies = [ "block", "bytemuck", "cfg_aliases", - "core-graphics-types", + "core-graphics-types 0.1.3", "glow", "glutin_wgl_sys", "gpu-alloc", @@ -6566,7 +6861,7 @@ dependencies = [ "libc", "libloading", "log", - "metal", + "metal 0.31.0", "naga 24.0.0", "ndk-sys 0.5.0+25.2.9519653", "objc", @@ -6598,9 +6893,9 @@ dependencies = [ "bitflags 2.10.0", "block", "bytemuck", - "cfg-if", + "cfg-if 1.0.3", "cfg_aliases", - "core-graphics-types", + "core-graphics-types 0.1.3", "glow", "glutin_wgl_sys", "gpu-alloc", @@ -6612,7 +6907,7 @@ dependencies = [ "libc", "libloading", "log", - "metal", + "metal 0.31.0", "naga 25.0.1", "ndk-sys 0.5.0+25.2.9519653", "objc", @@ -6680,6 +6975,18 @@ dependencies = [ "winit", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "winapi" version = "0.3.9" @@ -6731,6 +7038,28 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections", + "windows-core 0.61.2", + "windows-future", + "windows-link 0.1.3", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core 0.61.2", +] + [[package]] name = "windows-core" version = "0.54.0" @@ -6754,6 +7083,19 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement 0.60.2", + "windows-interface 0.59.3", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.4.2", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -6762,11 +7104,22 @@ checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement 0.60.2", "windows-interface 0.59.3", - "windows-link", + "windows-link 0.2.1", "windows-result 0.4.1", "windows-strings 0.5.1", ] +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", + "windows-threading", +] + [[package]] name = "windows-implement" version = "0.58.0" @@ -6811,12 +7164,28 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", +] + [[package]] name = "windows-result" version = "0.1.2" @@ -6835,13 +7204,22 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-result" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -6854,13 +7232,22 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-strings" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -6914,7 +7301,7 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -6969,7 +7356,7 @@ version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ - "windows-link", + "windows-link 0.2.1", "windows_aarch64_gnullvm 0.53.1", "windows_aarch64_msvc 0.53.1", "windows_i686_gnu 0.53.1", @@ -6980,6 +7367,15 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -7176,7 +7572,7 @@ dependencies = [ "cfg_aliases", "concurrent-queue", "core-foundation 0.9.4", - "core-graphics", + "core-graphics 0.23.2", "cursor-icon", "dpi", "js-sys", @@ -7323,8 +7719,9 @@ dependencies = [ name = "yuv-sys" version = "0.3.10" dependencies = [ - "bindgen", + "bindgen 0.72.1", "cc", + "pkg-config", "rayon", "regex", ] diff --git a/Cargo.toml b/Cargo.toml index 2e9267ecb..c42dcbaa2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ "examples/basic_text_stream", "examples/encrypted_text_stream", "examples/local_audio", + "examples/local_video", "examples/mobile", "examples/play_from_disk", "examples/rpc", diff --git a/examples/local_video/Cargo.toml b/examples/local_video/Cargo.toml new file mode 100644 index 000000000..8d01c1086 --- /dev/null +++ b/examples/local_video/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "local_video" +version = "0.1.0" +edition = "2021" +publish = false + +[[bin]] +name = "publisher" +path = "src/publisher.rs" + +[[bin]] +name = "subscriber" +path = "src/subscriber.rs" + +[dependencies] +tokio = { version = "1", features = ["full", "parking_lot"] } +livekit = { workspace = true, features = ["rustls-tls-native-roots"] } +webrtc-sys = { workspace = true } +libwebrtc = { workspace = true } +livekit-api = { workspace = true } +yuv-sys = { workspace = true } +futures = "0.3" +clap = { version = "4.5", features = ["derive"] } +log = "0.4" +env_logger = "0.10.0" +nokhwa = { version = "0.10", default-features = false, features = ["input-avfoundation", "input-v4l", "input-msmf", "output-threaded"] } +image = "0.24" +egui = "0.31.1" +egui-wgpu = "0.31.1" +eframe = { version = "0.31.1", default-features = false, features = ["default_fonts", "wgpu", "persistence"] } +wgpu = "25.0" +winit = { version = "0.30.11", features = ["android-native-activity"] } +parking_lot = { version = "0.12.1", features = ["deadlock_detection"] } +anyhow = "1" +bytemuck = { version = "1.16", features = ["derive"] } + +[target.'cfg(target_os = "macos")'.dependencies] +objc2 = { version = "0.6.0", features = ["relax-sign-encoding"] } + + diff --git a/examples/local_video/README.md b/examples/local_video/README.md new file mode 100644 index 000000000..aa39facbd --- /dev/null +++ b/examples/local_video/README.md @@ -0,0 +1,50 @@ +# local_video + +Two examples demonstrating capturing frames from a local camera video and publishing to LiveKit, and subscribing to render video in a window. + +- publisher: capture from a selected camera and publish a video track +- subscriber: connect to a room, subscribe to video tracks, and display in a window + +LiveKit connection can be provided via flags or environment variables: +- `--url` or `LIVEKIT_URL` +- `--api-key` or `LIVEKIT_API_KEY` +- `--api-secret` or `LIVEKIT_API_SECRET` + +Publisher usage: +``` + cargo run -p local_video --bin publisher -- --list-cameras + cargo run -p local_video --bin publisher -- --camera-index 0 --room-name demo --identity cam-1 + + # with explicit LiveKit connection flags + cargo run -p local_video --bin publisher -- \ + --camera-index 0 \ + --room-name demo \ + --identity cam-1 \ + --url https://your.livekit.server \ + --api-key YOUR_KEY \ + --api-secret YOUR_SECRET +``` + +Subscriber usage: +``` + # relies on env vars LIVEKIT_URL, LIVEKIT_API_KEY, LIVEKIT_API_SECRET + cargo run -p local_video --bin subscriber -- --room-name demo --identity viewer-1 + + # or pass credentials via flags + cargo run -p local_video --bin subscriber -- \ + --room-name demo \ + --identity viewer-1 \ + --url https://your.livekit.server \ + --api-key YOUR_KEY \ + --api-secret YOUR_SECRET + + # subscribe to a specific participant's video only + cargo run -p local_video --bin subscriber -- \ + --room-name demo \ + --identity viewer-1 \ + --participant alice +``` + +Notes: +- `--participant` limits subscription to video tracks from the specified participant identity. +- If the active video track is unsubscribed or unpublished, the app clears its state and will automatically attach to the next matching video track when it appears. diff --git a/examples/local_video/src/publisher.rs b/examples/local_video/src/publisher.rs new file mode 100644 index 000000000..d311129b9 --- /dev/null +++ b/examples/local_video/src/publisher.rs @@ -0,0 +1,416 @@ +use anyhow::Result; +use clap::Parser; +use livekit::options::{TrackPublishOptions, VideoCodec, VideoEncoding}; +use livekit::prelude::*; +use livekit::webrtc::video_frame::{I420Buffer, VideoFrame, VideoRotation}; +use livekit::webrtc::video_source::native::NativeVideoSource; +use livekit::webrtc::video_source::{RtcVideoSource, VideoResolution}; +use livekit_api::access_token; +use log::{debug, info}; +use yuv_sys as yuv_sys; +use nokhwa::pixel_format::RgbFormat; +use nokhwa::utils::{ApiBackend, CameraFormat, CameraIndex, FrameFormat, RequestedFormat, RequestedFormatType, Resolution}; +use nokhwa::Camera; +use std::env; +use std::time::{Duration, Instant}; + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +struct Args { + /// List available cameras and exit + #[arg(long)] + list_cameras: bool, + + /// Camera index to use (numeric) + #[arg(long, default_value_t = 0)] + camera_index: usize, + + /// Desired width + #[arg(long, default_value_t = 1280)] + width: u32, + + /// Desired height + #[arg(long, default_value_t = 720)] + height: u32, + + /// Desired framerate + #[arg(long, default_value_t = 30)] + fps: u32, + + /// Max video bitrate for the main layer in bps (optional) + #[arg(long)] + max_bitrate: Option, + + /// LiveKit participant identity + #[arg(long, default_value = "rust-camera-pub")] + identity: String, + + /// LiveKit room name + #[arg(long, default_value = "video-room")] + room_name: String, + + /// LiveKit server URL + #[arg(long)] + url: Option, + + /// LiveKit API key + #[arg(long)] + api_key: Option, + + /// LiveKit API secret + #[arg(long)] + api_secret: Option, + + /// Use H.265/HEVC encoding if supported (falls back to H.264 on failure) + #[arg(long, default_value_t = false)] + h265: bool, +} + +fn list_cameras() -> Result<()> { + let cams = nokhwa::query(ApiBackend::Auto)?; + println!("Available cameras:"); + for (i, cam) in cams.iter().enumerate() { + println!("{}. {}", i, cam.human_name()); + } + Ok(()) +} + +#[tokio::main] +async fn main() -> Result<()> { + env_logger::init(); + let args = Args::parse(); + + if args.list_cameras { + return list_cameras(); + } + + // LiveKit connection details + let url = args.url.or_else(|| env::var("LIVEKIT_URL").ok()).expect( + "LIVEKIT_URL must be provided via --url or env", + ); + let api_key = args + .api_key + .or_else(|| env::var("LIVEKIT_API_KEY").ok()) + .expect("LIVEKIT_API_KEY must be provided via --api-key or env"); + let api_secret = args + .api_secret + .or_else(|| env::var("LIVEKIT_API_SECRET").ok()) + .expect("LIVEKIT_API_SECRET must be provided via --api-secret or env"); + + let token = access_token::AccessToken::with_api_key(&api_key, &api_secret) + .with_identity(&args.identity) + .with_name(&args.identity) + .with_grants(access_token::VideoGrants { + room_join: true, + room: args.room_name.clone(), + can_publish: true, + ..Default::default() + }) + .to_jwt()?; + + info!("Connecting to LiveKit room '{}' as '{}'...", args.room_name, args.identity); + let mut room_options = RoomOptions::default(); + room_options.auto_subscribe = true; + let (room, _) = Room::connect(&url, &token, room_options).await?; + let room = std::sync::Arc::new(room); + info!("Connected: {} - {}", room.name(), room.sid().await); + + // Log room events + { + let room_clone = room.clone(); + tokio::spawn(async move { + let mut events = room_clone.subscribe(); + info!("Subscribed to room events"); + while let Some(evt) = events.recv().await { + debug!("Room event: {:?}", evt); + } + }); + } + + // Setup camera + let index = CameraIndex::Index(args.camera_index as u32); + let requested = RequestedFormat::new::(RequestedFormatType::AbsoluteHighestFrameRate); + let mut camera = Camera::new(index, requested)?; + // Try raw YUYV first (cheaper than MJPEG), fall back to MJPEG + let wanted = CameraFormat::new( + Resolution::new(args.width, args.height), + FrameFormat::YUYV, + args.fps, + ); + let mut using_fmt = "YUYV"; + if let Err(_) = camera.set_camera_requset(RequestedFormat::new::(RequestedFormatType::Exact(wanted))) { + let alt = CameraFormat::new( + Resolution::new(args.width, args.height), + FrameFormat::MJPEG, + args.fps, + ); + using_fmt = "MJPEG"; + let _ = camera.set_camera_requset(RequestedFormat::new::(RequestedFormatType::Exact(alt))); + } + camera.open_stream()?; + let fmt = camera.camera_format(); + let width = fmt.width(); + let height = fmt.height(); + let fps = fmt.frame_rate(); + info!("Camera opened: {}x{} @ {} fps (format: {})", width, height, fps, using_fmt); + debug!("Negotiated nokhwa CameraFormat: {:?}", fmt); + // Pace publishing at the requested FPS (not the camera-reported FPS) to hit desired cadence + let pace_fps = args.fps as f64; + + // Create LiveKit video source and track + let rtc_source = NativeVideoSource::new(VideoResolution { width, height }); + let track = LocalVideoTrack::create_video_track( + "camera", + RtcVideoSource::Native(rtc_source.clone()), + ); + + // Choose requested codec and attempt to publish; if H.265 fails, retry with H.264 + let requested_codec = if args.h265 { VideoCodec::H265 } else { VideoCodec::H264 }; + info!("Attempting publish with codec: {}", requested_codec.as_str()); + + let publish_opts = |codec: VideoCodec| { + let mut opts = TrackPublishOptions { + source: TrackSource::Camera, + simulcast: false, + video_codec: codec, + ..Default::default() + }; + if let Some(bitrate) = args.max_bitrate { + opts.video_encoding = Some(VideoEncoding { + max_bitrate: bitrate, + max_framerate: args.fps as f64, + }); + } + opts + }; + + let publish_result = room + .local_participant() + .publish_track(LocalTrack::Video(track.clone()), publish_opts(requested_codec)) + .await; + + if let Err(e) = publish_result { + if matches!(requested_codec, VideoCodec::H265) { + log::warn!( + "H.265 publish failed ({}). Falling back to H.264...", + e + ); + room + .local_participant() + .publish_track(LocalTrack::Video(track.clone()), publish_opts(VideoCodec::H264)) + .await?; + info!("Published camera track with H.264 fallback"); + } else { + return Err(e.into()); + } + } else { + info!("Published camera track"); + } + + // Reusable I420 buffer and frame + let mut frame = VideoFrame { rotation: VideoRotation::VideoRotation0, timestamp_us: 0, buffer: I420Buffer::new(width, height) }; + let is_yuyv = fmt.format() == FrameFormat::YUYV; + info!( + "Selected conversion path: {}", + if is_yuyv { "YUYV->I420 (libyuv)" } else { "Auto (RGB24 or MJPEG)" } + ); + + // Accurate pacing using absolute schedule (no drift) + let mut ticker = tokio::time::interval(Duration::from_secs_f64(1.0 / pace_fps)); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + // Align the first tick to now + ticker.tick().await; + let start_ts = Instant::now(); + + // Capture loop + let mut frames: u64 = 0; + let mut last_fps_log = Instant::now(); + let target = Duration::from_secs_f64(1.0 / pace_fps); + info!("Target frame interval: {:.2} ms", target.as_secs_f64() * 1000.0); + + // Timing accumulators (ms) for rolling stats + let mut sum_get_ms = 0.0; + let mut sum_decode_ms = 0.0; + let mut sum_convert_ms = 0.0; + let mut sum_capture_ms = 0.0; + let mut sum_sleep_ms = 0.0; + let mut sum_iter_ms = 0.0; + let mut logged_mjpeg_fallback = false; + loop { + // Wait until the scheduled next frame time + let wait_start = Instant::now(); + ticker.tick().await; + let iter_start = Instant::now(); + + // Get frame as RGB24 (decoded by nokhwa if needed) + let t0 = Instant::now(); + let frame_buf = camera.frame()?; + let t1 = Instant::now(); + let (stride_y, stride_u, stride_v) = frame.buffer.strides(); + let (data_y, data_u, data_v) = frame.buffer.data_mut(); + // Fast path for YUYV: convert directly to I420 via libyuv + let t2 = if is_yuyv { + let src = frame_buf.buffer(); + let src_bytes = src.as_ref(); + let src_stride = (width * 2) as i32; // YUYV packed 4:2:2 + let t2_local = t1; // no decode step in YUYV path + unsafe { + // returns 0 on success + let _ = yuv_sys::rs_YUY2ToI420( + src_bytes.as_ptr(), + src_stride, + data_y.as_mut_ptr(), + stride_y as i32, + data_u.as_mut_ptr(), + stride_u as i32, + data_v.as_mut_ptr(), + stride_v as i32, + width as i32, + height as i32, + ); + } + t2_local + } else { + // Auto path (either RGB24 already or compressed MJPEG) + let src = frame_buf.buffer(); + let t2_local = if src.len() == (width as usize * height as usize * 3) { + // Already RGB24 from backend; convert directly + unsafe { + let _ = yuv_sys::rs_RGB24ToI420( + src.as_ref().as_ptr(), + (width * 3) as i32, + data_y.as_mut_ptr(), + stride_y as i32, + data_u.as_mut_ptr(), + stride_u as i32, + data_v.as_mut_ptr(), + stride_v as i32, + width as i32, + height as i32, + ); + } + Instant::now() + } else { + // Try fast MJPEG->I420 via libyuv if available; fallback to image crate + let mut used_fast_mjpeg = false; + let t2_try = unsafe { + // rs_MJPGToI420 returns 0 on success + let ret = yuv_sys::rs_MJPGToI420( + src.as_ref().as_ptr(), + src.len(), + data_y.as_mut_ptr(), + stride_y as i32, + data_u.as_mut_ptr(), + stride_u as i32, + data_v.as_mut_ptr(), + stride_v as i32, + width as i32, + height as i32, + width as i32, + height as i32, + ); + if ret == 0 { used_fast_mjpeg = true; Instant::now() } else { t1 } + }; + if used_fast_mjpeg { + t2_try + } else { + // Fallback: decode MJPEG using image crate then RGB24->I420 + match image::load_from_memory(src.as_ref()) { + Ok(img_dyn) => { + let rgb8 = img_dyn.to_rgb8(); + let dec_w = rgb8.width() as u32; + let dec_h = rgb8.height() as u32; + if dec_w != width || dec_h != height { + log::warn!( + "Decoded MJPEG size {}x{} differs from requested {}x{}; dropping frame", + dec_w, dec_h, width, height + ); + continue; + } + unsafe { + let _ = yuv_sys::rs_RGB24ToI420( + rgb8.as_raw().as_ptr(), + (dec_w * 3) as i32, + data_y.as_mut_ptr(), + stride_y as i32, + data_u.as_mut_ptr(), + stride_u as i32, + data_v.as_mut_ptr(), + stride_v as i32, + width as i32, + height as i32, + ); + } + Instant::now() + } + Err(e2) => { + if !logged_mjpeg_fallback { + log::error!( + "MJPEG decode failed; buffer not RGB24 and image decode failed: {}", + e2 + ); + logged_mjpeg_fallback = true; + } + continue; + } + } + } + }; + t2_local + }; + let t3 = Instant::now(); + + // Update RTP timestamp (monotonic, microseconds since start) + frame.timestamp_us = start_ts.elapsed().as_micros() as i64; + rtc_source.capture_frame(&frame); + let t4 = Instant::now(); + + frames += 1; + // We already paced via interval; measure actual sleep time for logging only + let sleep_dur = iter_start - wait_start; + + // Per-iteration timing bookkeeping + let t_end = Instant::now(); + let get_ms = (t1 - t0).as_secs_f64() * 1000.0; + let decode_ms = (t2 - t1).as_secs_f64() * 1000.0; + let convert_ms = (t3 - t2).as_secs_f64() * 1000.0; + let capture_ms = (t4 - t3).as_secs_f64() * 1000.0; + let sleep_ms = sleep_dur.as_secs_f64() * 1000.0; + let iter_ms = (t_end - iter_start).as_secs_f64() * 1000.0; + sum_get_ms += get_ms; + sum_decode_ms += decode_ms; + sum_convert_ms += convert_ms; + sum_capture_ms += capture_ms; + sum_sleep_ms += sleep_ms; + sum_iter_ms += iter_ms; + + if last_fps_log.elapsed() >= std::time::Duration::from_secs(2) { + let secs = last_fps_log.elapsed().as_secs_f64(); + let fps_est = frames as f64 / secs; + let n = frames.max(1) as f64; + info!( + "Publishing video: {}x{}, ~{:.1} fps | avg ms: get {:.2}, decode {:.2}, convert {:.2}, capture {:.2}, sleep {:.2}, iter {:.2} | target {:.2}", + width, + height, + fps_est, + sum_get_ms / n, + sum_decode_ms / n, + sum_convert_ms / n, + sum_capture_ms / n, + sum_sleep_ms / n, + sum_iter_ms / n, + target.as_secs_f64() * 1000.0, + ); + frames = 0; + sum_get_ms = 0.0; + sum_decode_ms = 0.0; + sum_convert_ms = 0.0; + sum_capture_ms = 0.0; + sum_sleep_ms = 0.0; + sum_iter_ms = 0.0; + last_fps_log = Instant::now(); + } + } +} + + diff --git a/examples/local_video/src/subscriber.rs b/examples/local_video/src/subscriber.rs new file mode 100644 index 000000000..db719ee02 --- /dev/null +++ b/examples/local_video/src/subscriber.rs @@ -0,0 +1,762 @@ +use anyhow::Result; +use clap::Parser; +use eframe::egui; +use egui_wgpu as egui_wgpu_backend; +use egui_wgpu_backend::CallbackTrait; +use eframe::wgpu::{self, util::DeviceExt}; +use futures::StreamExt; +use livekit::prelude::*; +use livekit::webrtc::video_stream::native::NativeVideoStream; +use livekit_api::access_token; +use log::{debug, info}; +use parking_lot::Mutex; +use std::{ + collections::HashMap, + env, + sync::Arc, + time::{Duration, Instant}, +}; + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +struct Args { + /// LiveKit participant identity + #[arg(long, default_value = "rust-video-subscriber")] + identity: String, + + /// LiveKit room name + #[arg(long, default_value = "video-room")] + room_name: String, + + /// LiveKit server URL + #[arg(long)] + url: Option, + + /// LiveKit API key (can also be set via LIVEKIT_API_KEY environment variable) + #[arg(long)] + api_key: Option, + + /// LiveKit API secret (can also be set via LIVEKIT_API_SECRET environment variable) + #[arg(long)] + api_secret: Option, + + /// Only subscribe to video from this participant identity + #[arg(long)] + participant: Option, +} + +struct SharedYuv { + width: u32, + height: u32, + stride_y: u32, + stride_u: u32, + stride_v: u32, + y: Vec, + u: Vec, + v: Vec, + dirty: bool, +} + +#[derive(Clone)] +struct SimulcastState { + available: bool, + publication: Option, + requested_quality: Option, + active_quality: Option, + full_dims: Option<(u32, u32)>, +} + +impl Default for SimulcastState { + fn default() -> Self { + Self { + available: false, + publication: None, + requested_quality: None, + active_quality: None, + full_dims: None, + } + } +} + +fn infer_quality_from_dims( + full_w: u32, + _full_h: u32, + cur_w: u32, + _cur_h: u32, +) -> livekit::track::VideoQuality { + if full_w == 0 { + return livekit::track::VideoQuality::High; + } + let ratio = cur_w as f32 / full_w as f32; + if ratio >= 0.75 { + livekit::track::VideoQuality::High + } else if ratio >= 0.45 { + livekit::track::VideoQuality::Medium + } else { + livekit::track::VideoQuality::Low + } +} + +fn simulcast_state_full_dims( + state: &Arc>, +) -> Option<(u32, u32)> { + let sc = state.lock(); + sc.full_dims +} + +struct VideoApp { + shared: Arc>, + simulcast: Arc>, +} + +impl eframe::App for VideoApp { + fn update(&mut self, ctx: &egui::Context, _frame: &mut eframe::Frame) { + egui::CentralPanel::default().show(ctx, |ui| { + let available = ui.available_size(); + let rect = egui::Rect::from_min_size(ui.min_rect().min, available); + + // Ensure we keep repainting for smooth playback + ui.ctx().request_repaint(); + + // Add a custom wgpu paint callback that renders I420 directly + let cb = egui_wgpu_backend::Callback::new_paint_callback( + rect, + YuvPaintCallback { shared: self.shared.clone() }, + ); + ui.painter().add(cb); + }); + + // Simulcast layer controls: bottom-left overlay + egui::Area::new("simulcast_controls".into()) + .anchor(egui::Align2::LEFT_BOTTOM, egui::vec2(10.0, -10.0)) + .interactable(true) + .show(ctx, |ui| { + let mut sc = self.simulcast.lock(); + if !sc.available { + return; + } + let selected = sc.requested_quality.or(sc.active_quality); + ui.horizontal(|ui| { + let choices = [ + (livekit::track::VideoQuality::Low, "Low"), + (livekit::track::VideoQuality::Medium, "Med"), + (livekit::track::VideoQuality::High, "High"), + ]; + for (q, label) in choices { + let is_selected = selected.is_some_and(|s| s == q); + let resp = ui.selectable_label(is_selected, label); + if resp.clicked() { + if let Some(ref pub_remote) = sc.publication { + pub_remote.set_video_quality(q); + sc.requested_quality = Some(q); + } + } + } + }); + }); + + ctx.request_repaint_after(Duration::from_millis(16)); + } +} + +#[tokio::main] +async fn main() -> Result<()> { + env_logger::init(); + let args = Args::parse(); + + // LiveKit connection details (prefer CLI args, fallback to env vars) + let url = args + .url + .or_else(|| env::var("LIVEKIT_URL").ok()) + .expect("LiveKit URL must be provided via --url argument or LIVEKIT_URL environment variable"); + let api_key = args + .api_key + .or_else(|| env::var("LIVEKIT_API_KEY").ok()) + .expect("LiveKit API key must be provided via --api-key argument or LIVEKIT_API_KEY environment variable"); + let api_secret = args + .api_secret + .or_else(|| env::var("LIVEKIT_API_SECRET").ok()) + .expect("LiveKit API secret must be provided via --api-secret argument or LIVEKIT_API_SECRET environment variable"); + + let token = access_token::AccessToken::with_api_key(&api_key, &api_secret) + .with_identity(&args.identity) + .with_name(&args.identity) + .with_grants(access_token::VideoGrants { + room_join: true, + room: args.room_name.clone(), + can_subscribe: true, + ..Default::default() + }) + .to_jwt()?; + + info!("Connecting to LiveKit room '{}' as '{}'...", args.room_name, args.identity); + let mut room_options = RoomOptions::default(); + room_options.auto_subscribe = true; + let (room, _) = Room::connect(&url, &token, room_options).await?; + let room = Arc::new(room); + info!("Connected: {} - {}", room.name(), room.sid().await); + + // Shared YUV buffer for UI/GPU + let shared = Arc::new(Mutex::new(SharedYuv { + width: 0, + height: 0, + stride_y: 0, + stride_u: 0, + stride_v: 0, + y: Vec::new(), + u: Vec::new(), + v: Vec::new(), + dirty: false, + })); + + // Subscribe to room events: on first video track, start sink task + let allowed_identity = args.participant.clone(); + let shared_clone = shared.clone(); + let rt = tokio::runtime::Handle::current(); + // Track currently active video track SID to handle unpublish/unsubscribe + let active_sid = Arc::new(Mutex::new(None::)); + // Shared simulcast UI/control state + let simulcast = Arc::new(Mutex::new(SimulcastState::default())); + let simulcast_events = simulcast.clone(); + tokio::spawn(async move { + let active_sid = active_sid.clone(); + let simulcast = simulcast_events; + let mut events = room.subscribe(); + info!("Subscribed to room events"); + while let Some(evt) = events.recv().await { + debug!("Room event: {:?}", evt); + match evt { + RoomEvent::TrackSubscribed { track, publication, participant } => { + // If a participant filter is set, skip others + if let Some(ref allow) = allowed_identity { + if participant.identity().as_str() != allow { + debug!("Skipping track from '{}' (filter set to '{}')", participant.identity(), allow); + continue; + } + } + if let livekit::track::RemoteTrack::Video(video_track) = track { + let sid = publication.sid().clone(); + // Only handle if we don't already have an active video track + { + let mut active = active_sid.lock(); + if active.as_ref() == Some(&sid) { + debug!("Track {} already active, ignoring duplicate subscribe", sid); + continue; + } + if active.is_some() { + debug!("A video track is already active ({}), ignoring new subscribe {}", active.as_ref().unwrap(), sid); + continue; + } + *active = Some(sid.clone()); + } + + info!( + "Subscribed to video track: {} (sid {}) from {} - codec: {}, simulcast: {}, dimension: {}x{}", + publication.name(), + publication.sid(), + participant.identity(), + publication.mime_type(), + publication.simulcasted(), + publication.dimension().0, + publication.dimension().1 + ); + + // Try to fetch inbound RTP/codec stats for more details + match video_track.get_stats().await { + Ok(stats) => { + let mut codec_by_id: HashMap = HashMap::new(); + let mut inbound: Option = None; + for s in stats.iter() { + match s { + livekit::webrtc::stats::RtcStats::Codec(c) => { + codec_by_id.insert( + c.rtc.id.clone(), + (c.codec.mime_type.clone(), c.codec.sdp_fmtp_line.clone()), + ); + } + livekit::webrtc::stats::RtcStats::InboundRtp(i) => { + if i.stream.kind == "video" { + inbound = Some(i.clone()); + } + } + _ => {} + } + } + + if let Some(i) = inbound { + if let Some((mime, fmtp)) = codec_by_id.get(&i.stream.codec_id) { + info!("Inbound codec: {} (fmtp: {})", mime, fmtp); + } else { + info!("Inbound codec id: {}", i.stream.codec_id); + } + info!( + "Inbound current layer: {}x{} ~{:.1} fps, decoder: {}, power_efficient: {}", + i.inbound.frame_width, + i.inbound.frame_height, + i.inbound.frames_per_second, + i.inbound.decoder_implementation, + i.inbound.power_efficient_decoder + ); + } + } + Err(e) => debug!("Failed to get stats for video track: {:?}", e), + } + // Start background sink thread + let shared2 = shared_clone.clone(); + let active_sid2 = active_sid.clone(); + let my_sid = sid.clone(); + let rt_clone = rt.clone(); + // Initialize simulcast state for this publication + { + let mut sc = simulcast.lock(); + sc.available = publication.simulcasted(); + let dim = publication.dimension(); + sc.full_dims = Some((dim.0, dim.1)); + sc.requested_quality = None; + sc.active_quality = None; + sc.publication = Some(publication.clone()); + } + let simulcast2 = simulcast.clone(); + std::thread::spawn(move || { + let mut sink = NativeVideoStream::new(video_track.rtc_track()); + let mut frames: u64 = 0; + let mut last_log = Instant::now(); + let mut logged_first = false; + let mut last_stats = Instant::now(); + // YUV buffers reused to avoid per-frame allocations + let mut y_buf: Vec = Vec::new(); + let mut u_buf: Vec = Vec::new(); + let mut v_buf: Vec = Vec::new(); + while let Some(frame) = rt_clone.block_on(sink.next()) { + let w = frame.buffer.width(); + let h = frame.buffer.height(); + + if !logged_first { + debug!( + "First frame: {}x{}, type {:?}", + w, h, frame.buffer.buffer_type() + ); + logged_first = true; + } + + // Convert to I420 on CPU, but keep planes separate for GPU sampling + let i420 = frame.buffer.to_i420(); + let (sy, su, sv) = i420.strides(); + let (dy, du, dv) = i420.data(); + + let ch = (h + 1) / 2; + + // Ensure capacity and copy full plane slices + let y_size = (sy * h) as usize; + let u_size = (su * ch) as usize; + let v_size = (sv * ch) as usize; + if y_buf.len() != y_size { y_buf.resize(y_size, 0); } + if u_buf.len() != u_size { u_buf.resize(u_size, 0); } + if v_buf.len() != v_size { v_buf.resize(v_size, 0); } + y_buf.copy_from_slice(dy); + u_buf.copy_from_slice(du); + v_buf.copy_from_slice(dv); + + // Swap buffers into shared state + let mut s = shared2.lock(); + s.width = w as u32; + s.height = h as u32; + s.stride_y = sy as u32; + s.stride_u = su as u32; + s.stride_v = sv as u32; + std::mem::swap(&mut s.y, &mut y_buf); + std::mem::swap(&mut s.u, &mut u_buf); + std::mem::swap(&mut s.v, &mut v_buf); + s.dirty = true; + + frames += 1; + let elapsed = last_log.elapsed(); + if elapsed >= Duration::from_secs(2) { + let fps = frames as f64 / elapsed.as_secs_f64(); + info!("Receiving video: {}x{}, ~{:.1} fps", w, h, fps); + frames = 0; + last_log = Instant::now(); + } + // Periodically infer active simulcast quality from inbound stats + if last_stats.elapsed() >= Duration::from_secs(1) { + if let Ok(stats) = rt_clone.block_on(video_track.get_stats()) { + let mut inbound: Option = None; + for s in stats.iter() { + if let livekit::webrtc::stats::RtcStats::InboundRtp(i) = s { + if i.stream.kind == "video" { + inbound = Some(i.clone()); + } + } + } + if let Some(i) = inbound { + if let Some((fw, fh)) = simulcast_state_full_dims(&simulcast2) { + let q = infer_quality_from_dims(fw, fh, i.inbound.frame_width as u32, i.inbound.frame_height as u32); + let mut sc = simulcast2.lock(); + sc.active_quality = Some(q); + } + } + } + last_stats = Instant::now(); + } + } + info!("Video stream ended for {}", my_sid); + // Clear active sid if still ours + let mut active = active_sid2.lock(); + if active.as_ref() == Some(&my_sid) { + *active = None; + } + }); + } + } + RoomEvent::TrackUnsubscribed { publication, .. } => { + let sid = publication.sid().clone(); + let mut active = active_sid.lock(); + if active.as_ref() == Some(&sid) { + info!("Video track unsubscribed ({}), clearing active sink", sid); + *active = None; + } + // Clear simulcast state + let mut sc = simulcast.lock(); + *sc = SimulcastState::default(); + } + RoomEvent::TrackUnpublished { publication, .. } => { + let sid = publication.sid().clone(); + let mut active = active_sid.lock(); + if active.as_ref() == Some(&sid) { + info!("Video track unpublished ({}), clearing active sink", sid); + *active = None; + } + // Clear simulcast state + let mut sc = simulcast.lock(); + *sc = SimulcastState::default(); + } + _ => {} + } + } + }); + + // Start UI + let app = VideoApp { shared, simulcast }; + let native_options = eframe::NativeOptions::default(); + eframe::run_native("LiveKit Video Subscriber", native_options, Box::new(|_| Ok::, _>(Box::new(app))))?; + + Ok(()) +} + + +// ===== WGPU I420 renderer ===== + +struct YuvPaintCallback { + shared: Arc>, +} + +struct YuvGpuState { + pipeline: wgpu::RenderPipeline, + sampler: wgpu::Sampler, + bind_layout: wgpu::BindGroupLayout, + y_tex: wgpu::Texture, + u_tex: wgpu::Texture, + v_tex: wgpu::Texture, + y_view: wgpu::TextureView, + u_view: wgpu::TextureView, + v_view: wgpu::TextureView, + bind_group: wgpu::BindGroup, + params_buf: wgpu::Buffer, + y_pad_w: u32, + uv_pad_w: u32, + dims: (u32, u32), +} + +impl YuvGpuState { + fn create_textures(device: &wgpu::Device, _width: u32, height: u32, y_pad_w: u32, uv_pad_w: u32) -> (wgpu::Texture, wgpu::Texture, wgpu::Texture, wgpu::TextureView, wgpu::TextureView, wgpu::TextureView) { + let y_size = wgpu::Extent3d { width: y_pad_w, height, depth_or_array_layers: 1 }; + let uv_size = wgpu::Extent3d { width: uv_pad_w, height: (height + 1) / 2, depth_or_array_layers: 1 }; + let usage = wgpu::TextureUsages::COPY_DST | wgpu::TextureUsages::TEXTURE_BINDING; + let desc = |size: wgpu::Extent3d| wgpu::TextureDescriptor { + label: Some("yuv_plane"), + size, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::R8Unorm, + usage, + view_formats: &[], + }; + let y_tex = device.create_texture(&desc(y_size)); + let u_tex = device.create_texture(&desc(uv_size)); + let v_tex = device.create_texture(&desc(uv_size)); + let y_view = y_tex.create_view(&wgpu::TextureViewDescriptor::default()); + let u_view = u_tex.create_view(&wgpu::TextureViewDescriptor::default()); + let v_view = v_tex.create_view(&wgpu::TextureViewDescriptor::default()); + (y_tex, u_tex, v_tex, y_view, u_view, v_view) + } +} + +fn align_up(value: u32, alignment: u32) -> u32 { + ((value + alignment - 1) / alignment) * alignment +} + +#[repr(C)] +#[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)] +struct ParamsUniform { + src_w: u32, + src_h: u32, + y_tex_w: u32, + uv_tex_w: u32, +} + +impl CallbackTrait for YuvPaintCallback { + fn prepare(&self, device: &wgpu::Device, queue: &wgpu::Queue, _screen_desc: &egui_wgpu_backend::ScreenDescriptor, _encoder: &mut wgpu::CommandEncoder, resources: &mut egui_wgpu_backend::CallbackResources) -> Vec { + // Initialize or update GPU state lazily based on current frame + let mut shared = self.shared.lock(); + + // Nothing to draw yet + if shared.width == 0 || shared.height == 0 { + return Vec::new(); + } + + // Fetch or create our GPU state + if resources.get::().is_none() { + // Build pipeline and initial small textures; will be recreated on first upload + let shader_src = include_str!("yuv_shader.wgsl"); + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("yuv_shader"), + source: wgpu::ShaderSource::Wgsl(shader_src.into()), + }); + + let bind_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("yuv_bind_layout"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { binding: 2, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Texture { sample_type: wgpu::TextureSampleType::Float { filterable: true }, view_dimension: wgpu::TextureViewDimension::D2, multisampled: false }, count: None }, + wgpu::BindGroupLayoutEntry { binding: 3, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Texture { sample_type: wgpu::TextureSampleType::Float { filterable: true }, view_dimension: wgpu::TextureViewDimension::D2, multisampled: false }, count: None }, + wgpu::BindGroupLayoutEntry { + binding: 4, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: Some(std::num::NonZeroU64::new(std::mem::size_of::() as u64).unwrap()), + }, + count: None, + }, + ], + }); + + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("yuv_pipeline_layout"), + bind_group_layouts: &[&bind_layout], + push_constant_ranges: &[], + }); + + let render_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("yuv_pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { module: &shader, entry_point: Some("vs_main"), buffers: &[], compilation_options: wgpu::PipelineCompilationOptions::default() }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Bgra8Unorm, + blend: Some(wgpu::BlendState::ALPHA_BLENDING), + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }), + primitive: wgpu::PrimitiveState { topology: wgpu::PrimitiveTopology::TriangleList, strip_index_format: None, front_face: wgpu::FrontFace::Ccw, cull_mode: None, unclipped_depth: false, polygon_mode: wgpu::PolygonMode::Fill, conservative: false }, + depth_stencil: None, + multisample: wgpu::MultisampleState { count: 1, mask: !0, alpha_to_coverage_enabled: false }, + multiview: None, + cache: None, + }); + + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("yuv_sampler"), + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + address_mode_w: wgpu::AddressMode::ClampToEdge, + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + mipmap_filter: wgpu::FilterMode::Nearest, + ..Default::default() + }); + + let params_buf = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("yuv_params"), + contents: bytemuck::bytes_of(&ParamsUniform { src_w: 1, src_h: 1, y_tex_w: 1, uv_tex_w: 1 }), + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + }); + + // Initial tiny textures + let (y_tex, u_tex, v_tex, y_view, u_view, v_view) = YuvGpuState::create_textures(device, 1, 1, 256, 256); + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("yuv_bind_group"), + layout: &bind_layout, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::Sampler(&sampler) }, + wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::TextureView(&y_view) }, + wgpu::BindGroupEntry { binding: 2, resource: wgpu::BindingResource::TextureView(&u_view) }, + wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(&v_view) }, + wgpu::BindGroupEntry { binding: 4, resource: params_buf.as_entire_binding() }, + ], + }); + + let new_state = YuvGpuState { + pipeline: render_pipeline, + sampler, + bind_layout, + y_tex, + u_tex, + v_tex, + y_view, + u_view, + v_view, + bind_group, + params_buf, + y_pad_w: 256, + uv_pad_w: 256, + dims: (0, 0), + }; + resources.insert(new_state); + } + let state = resources.get_mut::().unwrap(); + + // Upload planes when marked dirty + // Recreate textures/bind group on size change + if state.dims != (shared.width, shared.height) { + let y_pad_w = align_up(shared.width, 256); + let uv_w = (shared.width + 1) / 2; + let uv_pad_w = align_up(uv_w, 256); + let (y_tex, u_tex, v_tex, y_view, u_view, v_view) = YuvGpuState::create_textures(device, shared.width, shared.height, y_pad_w, uv_pad_w); + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("yuv_bind_group"), + layout: &state.bind_layout, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::Sampler(&state.sampler) }, + wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::TextureView(&y_view) }, + wgpu::BindGroupEntry { binding: 2, resource: wgpu::BindingResource::TextureView(&u_view) }, + wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(&v_view) }, + wgpu::BindGroupEntry { binding: 4, resource: state.params_buf.as_entire_binding() }, + ], + }); + state.y_tex = y_tex; + state.u_tex = u_tex; + state.v_tex = v_tex; + state.y_view = y_view; + state.u_view = u_view; + state.v_view = v_view; + state.bind_group = bind_group; + state.y_pad_w = y_pad_w; + state.uv_pad_w = uv_pad_w; + state.dims = (shared.width, shared.height); + } + + if shared.dirty { + let y_bytes_per_row = align_up(shared.width, 256); + let uv_w = (shared.width + 1) / 2; + let uv_h = (shared.height + 1) / 2; + let uv_bytes_per_row = align_up(uv_w, 256); + + // Pack and upload Y + if shared.stride_y >= shared.width { + let mut packed = vec![0u8; (y_bytes_per_row * shared.height) as usize]; + for row in 0..shared.height { + let src = &shared.y[(row * shared.stride_y) as usize..][..shared.width as usize]; + let dst_off = (row * y_bytes_per_row) as usize; + packed[dst_off..dst_off + shared.width as usize].copy_from_slice(src); + } + queue.write_texture( + wgpu::ImageCopyTexture { + texture: &state.y_tex, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + &packed, + wgpu::ImageDataLayout { + offset: 0, + bytes_per_row: Some(y_bytes_per_row), + rows_per_image: Some(shared.height), + }, + wgpu::Extent3d { width: state.y_pad_w, height: shared.height, depth_or_array_layers: 1 }, + ); + } + + // Pack and upload U,V + if shared.stride_u >= uv_w && shared.stride_v >= uv_w { + let mut packed_u = vec![0u8; (uv_bytes_per_row * uv_h) as usize]; + let mut packed_v = vec![0u8; (uv_bytes_per_row * uv_h) as usize]; + for row in 0..uv_h { + let src_u = &shared.u[(row * shared.stride_u) as usize..][..uv_w as usize]; + let src_v = &shared.v[(row * shared.stride_v) as usize..][..uv_w as usize]; + let dst_off = (row * uv_bytes_per_row) as usize; + packed_u[dst_off..dst_off + uv_w as usize].copy_from_slice(src_u); + packed_v[dst_off..dst_off + uv_w as usize].copy_from_slice(src_v); + } + queue.write_texture( + wgpu::ImageCopyTexture { texture: &state.u_tex, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + &packed_u, + wgpu::ImageDataLayout { offset: 0, bytes_per_row: Some(uv_bytes_per_row), rows_per_image: Some(uv_h) }, + wgpu::Extent3d { width: state.uv_pad_w, height: uv_h, depth_or_array_layers: 1 }, + ); + queue.write_texture( + wgpu::ImageCopyTexture { texture: &state.v_tex, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + &packed_v, + wgpu::ImageDataLayout { offset: 0, bytes_per_row: Some(uv_bytes_per_row), rows_per_image: Some(uv_h) }, + wgpu::Extent3d { width: state.uv_pad_w, height: uv_h, depth_or_array_layers: 1 }, + ); + } + + // Update params uniform + let params = ParamsUniform { src_w: shared.width, src_h: shared.height, y_tex_w: state.y_pad_w, uv_tex_w: state.uv_pad_w }; + queue.write_buffer(&state.params_buf, 0, bytemuck::bytes_of(¶ms)); + + shared.dirty = false; + } + + Vec::new() + } + + fn paint(&self, _info: egui::PaintCallbackInfo, render_pass: &mut wgpu::RenderPass<'static>, resources: &egui_wgpu_backend::CallbackResources) { + // Acquire device/queue via screen_descriptor? Not available; use resources to fetch our state + let shared = self.shared.lock(); + if shared.width == 0 || shared.height == 0 { + return; + } + + // Build pipeline and textures on first paint or on resize + let Some(state) = resources.get::() else { + // prepare may not have created the state yet (race with first frame); skip this paint + return; + }; + + if state.dims != (shared.width, shared.height) { + // We cannot rebuild here (no device access); skip drawing until next frame where prepare will rebuild + return; + } + + render_pass.set_pipeline(&state.pipeline); + render_pass.set_bind_group(0, &state.bind_group, &[]); + // Fullscreen triangle without vertex buffer + render_pass.draw(0..3, 0..1); + } +} + +// Build or rebuild GPU state. This helper is intended to be called from prepare, but we lack device there in current API constraints. +// Note: eframe/egui-wgpu provides device in paint via RenderPass context; however, to keep this example concise, we set up the state once externally. + diff --git a/examples/local_video/src/yuv_shader.wgsl b/examples/local_video/src/yuv_shader.wgsl new file mode 100644 index 000000000..4a66065fc --- /dev/null +++ b/examples/local_video/src/yuv_shader.wgsl @@ -0,0 +1,63 @@ +struct VSOut { + @builtin(position) pos: vec4, + @location(0) uv: vec2, +}; + +@vertex +fn vs_main(@builtin(vertex_index) vid: u32) -> VSOut { + var pos = array, 3>( + vec2(-1.0, -3.0), + vec2(-1.0, 1.0), + vec2( 3.0, 1.0) + ); + let p = pos[vid]; + var out: VSOut; + out.pos = vec4(p, 0.0, 1.0); + out.uv = 0.5 * (p + vec2(1.0, 1.0)); + return out; +} + +@group(0) @binding(0) var samp: sampler; +@group(0) @binding(1) var y_tex: texture_2d; +@group(0) @binding(2) var u_tex: texture_2d; +@group(0) @binding(3) var v_tex: texture_2d; + +struct Params { + src_w: u32, + src_h: u32, + y_tex_w: u32, + uv_tex_w: u32, +}; +@group(0) @binding(4) var params: Params; + +fn yuv_to_rgb(y: f32, u: f32, v: f32) -> vec3 { + let c = y - (16.0/255.0); + let d = u - 0.5; + let e = v - 0.5; + let r = 1.164 * c + 1.596 * e; + let g = 1.164 * c - 0.392 * d - 0.813 * e; + let b = 1.164 * c + 2.017 * d; + return clamp(vec3(r, g, b), vec3(0.0), vec3(1.0)); +} + +@fragment +fn fs_main(in_: VSOut) -> @location(0) vec4 { + let src_w = f32(params.src_w); + let src_h = f32(params.src_h); + let y_tex_w = f32(params.y_tex_w); + let uv_tex_w = f32(params.uv_tex_w); + + // Flip vertically and scale X to avoid sampling padded columns + let flipped = vec2(in_.uv.x, 1.0 - in_.uv.y); + let uv_y = vec2(flipped.x * (src_w / y_tex_w), flipped.y); + let uv_uv = vec2(flipped.x * ((src_w * 0.5) / uv_tex_w), flipped.y); + + let y = textureSample(y_tex, samp, uv_y).r; + let u = textureSample(u_tex, samp, uv_uv).r; + let v = textureSample(v_tex, samp, uv_uv).r; + + let rgb = yuv_to_rgb(y, u, v); + return vec4(rgb, 1.0); +} + + diff --git a/yuv-sys/Cargo.toml b/yuv-sys/Cargo.toml index 47fdaab61..f75e103d3 100644 --- a/yuv-sys/Cargo.toml +++ b/yuv-sys/Cargo.toml @@ -11,3 +11,4 @@ bindgen = "0.72.1" cc = "1.0" regex = "1" rayon = "1.8" +pkg-config = "0.3" diff --git a/yuv-sys/build.rs b/yuv-sys/build.rs index 4c5c503cb..3e9a40bd3 100644 --- a/yuv-sys/build.rs +++ b/yuv-sys/build.rs @@ -134,11 +134,29 @@ fn main() { rename_symbols(&fnc_list, &include_files, &source_files); } - cc::Build::new() + // Try to detect system libjpeg (or libjpeg-turbo) via pkg-config to enable MJPEG fast path + let jpeg_pkg = + pkg_config::Config::new() + .probe("libjpeg") + .or_else(|_| pkg_config::Config::new().probe("libjpeg-turbo")) + .or_else(|_| pkg_config::Config::new().probe("jpeg")) + .ok(); + + let mut build = cc::Build::new(); + build .warnings(false) .include(libyuv_dir.join("include")) - .files(source_files.iter().map(|f| f.path())) - .compile("yuv"); + .files(source_files.iter().map(|f| f.path())); + + if let Some(pkg) = &jpeg_pkg { + // Enable JPEG in libyuv and add include paths from pkg-config + build.define("HAVE_JPEG", None); + for p in &pkg.include_paths { + build.include(p); + } + } + + build.compile("yuv"); let mut bindgen = bindgen::Builder::default() .header(include_dir.join("libyuv.h").to_string_lossy()) From e0a43386b959c7c501e5ea8e7248f8b83f84213f Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 9 Jan 2026 16:16:51 -0800 Subject: [PATCH 02/15] improve ctrl-c handling --- Cargo.lock | 68 ++++++++++++++------------ examples/local_video/src/publisher.rs | 19 +++++++ examples/local_video/src/subscriber.rs | 49 +++++++++++++++++-- 3 files changed, 102 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c37392bb5..52f6386f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,7 +49,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ - "cfg-if 1.0.3", + "cfg-if 1.0.4", "cipher", "cpufeatures", ] @@ -72,7 +72,7 @@ version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ - "cfg-if 1.0.3", + "cfg-if 1.0.4", "getrandom 0.3.4", "once_cell", "serde", @@ -97,7 +97,7 @@ checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43" dependencies = [ "alsa-sys", "bitflags 2.10.0", - "cfg-if", + "cfg-if 1.0.4", "libc", ] @@ -335,7 +335,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" dependencies = [ "autocfg", - "cfg-if 1.0.3", + "cfg-if 1.0.4", "concurrent-queue", "futures-io", "futures-lite 2.6.1", @@ -491,7 +491,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" dependencies = [ "addr2line", - "cfg-if 1.0.3", + "cfg-if 1.0.4", "libc", "miniz_oxide", "object", @@ -558,7 +558,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.106", + "syn 2.0.110", "which", ] @@ -834,6 +834,12 @@ dependencies = [ "target-lexicon", ] +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.4" @@ -950,7 +956,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81411967c50ee9a1fc11365f8c585f863a22a9697c89239c452292c40ba79b0d" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "block", "core-foundation 0.10.1", "core-graphics-types 0.2.0", @@ -1156,7 +1162,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "core-foundation 0.10.1", "libc", ] @@ -1244,7 +1250,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ - "cfg-if 1.0.3", + "cfg-if 1.0.4", ] [[package]] @@ -1436,7 +1442,7 @@ version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ - "cfg-if 1.0.3", + "cfg-if 1.0.4", "hashbrown 0.14.5", "lock_api", "once_cell", @@ -1660,7 +1666,7 @@ version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ - "cfg-if 1.0.3", + "cfg-if 1.0.4", ] [[package]] @@ -2138,7 +2144,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ - "cfg-if 1.0.3", + "cfg-if 1.0.4", "js-sys", "libc", "wasi", @@ -2151,7 +2157,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ - "cfg-if 1.0.3", + "cfg-if 1.0.4", "js-sys", "libc", "r-efi", @@ -2369,7 +2375,7 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ - "cfg-if 1.0.3", + "cfg-if 1.0.4", "crunchy", "num-traits", "zerocopy", @@ -2858,7 +2864,7 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ - "cfg-if 1.0.3", + "cfg-if 1.0.4", ] [[package]] @@ -3005,7 +3011,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" dependencies = [ "cesu8", - "cfg-if 1.0.3", + "cfg-if 1.0.4", "combine", "jni-sys", "log", @@ -3118,7 +3124,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ - "cfg-if 1.0.3", + "cfg-if 1.0.4", "windows-link 0.2.1", ] @@ -3666,7 +3672,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c35ed9613f002f8095aafc97ad839e0bb6cebf79111c68265d8df212a5a294" dependencies = [ "flume", - "image 0.25.8", + "image 0.25.9", "nokhwa-bindings-linux", "nokhwa-bindings-macos", "nokhwa-bindings-windows", @@ -3721,7 +3727,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "903f3e0f406f7e9aad4fa0566c1d97cc7f88aab57847e1f919d1a34812dedee3" dependencies = [ "bytes", - "image 0.25.8", + "image 0.25.9", "thiserror 2.0.17", ] @@ -4133,7 +4139,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" dependencies = [ "bitflags 2.10.0", - "cfg-if", + "cfg-if 1.0.4", "foreign-types 0.3.2", "libc", "once_cell", @@ -4230,7 +4236,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "backtrace", - "cfg-if 1.0.3", + "cfg-if 1.0.4", "libc", "petgraph 0.6.5", "redox_syscall 0.5.18", @@ -4430,7 +4436,7 @@ checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" dependencies = [ "autocfg", "bitflags 1.3.2", - "cfg-if 1.0.3", + "cfg-if 1.0.4", "concurrent-queue", "libc", "log", @@ -4444,7 +4450,7 @@ version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" dependencies = [ - "cfg-if 1.0.3", + "cfg-if 1.0.4", "concurrent-queue", "hermit-abi", "pin-project-lite", @@ -4972,7 +4978,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", - "cfg-if 1.0.3", + "cfg-if 1.0.4", "getrandom 0.2.16", "libc", "untrusted", @@ -5356,7 +5362,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ - "cfg-if 1.0.3", + "cfg-if 1.0.4", "cpufeatures", "digest", ] @@ -5367,7 +5373,7 @@ version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ - "cfg-if 1.0.3", + "cfg-if 1.0.4", "cpufeatures", "digest", ] @@ -5767,7 +5773,7 @@ version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ - "cfg-if 1.0.3", + "cfg-if 1.0.4", ] [[package]] @@ -5823,7 +5829,7 @@ dependencies = [ "arrayref", "arrayvec", "bytemuck", - "cfg-if 1.0.3", + "cfg-if 1.0.4", "log", "tiny-skia-path", ] @@ -6421,7 +6427,7 @@ version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60" dependencies = [ - "cfg-if 1.0.3", + "cfg-if 1.0.4", "once_cell", "rustversion", "wasm-bindgen-macro", @@ -6434,7 +6440,7 @@ version = "0.4.55" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "551f88106c6d5e7ccc7cd9a16f312dd3b5d36ea8b4954304657d5dfba115d4a0" dependencies = [ - "cfg-if 1.0.3", + "cfg-if 1.0.4", "js-sys", "once_cell", "wasm-bindgen", @@ -6893,7 +6899,7 @@ dependencies = [ "bitflags 2.10.0", "block", "bytemuck", - "cfg-if 1.0.3", + "cfg-if 1.0.4", "cfg_aliases", "core-graphics-types 0.1.3", "glow", diff --git a/examples/local_video/src/publisher.rs b/examples/local_video/src/publisher.rs index d311129b9..3e48816f3 100644 --- a/examples/local_video/src/publisher.rs +++ b/examples/local_video/src/publisher.rs @@ -12,6 +12,10 @@ use nokhwa::pixel_format::RgbFormat; use nokhwa::utils::{ApiBackend, CameraFormat, CameraIndex, FrameFormat, RequestedFormat, RequestedFormatType, Resolution}; use nokhwa::Camera; use std::env; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; use std::time::{Duration, Instant}; #[derive(Parser, Debug)] @@ -80,6 +84,15 @@ async fn main() -> Result<()> { env_logger::init(); let args = Args::parse(); + let ctrl_c_received = Arc::new(AtomicBool::new(false)); + tokio::spawn({ + let ctrl_c_received = ctrl_c_received.clone(); + async move { + tokio::signal::ctrl_c().await.unwrap(); + ctrl_c_received.store(true, Ordering::Release); + } + }); + if args.list_cameras { return list_cameras(); } @@ -237,6 +250,10 @@ async fn main() -> Result<()> { let mut sum_iter_ms = 0.0; let mut logged_mjpeg_fallback = false; loop { + if ctrl_c_received.load(Ordering::Acquire) { + info!("Ctrl-C received, exiting..."); + break; + } // Wait until the scheduled next frame time let wait_start = Instant::now(); ticker.tick().await; @@ -411,6 +428,8 @@ async fn main() -> Result<()> { last_fps_log = Instant::now(); } } + + Ok(()) } diff --git a/examples/local_video/src/subscriber.rs b/examples/local_video/src/subscriber.rs index db719ee02..4e1110c50 100644 --- a/examples/local_video/src/subscriber.rs +++ b/examples/local_video/src/subscriber.rs @@ -13,10 +13,19 @@ use parking_lot::Mutex; use std::{ collections::HashMap, env, - sync::Arc, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, time::{Duration, Instant}, }; +async fn wait_for_shutdown(flag: Arc) { + while !flag.load(Ordering::Acquire) { + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] struct Args { @@ -107,10 +116,16 @@ fn simulcast_state_full_dims( struct VideoApp { shared: Arc>, simulcast: Arc>, + ctrl_c_received: Arc, } impl eframe::App for VideoApp { fn update(&mut self, ctx: &egui::Context, _frame: &mut eframe::Frame) { + if self.ctrl_c_received.load(Ordering::Acquire) { + ctx.send_viewport_cmd(egui::ViewportCommand::Close); + return; + } + egui::CentralPanel::default().show(ctx, |ui| { let available = ui.available_size(); let rect = egui::Rect::from_min_size(ui.min_rect().min, available); @@ -164,6 +179,15 @@ async fn main() -> Result<()> { env_logger::init(); let args = Args::parse(); + let ctrl_c_received = Arc::new(AtomicBool::new(false)); + tokio::spawn({ + let ctrl_c_received = ctrl_c_received.clone(); + async move { + tokio::signal::ctrl_c().await.unwrap(); + ctrl_c_received.store(true, Ordering::Release); + } + }); + // LiveKit connection details (prefer CLI args, fallback to env vars) let url = args .url @@ -218,6 +242,7 @@ async fn main() -> Result<()> { // Shared simulcast UI/control state let simulcast = Arc::new(Mutex::new(SimulcastState::default())); let simulcast_events = simulcast.clone(); + let ctrl_c_events = ctrl_c_received.clone(); tokio::spawn(async move { let active_sid = active_sid.clone(); let simulcast = simulcast_events; @@ -306,6 +331,7 @@ async fn main() -> Result<()> { let active_sid2 = active_sid.clone(); let my_sid = sid.clone(); let rt_clone = rt.clone(); + let ctrl_c_sink = ctrl_c_events.clone(); // Initialize simulcast state for this publication { let mut sc = simulcast.lock(); @@ -327,7 +353,17 @@ async fn main() -> Result<()> { let mut y_buf: Vec = Vec::new(); let mut u_buf: Vec = Vec::new(); let mut v_buf: Vec = Vec::new(); - while let Some(frame) = rt_clone.block_on(sink.next()) { + loop { + if ctrl_c_sink.load(Ordering::Acquire) { + break; + } + let next = rt_clone.block_on(async { + tokio::select! { + _ = wait_for_shutdown(ctrl_c_sink.clone()) => None, + frame = sink.next() => frame, + } + }); + let Some(frame) = next else { break }; let w = frame.buffer.width(); let h = frame.buffer.height(); @@ -436,10 +472,17 @@ async fn main() -> Result<()> { }); // Start UI - let app = VideoApp { shared, simulcast }; + let app = VideoApp { + shared, + simulcast, + ctrl_c_received: ctrl_c_received.clone(), + }; let native_options = eframe::NativeOptions::default(); eframe::run_native("LiveKit Video Subscriber", native_options, Box::new(|_| Ok::, _>(Box::new(app))))?; + // If the window was closed manually, still signal shutdown to background threads. + ctrl_c_received.store(true, Ordering::Release); + Ok(()) } From 78fac50d05b8c6600bd440a9bd076e3bf69f19f6 Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 9 Jan 2026 16:17:44 -0800 Subject: [PATCH 03/15] update .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 97ff64931..a279f9e8e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ target soxr-sys/test-input.wav soxr-sys/test-output.wav .DS_Store -.env \ No newline at end of file +.env +.cursor \ No newline at end of file From 81343cd7aa4d47508a7ac503a4596b473ba4f78e Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 9 Jan 2026 16:22:49 -0800 Subject: [PATCH 04/15] lock aspect ratio --- examples/local_video/src/subscriber.rs | 54 ++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/examples/local_video/src/subscriber.rs b/examples/local_video/src/subscriber.rs index 4e1110c50..8521224ca 100644 --- a/examples/local_video/src/subscriber.rs +++ b/examples/local_video/src/subscriber.rs @@ -117,6 +117,8 @@ struct VideoApp { shared: Arc>, simulcast: Arc>, ctrl_c_received: Arc, + locked_aspect: Option, + last_inner_size: Option, } impl eframe::App for VideoApp { @@ -126,6 +128,52 @@ impl eframe::App for VideoApp { return; } + // Lock aspect ratio based on the first received video frame, then keep the window snapped + // to sizes matching that ratio. + if self.locked_aspect.is_none() { + let s = self.shared.lock(); + if s.width > 0 && s.height > 0 { + self.locked_aspect = Some(s.width as f32 / s.height as f32); + } + } + + if let Some(aspect) = self.locked_aspect { + let (inner_rect, maximized, fullscreen) = ctx.input(|i| { + let vp = i.viewport(); + (vp.inner_rect, vp.maximized, vp.fullscreen) + }); + + // Don't fight the OS when maximized/fullscreen. + if maximized != Some(true) && fullscreen != Some(true) { + if let Some(rect) = inner_rect { + let cur = rect.size(); + let prev = self.last_inner_size.unwrap_or(cur); + let dw = (cur.x - prev.x).abs(); + let dh = (cur.y - prev.y).abs(); + + // Determine which axis the user most likely dragged, and adjust the other. + let mut target = cur; + const MIN_W: f32 = 320.0; + const MIN_H: f32 = 240.0; + if dw >= dh { + target.x = target.x.max(MIN_W); + target.y = (target.x / aspect).max(MIN_H); + } else { + target.y = target.y.max(MIN_H); + target.x = (target.y * aspect).max(MIN_W); + } + + // Avoid resize feedback loops by only issuing a command when we are meaningfully off. + if (cur.x - target.x).abs() > 0.5 || (cur.y - target.y).abs() > 0.5 { + ctx.send_viewport_cmd(egui::ViewportCommand::InnerSize(target)); + self.last_inner_size = Some(target); + } else { + self.last_inner_size = Some(cur); + } + } + } + } + egui::CentralPanel::default().show(ctx, |ui| { let available = ui.available_size(); let rect = egui::Rect::from_min_size(ui.min_rect().min, available); @@ -476,6 +524,8 @@ async fn main() -> Result<()> { shared, simulcast, ctrl_c_received: ctrl_c_received.clone(), + locked_aspect: None, + last_inner_size: None, }; let native_options = eframe::NativeOptions::default(); eframe::run_native("LiveKit Video Subscriber", native_options, Box::new(|_| Ok::, _>(Box::new(app))))?; @@ -799,7 +849,3 @@ impl CallbackTrait for YuvPaintCallback { render_pass.draw(0..3, 0..1); } } - -// Build or rebuild GPU state. This helper is intended to be called from prepare, but we lack device there in current API constraints. -// Note: eframe/egui-wgpu provides device in paint via RenderPass context; however, to keep this example concise, we set up the state once externally. - From 11c4d5c965b13e2f7f73ef137a2bb451247e185d Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 9 Jan 2026 16:38:17 -0800 Subject: [PATCH 05/15] fix window resize & display some stats on video --- examples/local_video/src/publisher.rs | 70 ++-- examples/local_video/src/subscriber.rs | 435 +++++++++++++++++++------ 2 files changed, 365 insertions(+), 140 deletions(-) diff --git a/examples/local_video/src/publisher.rs b/examples/local_video/src/publisher.rs index 3e48816f3..cd9ec16e3 100644 --- a/examples/local_video/src/publisher.rs +++ b/examples/local_video/src/publisher.rs @@ -7,9 +7,11 @@ use livekit::webrtc::video_source::native::NativeVideoSource; use livekit::webrtc::video_source::{RtcVideoSource, VideoResolution}; use livekit_api::access_token; use log::{debug, info}; -use yuv_sys as yuv_sys; use nokhwa::pixel_format::RgbFormat; -use nokhwa::utils::{ApiBackend, CameraFormat, CameraIndex, FrameFormat, RequestedFormat, RequestedFormatType, Resolution}; +use nokhwa::utils::{ + ApiBackend, CameraFormat, CameraIndex, FrameFormat, RequestedFormat, RequestedFormatType, + Resolution, +}; use nokhwa::Camera; use std::env; use std::sync::{ @@ -17,6 +19,7 @@ use std::sync::{ Arc, }; use std::time::{Duration, Instant}; +use yuv_sys; #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] @@ -46,11 +49,11 @@ struct Args { max_bitrate: Option, /// LiveKit participant identity - #[arg(long, default_value = "rust-camera-pub")] + #[arg(long, default_value = "rust-camera-pub")] identity: String, /// LiveKit room name - #[arg(long, default_value = "video-room")] + #[arg(long, default_value = "video-room")] room_name: String, /// LiveKit server URL @@ -98,9 +101,10 @@ async fn main() -> Result<()> { } // LiveKit connection details - let url = args.url.or_else(|| env::var("LIVEKIT_URL").ok()).expect( - "LIVEKIT_URL must be provided via --url or env", - ); + let url = args + .url + .or_else(|| env::var("LIVEKIT_URL").ok()) + .expect("LIVEKIT_URL must be provided via --url or env"); let api_key = args .api_key .or_else(|| env::var("LIVEKIT_API_KEY").ok()) @@ -142,23 +146,24 @@ async fn main() -> Result<()> { // Setup camera let index = CameraIndex::Index(args.camera_index as u32); - let requested = RequestedFormat::new::(RequestedFormatType::AbsoluteHighestFrameRate); + let requested = + RequestedFormat::new::(RequestedFormatType::AbsoluteHighestFrameRate); let mut camera = Camera::new(index, requested)?; // Try raw YUYV first (cheaper than MJPEG), fall back to MJPEG - let wanted = CameraFormat::new( - Resolution::new(args.width, args.height), - FrameFormat::YUYV, - args.fps, - ); + let wanted = + CameraFormat::new(Resolution::new(args.width, args.height), FrameFormat::YUYV, args.fps); let mut using_fmt = "YUYV"; - if let Err(_) = camera.set_camera_requset(RequestedFormat::new::(RequestedFormatType::Exact(wanted))) { + if let Err(_) = camera + .set_camera_requset(RequestedFormat::new::(RequestedFormatType::Exact(wanted))) + { let alt = CameraFormat::new( Resolution::new(args.width, args.height), FrameFormat::MJPEG, args.fps, ); using_fmt = "MJPEG"; - let _ = camera.set_camera_requset(RequestedFormat::new::(RequestedFormatType::Exact(alt))); + let _ = camera + .set_camera_requset(RequestedFormat::new::(RequestedFormatType::Exact(alt))); } camera.open_stream()?; let fmt = camera.camera_format(); @@ -172,10 +177,8 @@ async fn main() -> Result<()> { // Create LiveKit video source and track let rtc_source = NativeVideoSource::new(VideoResolution { width, height }); - let track = LocalVideoTrack::create_video_track( - "camera", - RtcVideoSource::Native(rtc_source.clone()), - ); + let track = + LocalVideoTrack::create_video_track("camera", RtcVideoSource::Native(rtc_source.clone())); // Choose requested codec and attempt to publish; if H.265 fails, retry with H.264 let requested_codec = if args.h265 { VideoCodec::H265 } else { VideoCodec::H264 }; @@ -189,10 +192,8 @@ async fn main() -> Result<()> { ..Default::default() }; if let Some(bitrate) = args.max_bitrate { - opts.video_encoding = Some(VideoEncoding { - max_bitrate: bitrate, - max_framerate: args.fps as f64, - }); + opts.video_encoding = + Some(VideoEncoding { max_bitrate: bitrate, max_framerate: args.fps as f64 }); } opts }; @@ -204,12 +205,8 @@ async fn main() -> Result<()> { if let Err(e) = publish_result { if matches!(requested_codec, VideoCodec::H265) { - log::warn!( - "H.265 publish failed ({}). Falling back to H.264...", - e - ); - room - .local_participant() + log::warn!("H.265 publish failed ({}). Falling back to H.264...", e); + room.local_participant() .publish_track(LocalTrack::Video(track.clone()), publish_opts(VideoCodec::H264)) .await?; info!("Published camera track with H.264 fallback"); @@ -221,7 +218,11 @@ async fn main() -> Result<()> { } // Reusable I420 buffer and frame - let mut frame = VideoFrame { rotation: VideoRotation::VideoRotation0, timestamp_us: 0, buffer: I420Buffer::new(width, height) }; + let mut frame = VideoFrame { + rotation: VideoRotation::VideoRotation0, + timestamp_us: 0, + buffer: I420Buffer::new(width, height), + }; let is_yuyv = fmt.format() == FrameFormat::YUYV; info!( "Selected conversion path: {}", @@ -326,7 +327,12 @@ async fn main() -> Result<()> { width as i32, height as i32, ); - if ret == 0 { used_fast_mjpeg = true; Instant::now() } else { t1 } + if ret == 0 { + used_fast_mjpeg = true; + Instant::now() + } else { + t1 + } }; if used_fast_mjpeg { t2_try @@ -431,5 +437,3 @@ async fn main() -> Result<()> { Ok(()) } - - diff --git a/examples/local_video/src/subscriber.rs b/examples/local_video/src/subscriber.rs index 8521224ca..a54cc4a29 100644 --- a/examples/local_video/src/subscriber.rs +++ b/examples/local_video/src/subscriber.rs @@ -1,9 +1,9 @@ use anyhow::Result; use clap::Parser; use eframe::egui; +use eframe::wgpu::{self, util::DeviceExt}; use egui_wgpu as egui_wgpu_backend; use egui_wgpu_backend::CallbackTrait; -use eframe::wgpu::{self, util::DeviceExt}; use futures::StreamExt; use livekit::prelude::*; use livekit::webrtc::video_stream::native::NativeVideoStream; @@ -30,11 +30,11 @@ async fn wait_for_shutdown(flag: Arc) { #[command(author, version, about, long_about = None)] struct Args { /// LiveKit participant identity - #[arg(long, default_value = "rust-video-subscriber")] + #[arg(long, default_value = "rust-video-subscriber")] identity: String, /// LiveKit room name - #[arg(long, default_value = "video-room")] + #[arg(long, default_value = "video-room")] room_name: String, /// LiveKit server URL @@ -63,6 +63,8 @@ struct SharedYuv { y: Vec, u: Vec, v: Vec, + codec: String, + fps: f32, dirty: bool, } @@ -87,6 +89,12 @@ impl Default for SimulcastState { } } +fn codec_label(mime: &str) -> String { + let base = mime.split(';').next().unwrap_or(mime).trim(); + let last = base.rsplit('/').next().unwrap_or(base).trim(); + last.to_ascii_uppercase() +} + fn infer_quality_from_dims( full_w: u32, _full_h: u32, @@ -106,9 +114,7 @@ fn infer_quality_from_dims( } } -fn simulcast_state_full_dims( - state: &Arc>, -) -> Option<(u32, u32)> { +fn simulcast_state_full_dims(state: &Arc>) -> Option<(u32, u32)> { let sc = state.lock(); sc.full_dims } @@ -118,7 +124,6 @@ struct VideoApp { simulcast: Arc>, ctrl_c_received: Arc, locked_aspect: Option, - last_inner_size: Option, } impl eframe::App for VideoApp { @@ -128,8 +133,7 @@ impl eframe::App for VideoApp { return; } - // Lock aspect ratio based on the first received video frame, then keep the window snapped - // to sizes matching that ratio. + // Lock aspect ratio based on the first received video frame. if self.locked_aspect.is_none() { let s = self.shared.lock(); if s.width > 0 && s.height > 0 { @@ -137,58 +141,60 @@ impl eframe::App for VideoApp { } } - if let Some(aspect) = self.locked_aspect { - let (inner_rect, maximized, fullscreen) = ctx.input(|i| { - let vp = i.viewport(); - (vp.inner_rect, vp.maximized, vp.fullscreen) - }); - - // Don't fight the OS when maximized/fullscreen. - if maximized != Some(true) && fullscreen != Some(true) { - if let Some(rect) = inner_rect { - let cur = rect.size(); - let prev = self.last_inner_size.unwrap_or(cur); - let dw = (cur.x - prev.x).abs(); - let dh = (cur.y - prev.y).abs(); - - // Determine which axis the user most likely dragged, and adjust the other. - let mut target = cur; - const MIN_W: f32 = 320.0; - const MIN_H: f32 = 240.0; - if dw >= dh { - target.x = target.x.max(MIN_W); - target.y = (target.x / aspect).max(MIN_H); - } else { - target.y = target.y.max(MIN_H); - target.x = (target.y * aspect).max(MIN_W); - } - - // Avoid resize feedback loops by only issuing a command when we are meaningfully off. - if (cur.x - target.x).abs() > 0.5 || (cur.y - target.y).abs() > 0.5 { - ctx.send_viewport_cmd(egui::ViewportCommand::InnerSize(target)); - self.last_inner_size = Some(target); - } else { - self.last_inner_size = Some(cur); - } - } - } - } - egui::CentralPanel::default().show(ctx, |ui| { - let available = ui.available_size(); - let rect = egui::Rect::from_min_size(ui.min_rect().min, available); - - // Ensure we keep repainting for smooth playback + // Ensure we keep repainting for smooth playback. ui.ctx().request_repaint(); - // Add a custom wgpu paint callback that renders I420 directly - let cb = egui_wgpu_backend::Callback::new_paint_callback( - rect, - YuvPaintCallback { shared: self.shared.clone() }, + // Render into a centered rect that matches the source aspect ratio. This keeps resize + // smooth (no feedback loop) and avoids stretching/distortion while dragging. + let available = ui.available_size(); + let size = if let Some(aspect) = self.locked_aspect { + let mut w = available.x.max(1.0); + let mut h = (w / aspect).max(1.0); + if h > available.y.max(1.0) { + h = available.y.max(1.0); + w = (h * aspect).max(1.0); + } + egui::vec2(w, h) + } else { + egui::vec2(available.x.max(1.0), available.y.max(1.0)) + }; + + ui.with_layout( + egui::Layout::centered_and_justified(egui::Direction::LeftToRight), + |ui| { + let (rect, _) = ui.allocate_exact_size(size, egui::Sense::hover()); + let cb = egui_wgpu_backend::Callback::new_paint_callback( + rect, + YuvPaintCallback { shared: self.shared.clone() }, + ); + ui.painter().add(cb); + }, ); - ui.painter().add(cb); }); + // Resolution/FPS overlay: top-left + egui::Area::new("video_hud".into()) + .anchor(egui::Align2::LEFT_TOP, egui::vec2(10.0, 10.0)) + .interactable(false) + .show(ctx, |ui| { + let s = self.shared.lock(); + if s.width == 0 || s.height == 0 || s.fps <= 0.0 || s.codec.is_empty() { + return; + } + let text = format!("{} {}x{} {:.1}fps", s.codec, s.width, s.height, s.fps); + egui::Frame::NONE + .fill(egui::Color32::from_black_alpha(140)) + .corner_radius(egui::CornerRadius::same(4)) + .inner_margin(egui::Margin::same(6)) + .show(ui, |ui| { + ui.add( + egui::Label::new(egui::RichText::new(text).color(egui::Color32::WHITE)) + .extend(), + ); + }); + }); + // Simulcast layer controls: bottom-left overlay egui::Area::new("simulcast_controls".into()) .anchor(egui::Align2::LEFT_BOTTOM, egui::vec2(10.0, -10.0)) @@ -237,10 +243,9 @@ async fn main() -> Result<()> { }); // LiveKit connection details (prefer CLI args, fallback to env vars) - let url = args - .url - .or_else(|| env::var("LIVEKIT_URL").ok()) - .expect("LiveKit URL must be provided via --url argument or LIVEKIT_URL environment variable"); + let url = args.url.or_else(|| env::var("LIVEKIT_URL").ok()).expect( + "LiveKit URL must be provided via --url argument or LIVEKIT_URL environment variable", + ); let api_key = args .api_key .or_else(|| env::var("LIVEKIT_API_KEY").ok()) @@ -278,6 +283,8 @@ async fn main() -> Result<()> { y: Vec::new(), u: Vec::new(), v: Vec::new(), + codec: String::new(), + fps: 0.0, dirty: false, })); @@ -303,17 +310,25 @@ async fn main() -> Result<()> { // If a participant filter is set, skip others if let Some(ref allow) = allowed_identity { if participant.identity().as_str() != allow { - debug!("Skipping track from '{}' (filter set to '{}')", participant.identity(), allow); + debug!( + "Skipping track from '{}' (filter set to '{}')", + participant.identity(), + allow + ); continue; } } if let livekit::track::RemoteTrack::Video(video_track) = track { let sid = publication.sid().clone(); + let codec = codec_label(&publication.mime_type()); // Only handle if we don't already have an active video track { let mut active = active_sid.lock(); if active.as_ref() == Some(&sid) { - debug!("Track {} already active, ignoring duplicate subscribe", sid); + debug!( + "Track {} already active, ignoring duplicate subscribe", + sid + ); continue; } if active.is_some() { @@ -323,6 +338,12 @@ async fn main() -> Result<()> { *active = Some(sid.clone()); } + // Update HUD codec label early (before first frame arrives) + { + let mut s = shared_clone.lock(); + s.codec = codec; + } + info!( "Subscribed to video track: {} (sid {}) from {} - codec: {}, simulcast: {}, dimension: {}x{}", publication.name(), @@ -337,14 +358,19 @@ async fn main() -> Result<()> { // Try to fetch inbound RTP/codec stats for more details match video_track.get_stats().await { Ok(stats) => { - let mut codec_by_id: HashMap = HashMap::new(); - let mut inbound: Option = None; + let mut codec_by_id: HashMap = + HashMap::new(); + let mut inbound: Option = + None; for s in stats.iter() { match s { livekit::webrtc::stats::RtcStats::Codec(c) => { codec_by_id.insert( c.rtc.id.clone(), - (c.codec.mime_type.clone(), c.codec.sdp_fmtp_line.clone()), + ( + c.codec.mime_type.clone(), + c.codec.sdp_fmtp_line.clone(), + ), ); } livekit::webrtc::stats::RtcStats::InboundRtp(i) => { @@ -357,7 +383,8 @@ async fn main() -> Result<()> { } if let Some(i) = inbound { - if let Some((mime, fmtp)) = codec_by_id.get(&i.stream.codec_id) { + if let Some((mime, fmtp)) = codec_by_id.get(&i.stream.codec_id) + { info!("Inbound codec: {} (fmtp: {})", mime, fmtp); } else { info!("Inbound codec id: {}", i.stream.codec_id); @@ -397,6 +424,9 @@ async fn main() -> Result<()> { let mut last_log = Instant::now(); let mut logged_first = false; let mut last_stats = Instant::now(); + let mut fps_window_frames: u64 = 0; + let mut fps_window_start = Instant::now(); + let mut fps_smoothed: f32 = 0.0; // YUV buffers reused to avoid per-frame allocations let mut y_buf: Vec = Vec::new(); let mut u_buf: Vec = Vec::new(); @@ -418,7 +448,9 @@ async fn main() -> Result<()> { if !logged_first { debug!( "First frame: {}x{}, type {:?}", - w, h, frame.buffer.buffer_type() + w, + h, + frame.buffer.buffer_type() ); logged_first = true; } @@ -434,9 +466,15 @@ async fn main() -> Result<()> { let y_size = (sy * h) as usize; let u_size = (su * ch) as usize; let v_size = (sv * ch) as usize; - if y_buf.len() != y_size { y_buf.resize(y_size, 0); } - if u_buf.len() != u_size { u_buf.resize(u_size, 0); } - if v_buf.len() != v_size { v_buf.resize(v_size, 0); } + if y_buf.len() != y_size { + y_buf.resize(y_size, 0); + } + if u_buf.len() != u_size { + u_buf.resize(u_size, 0); + } + if v_buf.len() != v_size { + v_buf.resize(v_size, 0); + } y_buf.copy_from_slice(dy); u_buf.copy_from_slice(du); v_buf.copy_from_slice(dv); @@ -453,6 +491,23 @@ async fn main() -> Result<()> { std::mem::swap(&mut s.v, &mut v_buf); s.dirty = true; + // Update smoothed FPS (~500ms window) + fps_window_frames += 1; + let win_elapsed = fps_window_start.elapsed(); + if win_elapsed >= Duration::from_millis(500) { + let inst_fps = (fps_window_frames as f32) + / (win_elapsed.as_secs_f32().max(0.001)); + fps_smoothed = if fps_smoothed <= 0.0 { + inst_fps + } else { + // light EMA smoothing to reduce jitter + (fps_smoothed * 0.7) + (inst_fps * 0.3) + }; + s.fps = fps_smoothed; + fps_window_frames = 0; + fps_window_start = Instant::now(); + } + frames += 1; let elapsed = last_log.elapsed(); if elapsed >= Duration::from_secs(2) { @@ -464,17 +519,28 @@ async fn main() -> Result<()> { // Periodically infer active simulcast quality from inbound stats if last_stats.elapsed() >= Duration::from_secs(1) { if let Ok(stats) = rt_clone.block_on(video_track.get_stats()) { - let mut inbound: Option = None; + let mut inbound: Option< + livekit::webrtc::stats::InboundRtpStats, + > = None; for s in stats.iter() { - if let livekit::webrtc::stats::RtcStats::InboundRtp(i) = s { + if let livekit::webrtc::stats::RtcStats::InboundRtp(i) = + s + { if i.stream.kind == "video" { inbound = Some(i.clone()); } } } if let Some(i) = inbound { - if let Some((fw, fh)) = simulcast_state_full_dims(&simulcast2) { - let q = infer_quality_from_dims(fw, fh, i.inbound.frame_width as u32, i.inbound.frame_height as u32); + if let Some((fw, fh)) = + simulcast_state_full_dims(&simulcast2) + { + let q = infer_quality_from_dims( + fw, + fh, + i.inbound.frame_width as u32, + i.inbound.frame_height as u32, + ); let mut sc = simulcast2.lock(); sc.active_quality = Some(q); } @@ -499,6 +565,12 @@ async fn main() -> Result<()> { info!("Video track unsubscribed ({}), clearing active sink", sid); *active = None; } + // Clear HUD codec + { + let mut s = shared_clone.lock(); + s.codec.clear(); + s.fps = 0.0; + } // Clear simulcast state let mut sc = simulcast.lock(); *sc = SimulcastState::default(); @@ -510,6 +582,12 @@ async fn main() -> Result<()> { info!("Video track unpublished ({}), clearing active sink", sid); *active = None; } + // Clear HUD codec + { + let mut s = shared_clone.lock(); + s.codec.clear(); + s.fps = 0.0; + } // Clear simulcast state let mut sc = simulcast.lock(); *sc = SimulcastState::default(); @@ -525,10 +603,13 @@ async fn main() -> Result<()> { simulcast, ctrl_c_received: ctrl_c_received.clone(), locked_aspect: None, - last_inner_size: None, }; let native_options = eframe::NativeOptions::default(); - eframe::run_native("LiveKit Video Subscriber", native_options, Box::new(|_| Ok::, _>(Box::new(app))))?; + eframe::run_native( + "LiveKit Video Subscriber", + native_options, + Box::new(|_| Ok::, _>(Box::new(app))), + )?; // If the window was closed manually, still signal shutdown to background threads. ctrl_c_received.store(true, Ordering::Release); @@ -536,7 +617,6 @@ async fn main() -> Result<()> { Ok(()) } - // ===== WGPU I420 renderer ===== struct YuvPaintCallback { @@ -561,9 +641,23 @@ struct YuvGpuState { } impl YuvGpuState { - fn create_textures(device: &wgpu::Device, _width: u32, height: u32, y_pad_w: u32, uv_pad_w: u32) -> (wgpu::Texture, wgpu::Texture, wgpu::Texture, wgpu::TextureView, wgpu::TextureView, wgpu::TextureView) { + fn create_textures( + device: &wgpu::Device, + _width: u32, + height: u32, + y_pad_w: u32, + uv_pad_w: u32, + ) -> ( + wgpu::Texture, + wgpu::Texture, + wgpu::Texture, + wgpu::TextureView, + wgpu::TextureView, + wgpu::TextureView, + ) { let y_size = wgpu::Extent3d { width: y_pad_w, height, depth_or_array_layers: 1 }; - let uv_size = wgpu::Extent3d { width: uv_pad_w, height: (height + 1) / 2, depth_or_array_layers: 1 }; + let uv_size = + wgpu::Extent3d { width: uv_pad_w, height: (height + 1) / 2, depth_or_array_layers: 1 }; let usage = wgpu::TextureUsages::COPY_DST | wgpu::TextureUsages::TEXTURE_BINDING; let desc = |size: wgpu::Extent3d| wgpu::TextureDescriptor { label: Some("yuv_plane"), @@ -599,7 +693,14 @@ struct ParamsUniform { } impl CallbackTrait for YuvPaintCallback { - fn prepare(&self, device: &wgpu::Device, queue: &wgpu::Queue, _screen_desc: &egui_wgpu_backend::ScreenDescriptor, _encoder: &mut wgpu::CommandEncoder, resources: &mut egui_wgpu_backend::CallbackResources) -> Vec { + fn prepare( + &self, + device: &wgpu::Device, + queue: &wgpu::Queue, + _screen_desc: &egui_wgpu_backend::ScreenDescriptor, + _encoder: &mut wgpu::CommandEncoder, + resources: &mut egui_wgpu_backend::CallbackResources, + ) -> Vec { // Initialize or update GPU state lazily based on current frame let mut shared = self.shared.lock(); @@ -636,15 +737,38 @@ impl CallbackTrait for YuvPaintCallback { }, count: None, }, - wgpu::BindGroupLayoutEntry { binding: 2, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Texture { sample_type: wgpu::TextureSampleType::Float { filterable: true }, view_dimension: wgpu::TextureViewDimension::D2, multisampled: false }, count: None }, - wgpu::BindGroupLayoutEntry { binding: 3, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Texture { sample_type: wgpu::TextureSampleType::Float { filterable: true }, view_dimension: wgpu::TextureViewDimension::D2, multisampled: false }, count: None }, + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 3, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, wgpu::BindGroupLayoutEntry { binding: 4, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Uniform, has_dynamic_offset: false, - min_binding_size: Some(std::num::NonZeroU64::new(std::mem::size_of::() as u64).unwrap()), + min_binding_size: Some( + std::num::NonZeroU64::new( + std::mem::size_of::() as u64 + ) + .unwrap(), + ), }, count: None, }, @@ -660,7 +784,12 @@ impl CallbackTrait for YuvPaintCallback { let render_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { label: Some("yuv_pipeline"), layout: Some(&pipeline_layout), - vertex: wgpu::VertexState { module: &shader, entry_point: Some("vs_main"), buffers: &[], compilation_options: wgpu::PipelineCompilationOptions::default() }, + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }, fragment: Some(wgpu::FragmentState { module: &shader, entry_point: Some("fs_main"), @@ -671,9 +800,21 @@ impl CallbackTrait for YuvPaintCallback { })], compilation_options: wgpu::PipelineCompilationOptions::default(), }), - primitive: wgpu::PrimitiveState { topology: wgpu::PrimitiveTopology::TriangleList, strip_index_format: None, front_face: wgpu::FrontFace::Ccw, cull_mode: None, unclipped_depth: false, polygon_mode: wgpu::PolygonMode::Fill, conservative: false }, + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + strip_index_format: None, + front_face: wgpu::FrontFace::Ccw, + cull_mode: None, + unclipped_depth: false, + polygon_mode: wgpu::PolygonMode::Fill, + conservative: false, + }, depth_stencil: None, - multisample: wgpu::MultisampleState { count: 1, mask: !0, alpha_to_coverage_enabled: false }, + multisample: wgpu::MultisampleState { + count: 1, + mask: !0, + alpha_to_coverage_enabled: false, + }, multiview: None, cache: None, }); @@ -691,20 +832,38 @@ impl CallbackTrait for YuvPaintCallback { let params_buf = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("yuv_params"), - contents: bytemuck::bytes_of(&ParamsUniform { src_w: 1, src_h: 1, y_tex_w: 1, uv_tex_w: 1 }), + contents: bytemuck::bytes_of(&ParamsUniform { + src_w: 1, + src_h: 1, + y_tex_w: 1, + uv_tex_w: 1, + }), usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, }); // Initial tiny textures - let (y_tex, u_tex, v_tex, y_view, u_view, v_view) = YuvGpuState::create_textures(device, 1, 1, 256, 256); + let (y_tex, u_tex, v_tex, y_view, u_view, v_view) = + YuvGpuState::create_textures(device, 1, 1, 256, 256); let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("yuv_bind_group"), layout: &bind_layout, entries: &[ - wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::Sampler(&sampler) }, - wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::TextureView(&y_view) }, - wgpu::BindGroupEntry { binding: 2, resource: wgpu::BindingResource::TextureView(&u_view) }, - wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(&v_view) }, + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::Sampler(&sampler), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(&y_view), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::TextureView(&u_view), + }, + wgpu::BindGroupEntry { + binding: 3, + resource: wgpu::BindingResource::TextureView(&v_view), + }, wgpu::BindGroupEntry { binding: 4, resource: params_buf.as_entire_binding() }, ], }); @@ -735,16 +894,37 @@ impl CallbackTrait for YuvPaintCallback { let y_pad_w = align_up(shared.width, 256); let uv_w = (shared.width + 1) / 2; let uv_pad_w = align_up(uv_w, 256); - let (y_tex, u_tex, v_tex, y_view, u_view, v_view) = YuvGpuState::create_textures(device, shared.width, shared.height, y_pad_w, uv_pad_w); + let (y_tex, u_tex, v_tex, y_view, u_view, v_view) = YuvGpuState::create_textures( + device, + shared.width, + shared.height, + y_pad_w, + uv_pad_w, + ); let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("yuv_bind_group"), layout: &state.bind_layout, entries: &[ - wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::Sampler(&state.sampler) }, - wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::TextureView(&y_view) }, - wgpu::BindGroupEntry { binding: 2, resource: wgpu::BindingResource::TextureView(&u_view) }, - wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(&v_view) }, - wgpu::BindGroupEntry { binding: 4, resource: state.params_buf.as_entire_binding() }, + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::Sampler(&state.sampler), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(&y_view), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::TextureView(&u_view), + }, + wgpu::BindGroupEntry { + binding: 3, + resource: wgpu::BindingResource::TextureView(&v_view), + }, + wgpu::BindGroupEntry { + binding: 4, + resource: state.params_buf.as_entire_binding(), + }, ], }); state.y_tex = y_tex; @@ -769,7 +949,8 @@ impl CallbackTrait for YuvPaintCallback { if shared.stride_y >= shared.width { let mut packed = vec![0u8; (y_bytes_per_row * shared.height) as usize]; for row in 0..shared.height { - let src = &shared.y[(row * shared.stride_y) as usize..][..shared.width as usize]; + let src = + &shared.y[(row * shared.stride_y) as usize..][..shared.width as usize]; let dst_off = (row * y_bytes_per_row) as usize; packed[dst_off..dst_off + shared.width as usize].copy_from_slice(src); } @@ -786,7 +967,11 @@ impl CallbackTrait for YuvPaintCallback { bytes_per_row: Some(y_bytes_per_row), rows_per_image: Some(shared.height), }, - wgpu::Extent3d { width: state.y_pad_w, height: shared.height, depth_or_array_layers: 1 }, + wgpu::Extent3d { + width: state.y_pad_w, + height: shared.height, + depth_or_array_layers: 1, + }, ); } @@ -802,21 +987,52 @@ impl CallbackTrait for YuvPaintCallback { packed_v[dst_off..dst_off + uv_w as usize].copy_from_slice(src_v); } queue.write_texture( - wgpu::ImageCopyTexture { texture: &state.u_tex, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + wgpu::ImageCopyTexture { + texture: &state.u_tex, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, &packed_u, - wgpu::ImageDataLayout { offset: 0, bytes_per_row: Some(uv_bytes_per_row), rows_per_image: Some(uv_h) }, - wgpu::Extent3d { width: state.uv_pad_w, height: uv_h, depth_or_array_layers: 1 }, + wgpu::ImageDataLayout { + offset: 0, + bytes_per_row: Some(uv_bytes_per_row), + rows_per_image: Some(uv_h), + }, + wgpu::Extent3d { + width: state.uv_pad_w, + height: uv_h, + depth_or_array_layers: 1, + }, ); queue.write_texture( - wgpu::ImageCopyTexture { texture: &state.v_tex, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + wgpu::ImageCopyTexture { + texture: &state.v_tex, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, &packed_v, - wgpu::ImageDataLayout { offset: 0, bytes_per_row: Some(uv_bytes_per_row), rows_per_image: Some(uv_h) }, - wgpu::Extent3d { width: state.uv_pad_w, height: uv_h, depth_or_array_layers: 1 }, + wgpu::ImageDataLayout { + offset: 0, + bytes_per_row: Some(uv_bytes_per_row), + rows_per_image: Some(uv_h), + }, + wgpu::Extent3d { + width: state.uv_pad_w, + height: uv_h, + depth_or_array_layers: 1, + }, ); } // Update params uniform - let params = ParamsUniform { src_w: shared.width, src_h: shared.height, y_tex_w: state.y_pad_w, uv_tex_w: state.uv_pad_w }; + let params = ParamsUniform { + src_w: shared.width, + src_h: shared.height, + y_tex_w: state.y_pad_w, + uv_tex_w: state.uv_pad_w, + }; queue.write_buffer(&state.params_buf, 0, bytemuck::bytes_of(¶ms)); shared.dirty = false; @@ -825,7 +1041,12 @@ impl CallbackTrait for YuvPaintCallback { Vec::new() } - fn paint(&self, _info: egui::PaintCallbackInfo, render_pass: &mut wgpu::RenderPass<'static>, resources: &egui_wgpu_backend::CallbackResources) { + fn paint( + &self, + _info: egui::PaintCallbackInfo, + render_pass: &mut wgpu::RenderPass<'static>, + resources: &egui_wgpu_backend::CallbackResources, + ) { // Acquire device/queue via screen_descriptor? Not available; use resources to fetch our state let shared = self.shared.lock(); if shared.width == 0 || shared.height == 0 { From f8799a35b5aacdf97e2f31376bd7451ca7abd32e Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 9 Jan 2026 16:45:15 -0800 Subject: [PATCH 06/15] update readme --- examples/local_video/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/local_video/README.md b/examples/local_video/README.md index aa39facbd..eacd74bd5 100644 --- a/examples/local_video/README.md +++ b/examples/local_video/README.md @@ -20,11 +20,17 @@ Publisher usage: --camera-index 0 \ --room-name demo \ --identity cam-1 \ + --h265 \ + --max-bitrate 1500000 \ --url https://your.livekit.server \ --api-key YOUR_KEY \ --api-secret YOUR_SECRET ``` +Publisher flags (in addition to the common connection flags above): +- `--h265`: Use H.265/HEVC encoding if supported (falls back to H.264 on failure). +- `--max-bitrate `: Max video bitrate for the main layer in bits per second (e.g. `1500000`). + Subscriber usage: ``` # relies on env vars LIVEKIT_URL, LIVEKIT_API_KEY, LIVEKIT_API_SECRET From 9bb671c4f018b66760b1a2264b24356c0f071b06 Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 9 Jan 2026 16:47:32 -0800 Subject: [PATCH 07/15] lint --- yuv-sys/build.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/yuv-sys/build.rs b/yuv-sys/build.rs index 3e9a40bd3..d3736cb7c 100644 --- a/yuv-sys/build.rs +++ b/yuv-sys/build.rs @@ -135,12 +135,11 @@ fn main() { } // Try to detect system libjpeg (or libjpeg-turbo) via pkg-config to enable MJPEG fast path - let jpeg_pkg = - pkg_config::Config::new() - .probe("libjpeg") - .or_else(|_| pkg_config::Config::new().probe("libjpeg-turbo")) - .or_else(|_| pkg_config::Config::new().probe("jpeg")) - .ok(); + let jpeg_pkg = pkg_config::Config::new() + .probe("libjpeg") + .or_else(|_| pkg_config::Config::new().probe("libjpeg-turbo")) + .or_else(|_| pkg_config::Config::new().probe("jpeg")) + .ok(); let mut build = cc::Build::new(); build From eca22cb71535815098b56281852cbc45de006af1 Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 9 Jan 2026 21:21:10 -0800 Subject: [PATCH 08/15] Update examples/local_video/README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- examples/local_video/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/local_video/README.md b/examples/local_video/README.md index eacd74bd5..4678af367 100644 --- a/examples/local_video/README.md +++ b/examples/local_video/README.md @@ -42,7 +42,7 @@ Subscriber usage: --identity viewer-1 \ --url https://your.livekit.server \ --api-key YOUR_KEY \ - --api-secret YOUR_SECRET + --api-secret YOUR_SECRET # subscribe to a specific participant's video only cargo run -p local_video --bin subscriber -- \ From 2a6aae300046938385f9c261038b2d88f588c4cd Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 9 Jan 2026 22:07:24 -0800 Subject: [PATCH 09/15] Update examples/local_video/src/subscriber.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- examples/local_video/src/subscriber.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/local_video/src/subscriber.rs b/examples/local_video/src/subscriber.rs index a54cc4a29..bac42310e 100644 --- a/examples/local_video/src/subscriber.rs +++ b/examples/local_video/src/subscriber.rs @@ -142,12 +142,12 @@ impl eframe::App for VideoApp { } egui::CentralPanel::default().show(ctx, |ui| { - // Ensure we keep repainting for smooth playback. - ui.ctx().request_repaint(); - // Render into a centered rect that matches the source aspect ratio. This keeps resize // smooth (no feedback loop) and avoids stretching/distortion while dragging. let available = ui.available_size(); + let size = if let Some(aspect) = self.locked_aspect { + let mut w = available.x.max(1.0); + let mut h = (w / aspect).max(1.0); let size = if let Some(aspect) = self.locked_aspect { let mut w = available.x.max(1.0); let mut h = (w / aspect).max(1.0); From 5b5fde440cb99e76da6e929a9fd8b0baf68a43e8 Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 9 Jan 2026 22:10:04 -0800 Subject: [PATCH 10/15] Revert "Update examples/local_video/src/subscriber.rs" This reverts commit 2a6aae300046938385f9c261038b2d88f588c4cd. --- examples/local_video/src/subscriber.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/local_video/src/subscriber.rs b/examples/local_video/src/subscriber.rs index bac42310e..a54cc4a29 100644 --- a/examples/local_video/src/subscriber.rs +++ b/examples/local_video/src/subscriber.rs @@ -142,12 +142,12 @@ impl eframe::App for VideoApp { } egui::CentralPanel::default().show(ctx, |ui| { + // Ensure we keep repainting for smooth playback. + ui.ctx().request_repaint(); + // Render into a centered rect that matches the source aspect ratio. This keeps resize // smooth (no feedback loop) and avoids stretching/distortion while dragging. let available = ui.available_size(); - let size = if let Some(aspect) = self.locked_aspect { - let mut w = available.x.max(1.0); - let mut h = (w / aspect).max(1.0); let size = if let Some(aspect) = self.locked_aspect { let mut w = available.x.max(1.0); let mut h = (w / aspect).max(1.0); From b152159ead3e53b5c5943b925b3190595086cc21 Mon Sep 17 00:00:00 2001 From: David Chen Date: Sun, 11 Jan 2026 23:26:09 -0800 Subject: [PATCH 11/15] add simulcast option --- examples/local_video/README.md | 4 +++- examples/local_video/src/publisher.rs | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/examples/local_video/README.md b/examples/local_video/README.md index 4678af367..2d8e07896 100644 --- a/examples/local_video/README.md +++ b/examples/local_video/README.md @@ -20,6 +20,7 @@ Publisher usage: --camera-index 0 \ --room-name demo \ --identity cam-1 \ + --simulcast \ --h265 \ --max-bitrate 1500000 \ --url https://your.livekit.server \ @@ -29,7 +30,8 @@ Publisher usage: Publisher flags (in addition to the common connection flags above): - `--h265`: Use H.265/HEVC encoding if supported (falls back to H.264 on failure). -- `--max-bitrate `: Max video bitrate for the main layer in bits per second (e.g. `1500000`). +- `--simulcast`: Publish simulcast video (multiple layers when the resolution is large enough). +- `--max-bitrate `: Max video bitrate for the main (highest) layer in bits per second (e.g. `1500000`). Subscriber usage: ``` diff --git a/examples/local_video/src/publisher.rs b/examples/local_video/src/publisher.rs index cd9ec16e3..0384e1622 100644 --- a/examples/local_video/src/publisher.rs +++ b/examples/local_video/src/publisher.rs @@ -48,6 +48,10 @@ struct Args { #[arg(long)] max_bitrate: Option, + /// Enable simulcast publishing (low/medium/high layers as appropriate) + #[arg(long, default_value_t = false)] + simulcast: bool, + /// LiveKit participant identity #[arg(long, default_value = "rust-camera-pub")] identity: String, @@ -187,7 +191,7 @@ async fn main() -> Result<()> { let publish_opts = |codec: VideoCodec| { let mut opts = TrackPublishOptions { source: TrackSource::Camera, - simulcast: false, + simulcast: args.simulcast, video_codec: codec, ..Default::default() }; From 9faed68674ad55f05e21d97fc0115b914327b83f Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 16 Jan 2026 20:49:49 -0800 Subject: [PATCH 12/15] move main app loop into a `run` function --- examples/local_video/src/publisher.rs | 8 ++++++-- examples/local_video/src/subscriber.rs | 7 ++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/examples/local_video/src/publisher.rs b/examples/local_video/src/publisher.rs index 0384e1622..bc43745d4 100644 --- a/examples/local_video/src/publisher.rs +++ b/examples/local_video/src/publisher.rs @@ -95,11 +95,16 @@ async fn main() -> Result<()> { tokio::spawn({ let ctrl_c_received = ctrl_c_received.clone(); async move { - tokio::signal::ctrl_c().await.unwrap(); + let _ = tokio::signal::ctrl_c().await; ctrl_c_received.store(true, Ordering::Release); + info!("Ctrl-C received, exiting..."); } }); + run(args, ctrl_c_received).await +} + +async fn run(args: Args, ctrl_c_received: Arc) -> Result<()> { if args.list_cameras { return list_cameras(); } @@ -256,7 +261,6 @@ async fn main() -> Result<()> { let mut logged_mjpeg_fallback = false; loop { if ctrl_c_received.load(Ordering::Acquire) { - info!("Ctrl-C received, exiting..."); break; } // Wait until the scheduled next frame time diff --git a/examples/local_video/src/subscriber.rs b/examples/local_video/src/subscriber.rs index a54cc4a29..e0662813e 100644 --- a/examples/local_video/src/subscriber.rs +++ b/examples/local_video/src/subscriber.rs @@ -237,11 +237,16 @@ async fn main() -> Result<()> { tokio::spawn({ let ctrl_c_received = ctrl_c_received.clone(); async move { - tokio::signal::ctrl_c().await.unwrap(); + let _ = tokio::signal::ctrl_c().await; ctrl_c_received.store(true, Ordering::Release); + info!("Ctrl-C received, exiting..."); } }); + run(args, ctrl_c_received).await +} + +async fn run(args: Args, ctrl_c_received: Arc) -> Result<()> { // LiveKit connection details (prefer CLI args, fallback to env vars) let url = args.url.or_else(|| env::var("LIVEKIT_URL").ok()).expect( "LiveKit URL must be provided via --url argument or LIVEKIT_URL environment variable", From 1f0d89cbf5b2296cb309b8c133b2d2e97ba8cd99 Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 16 Jan 2026 20:57:37 -0800 Subject: [PATCH 13/15] refactor event handling into separate functions for readability --- examples/local_video/src/subscriber.rs | 597 +++++++++++++------------ 1 file changed, 317 insertions(+), 280 deletions(-) diff --git a/examples/local_video/src/subscriber.rs b/examples/local_video/src/subscriber.rs index e0662813e..be826edb0 100644 --- a/examples/local_video/src/subscriber.rs +++ b/examples/local_video/src/subscriber.rs @@ -119,6 +119,299 @@ fn simulcast_state_full_dims(state: &Arc>) -> Option<(u32, sc.full_dims } +async fn handle_track_subscribed( + track: livekit::track::RemoteTrack, + publication: RemoteTrackPublication, + participant: RemoteParticipant, + allowed_identity: &Option, + shared: &Arc>, + rt: &tokio::runtime::Handle, + active_sid: &Arc>>, + ctrl_c_received: &Arc, + simulcast: &Arc>, +) { + // If a participant filter is set, skip others + if let Some(ref allow) = allowed_identity { + if participant.identity().as_str() != allow { + debug!( + "Skipping track from '{}' (filter set to '{}')", + participant.identity(), + allow + ); + return; + } + } + + let livekit::track::RemoteTrack::Video(video_track) = track else { + return; + }; + + let sid = publication.sid().clone(); + let codec = codec_label(&publication.mime_type()); + // Only handle if we don't already have an active video track + { + let mut active = active_sid.lock(); + if active.as_ref() == Some(&sid) { + debug!("Track {} already active, ignoring duplicate subscribe", sid); + return; + } + if active.is_some() { + debug!( + "A video track is already active ({}), ignoring new subscribe {}", + active.as_ref().unwrap(), + sid + ); + return; + } + *active = Some(sid.clone()); + } + + // Update HUD codec label early (before first frame arrives) + { + let mut s = shared.lock(); + s.codec = codec; + } + + info!( + "Subscribed to video track: {} (sid {}) from {} - codec: {}, simulcast: {}, dimension: {}x{}", + publication.name(), + publication.sid(), + participant.identity(), + publication.mime_type(), + publication.simulcasted(), + publication.dimension().0, + publication.dimension().1 + ); + + // Try to fetch inbound RTP/codec stats for more details + match video_track.get_stats().await { + Ok(stats) => { + let mut codec_by_id: HashMap = HashMap::new(); + let mut inbound: Option = None; + for s in stats.iter() { + match s { + livekit::webrtc::stats::RtcStats::Codec(c) => { + codec_by_id.insert( + c.rtc.id.clone(), + (c.codec.mime_type.clone(), c.codec.sdp_fmtp_line.clone()), + ); + } + livekit::webrtc::stats::RtcStats::InboundRtp(i) => { + if i.stream.kind == "video" { + inbound = Some(i.clone()); + } + } + _ => {} + } + } + + if let Some(i) = inbound { + if let Some((mime, fmtp)) = codec_by_id.get(&i.stream.codec_id) { + info!("Inbound codec: {} (fmtp: {})", mime, fmtp); + } else { + info!("Inbound codec id: {}", i.stream.codec_id); + } + info!( + "Inbound current layer: {}x{} ~{:.1} fps, decoder: {}, power_efficient: {}", + i.inbound.frame_width, + i.inbound.frame_height, + i.inbound.frames_per_second, + i.inbound.decoder_implementation, + i.inbound.power_efficient_decoder + ); + } + } + Err(e) => debug!("Failed to get stats for video track: {:?}", e), + } + + // Start background sink thread + let shared2 = shared.clone(); + let active_sid2 = active_sid.clone(); + let my_sid = sid.clone(); + let rt_clone = rt.clone(); + let ctrl_c_sink = ctrl_c_received.clone(); + // Initialize simulcast state for this publication + { + let mut sc = simulcast.lock(); + sc.available = publication.simulcasted(); + let dim = publication.dimension(); + sc.full_dims = Some((dim.0, dim.1)); + sc.requested_quality = None; + sc.active_quality = None; + sc.publication = Some(publication.clone()); + } + let simulcast2 = simulcast.clone(); + std::thread::spawn(move || { + let mut sink = NativeVideoStream::new(video_track.rtc_track()); + let mut frames: u64 = 0; + let mut last_log = Instant::now(); + let mut logged_first = false; + let mut last_stats = Instant::now(); + let mut fps_window_frames: u64 = 0; + let mut fps_window_start = Instant::now(); + let mut fps_smoothed: f32 = 0.0; + // YUV buffers reused to avoid per-frame allocations + let mut y_buf: Vec = Vec::new(); + let mut u_buf: Vec = Vec::new(); + let mut v_buf: Vec = Vec::new(); + loop { + if ctrl_c_sink.load(Ordering::Acquire) { + break; + } + let next = rt_clone.block_on(async { + tokio::select! { + _ = wait_for_shutdown(ctrl_c_sink.clone()) => None, + frame = sink.next() => frame, + } + }); + let Some(frame) = next else { break }; + let w = frame.buffer.width(); + let h = frame.buffer.height(); + + if !logged_first { + debug!("First frame: {}x{}, type {:?}", w, h, frame.buffer.buffer_type()); + logged_first = true; + } + + // Convert to I420 on CPU, but keep planes separate for GPU sampling + let i420 = frame.buffer.to_i420(); + let (sy, su, sv) = i420.strides(); + let (dy, du, dv) = i420.data(); + + let ch = (h + 1) / 2; + + // Ensure capacity and copy full plane slices + let y_size = (sy * h) as usize; + let u_size = (su * ch) as usize; + let v_size = (sv * ch) as usize; + if y_buf.len() != y_size { + y_buf.resize(y_size, 0); + } + if u_buf.len() != u_size { + u_buf.resize(u_size, 0); + } + if v_buf.len() != v_size { + v_buf.resize(v_size, 0); + } + y_buf.copy_from_slice(dy); + u_buf.copy_from_slice(du); + v_buf.copy_from_slice(dv); + + // Swap buffers into shared state + let mut s = shared2.lock(); + s.width = w as u32; + s.height = h as u32; + s.stride_y = sy as u32; + s.stride_u = su as u32; + s.stride_v = sv as u32; + std::mem::swap(&mut s.y, &mut y_buf); + std::mem::swap(&mut s.u, &mut u_buf); + std::mem::swap(&mut s.v, &mut v_buf); + s.dirty = true; + + // Update smoothed FPS (~500ms window) + fps_window_frames += 1; + let win_elapsed = fps_window_start.elapsed(); + if win_elapsed >= Duration::from_millis(500) { + let inst_fps = (fps_window_frames as f32) / (win_elapsed.as_secs_f32().max(0.001)); + fps_smoothed = if fps_smoothed <= 0.0 { + inst_fps + } else { + // light EMA smoothing to reduce jitter + (fps_smoothed * 0.7) + (inst_fps * 0.3) + }; + s.fps = fps_smoothed; + fps_window_frames = 0; + fps_window_start = Instant::now(); + } + + frames += 1; + let elapsed = last_log.elapsed(); + if elapsed >= Duration::from_secs(2) { + let fps = frames as f64 / elapsed.as_secs_f64(); + info!("Receiving video: {}x{}, ~{:.1} fps", w, h, fps); + frames = 0; + last_log = Instant::now(); + } + // Periodically infer active simulcast quality from inbound stats + if last_stats.elapsed() >= Duration::from_secs(1) { + if let Ok(stats) = rt_clone.block_on(video_track.get_stats()) { + let mut inbound: Option = None; + for s in stats.iter() { + if let livekit::webrtc::stats::RtcStats::InboundRtp(i) = s { + if i.stream.kind == "video" { + inbound = Some(i.clone()); + } + } + } + if let Some(i) = inbound { + if let Some((fw, fh)) = simulcast_state_full_dims(&simulcast2) { + let q = infer_quality_from_dims( + fw, + fh, + i.inbound.frame_width as u32, + i.inbound.frame_height as u32, + ); + let mut sc = simulcast2.lock(); + sc.active_quality = Some(q); + } + } + } + last_stats = Instant::now(); + } + } + info!("Video stream ended for {}", my_sid); + // Clear active sid if still ours + let mut active = active_sid2.lock(); + if active.as_ref() == Some(&my_sid) { + *active = None; + } + }); +} + +fn clear_hud_and_simulcast( + shared: &Arc>, + simulcast: &Arc>, +) { + { + let mut s = shared.lock(); + s.codec.clear(); + s.fps = 0.0; + } + let mut sc = simulcast.lock(); + *sc = SimulcastState::default(); +} + +fn handle_track_unsubscribed( + publication: RemoteTrackPublication, + shared: &Arc>, + active_sid: &Arc>>, + simulcast: &Arc>, +) { + let sid = publication.sid().clone(); + let mut active = active_sid.lock(); + if active.as_ref() == Some(&sid) { + info!("Video track unsubscribed ({}), clearing active sink", sid); + *active = None; + } + clear_hud_and_simulcast(shared, simulcast); +} + +fn handle_track_unpublished( + publication: RemoteTrackPublication, + shared: &Arc>, + active_sid: &Arc>>, + simulcast: &Arc>, +) { + let sid = publication.sid().clone(); + let mut active = active_sid.lock(); + if active.as_ref() == Some(&sid) { + info!("Video track unpublished ({}), clearing active sink", sid); + *active = None; + } + clear_hud_and_simulcast(shared, simulcast); +} + struct VideoApp { shared: Arc>, simulcast: Arc>, @@ -312,290 +605,34 @@ async fn run(args: Args, ctrl_c_received: Arc) -> Result<()> { debug!("Room event: {:?}", evt); match evt { RoomEvent::TrackSubscribed { track, publication, participant } => { - // If a participant filter is set, skip others - if let Some(ref allow) = allowed_identity { - if participant.identity().as_str() != allow { - debug!( - "Skipping track from '{}' (filter set to '{}')", - participant.identity(), - allow - ); - continue; - } - } - if let livekit::track::RemoteTrack::Video(video_track) = track { - let sid = publication.sid().clone(); - let codec = codec_label(&publication.mime_type()); - // Only handle if we don't already have an active video track - { - let mut active = active_sid.lock(); - if active.as_ref() == Some(&sid) { - debug!( - "Track {} already active, ignoring duplicate subscribe", - sid - ); - continue; - } - if active.is_some() { - debug!("A video track is already active ({}), ignoring new subscribe {}", active.as_ref().unwrap(), sid); - continue; - } - *active = Some(sid.clone()); - } - - // Update HUD codec label early (before first frame arrives) - { - let mut s = shared_clone.lock(); - s.codec = codec; - } - - info!( - "Subscribed to video track: {} (sid {}) from {} - codec: {}, simulcast: {}, dimension: {}x{}", - publication.name(), - publication.sid(), - participant.identity(), - publication.mime_type(), - publication.simulcasted(), - publication.dimension().0, - publication.dimension().1 - ); - - // Try to fetch inbound RTP/codec stats for more details - match video_track.get_stats().await { - Ok(stats) => { - let mut codec_by_id: HashMap = - HashMap::new(); - let mut inbound: Option = - None; - for s in stats.iter() { - match s { - livekit::webrtc::stats::RtcStats::Codec(c) => { - codec_by_id.insert( - c.rtc.id.clone(), - ( - c.codec.mime_type.clone(), - c.codec.sdp_fmtp_line.clone(), - ), - ); - } - livekit::webrtc::stats::RtcStats::InboundRtp(i) => { - if i.stream.kind == "video" { - inbound = Some(i.clone()); - } - } - _ => {} - } - } - - if let Some(i) = inbound { - if let Some((mime, fmtp)) = codec_by_id.get(&i.stream.codec_id) - { - info!("Inbound codec: {} (fmtp: {})", mime, fmtp); - } else { - info!("Inbound codec id: {}", i.stream.codec_id); - } - info!( - "Inbound current layer: {}x{} ~{:.1} fps, decoder: {}, power_efficient: {}", - i.inbound.frame_width, - i.inbound.frame_height, - i.inbound.frames_per_second, - i.inbound.decoder_implementation, - i.inbound.power_efficient_decoder - ); - } - } - Err(e) => debug!("Failed to get stats for video track: {:?}", e), - } - // Start background sink thread - let shared2 = shared_clone.clone(); - let active_sid2 = active_sid.clone(); - let my_sid = sid.clone(); - let rt_clone = rt.clone(); - let ctrl_c_sink = ctrl_c_events.clone(); - // Initialize simulcast state for this publication - { - let mut sc = simulcast.lock(); - sc.available = publication.simulcasted(); - let dim = publication.dimension(); - sc.full_dims = Some((dim.0, dim.1)); - sc.requested_quality = None; - sc.active_quality = None; - sc.publication = Some(publication.clone()); - } - let simulcast2 = simulcast.clone(); - std::thread::spawn(move || { - let mut sink = NativeVideoStream::new(video_track.rtc_track()); - let mut frames: u64 = 0; - let mut last_log = Instant::now(); - let mut logged_first = false; - let mut last_stats = Instant::now(); - let mut fps_window_frames: u64 = 0; - let mut fps_window_start = Instant::now(); - let mut fps_smoothed: f32 = 0.0; - // YUV buffers reused to avoid per-frame allocations - let mut y_buf: Vec = Vec::new(); - let mut u_buf: Vec = Vec::new(); - let mut v_buf: Vec = Vec::new(); - loop { - if ctrl_c_sink.load(Ordering::Acquire) { - break; - } - let next = rt_clone.block_on(async { - tokio::select! { - _ = wait_for_shutdown(ctrl_c_sink.clone()) => None, - frame = sink.next() => frame, - } - }); - let Some(frame) = next else { break }; - let w = frame.buffer.width(); - let h = frame.buffer.height(); - - if !logged_first { - debug!( - "First frame: {}x{}, type {:?}", - w, - h, - frame.buffer.buffer_type() - ); - logged_first = true; - } - - // Convert to I420 on CPU, but keep planes separate for GPU sampling - let i420 = frame.buffer.to_i420(); - let (sy, su, sv) = i420.strides(); - let (dy, du, dv) = i420.data(); - - let ch = (h + 1) / 2; - - // Ensure capacity and copy full plane slices - let y_size = (sy * h) as usize; - let u_size = (su * ch) as usize; - let v_size = (sv * ch) as usize; - if y_buf.len() != y_size { - y_buf.resize(y_size, 0); - } - if u_buf.len() != u_size { - u_buf.resize(u_size, 0); - } - if v_buf.len() != v_size { - v_buf.resize(v_size, 0); - } - y_buf.copy_from_slice(dy); - u_buf.copy_from_slice(du); - v_buf.copy_from_slice(dv); - - // Swap buffers into shared state - let mut s = shared2.lock(); - s.width = w as u32; - s.height = h as u32; - s.stride_y = sy as u32; - s.stride_u = su as u32; - s.stride_v = sv as u32; - std::mem::swap(&mut s.y, &mut y_buf); - std::mem::swap(&mut s.u, &mut u_buf); - std::mem::swap(&mut s.v, &mut v_buf); - s.dirty = true; - - // Update smoothed FPS (~500ms window) - fps_window_frames += 1; - let win_elapsed = fps_window_start.elapsed(); - if win_elapsed >= Duration::from_millis(500) { - let inst_fps = (fps_window_frames as f32) - / (win_elapsed.as_secs_f32().max(0.001)); - fps_smoothed = if fps_smoothed <= 0.0 { - inst_fps - } else { - // light EMA smoothing to reduce jitter - (fps_smoothed * 0.7) + (inst_fps * 0.3) - }; - s.fps = fps_smoothed; - fps_window_frames = 0; - fps_window_start = Instant::now(); - } - - frames += 1; - let elapsed = last_log.elapsed(); - if elapsed >= Duration::from_secs(2) { - let fps = frames as f64 / elapsed.as_secs_f64(); - info!("Receiving video: {}x{}, ~{:.1} fps", w, h, fps); - frames = 0; - last_log = Instant::now(); - } - // Periodically infer active simulcast quality from inbound stats - if last_stats.elapsed() >= Duration::from_secs(1) { - if let Ok(stats) = rt_clone.block_on(video_track.get_stats()) { - let mut inbound: Option< - livekit::webrtc::stats::InboundRtpStats, - > = None; - for s in stats.iter() { - if let livekit::webrtc::stats::RtcStats::InboundRtp(i) = - s - { - if i.stream.kind == "video" { - inbound = Some(i.clone()); - } - } - } - if let Some(i) = inbound { - if let Some((fw, fh)) = - simulcast_state_full_dims(&simulcast2) - { - let q = infer_quality_from_dims( - fw, - fh, - i.inbound.frame_width as u32, - i.inbound.frame_height as u32, - ); - let mut sc = simulcast2.lock(); - sc.active_quality = Some(q); - } - } - } - last_stats = Instant::now(); - } - } - info!("Video stream ended for {}", my_sid); - // Clear active sid if still ours - let mut active = active_sid2.lock(); - if active.as_ref() == Some(&my_sid) { - *active = None; - } - }); - } + handle_track_subscribed( + track, + publication, + participant, + &allowed_identity, + &shared_clone, + &rt, + &active_sid, + &ctrl_c_events, + &simulcast, + ) + .await; } RoomEvent::TrackUnsubscribed { publication, .. } => { - let sid = publication.sid().clone(); - let mut active = active_sid.lock(); - if active.as_ref() == Some(&sid) { - info!("Video track unsubscribed ({}), clearing active sink", sid); - *active = None; - } - // Clear HUD codec - { - let mut s = shared_clone.lock(); - s.codec.clear(); - s.fps = 0.0; - } - // Clear simulcast state - let mut sc = simulcast.lock(); - *sc = SimulcastState::default(); + handle_track_unsubscribed( + publication, + &shared_clone, + &active_sid, + &simulcast, + ); } RoomEvent::TrackUnpublished { publication, .. } => { - let sid = publication.sid().clone(); - let mut active = active_sid.lock(); - if active.as_ref() == Some(&sid) { - info!("Video track unpublished ({}), clearing active sink", sid); - *active = None; - } - // Clear HUD codec - { - let mut s = shared_clone.lock(); - s.codec.clear(); - s.fps = 0.0; - } - // Clear simulcast state - let mut sc = simulcast.lock(); - *sc = SimulcastState::default(); + handle_track_unpublished( + publication, + &shared_clone, + &active_sid, + &simulcast, + ); } _ => {} } From eb88d86003fb3a4f71ec41e349172ab36716fafb Mon Sep 17 00:00:00 2001 From: David Chen Date: Fri, 16 Jan 2026 22:11:24 -0800 Subject: [PATCH 14/15] add script to list devices --- Cargo.lock | 1 + examples/local_video/Cargo.toml | 5 + examples/local_video/README.md | 8 +- examples/local_video/src/list_devices.rs | 114 +++++++++++++++++++++++ 4 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 examples/local_video/src/list_devices.rs diff --git a/Cargo.lock b/Cargo.lock index 52f6386f2..f14318d5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3360,6 +3360,7 @@ dependencies = [ "livekit-api", "log", "nokhwa", + "nokhwa-bindings-macos", "objc2 0.6.3", "parking_lot", "tokio", diff --git a/examples/local_video/Cargo.toml b/examples/local_video/Cargo.toml index 8d01c1086..e5fbfcc61 100644 --- a/examples/local_video/Cargo.toml +++ b/examples/local_video/Cargo.toml @@ -12,6 +12,10 @@ path = "src/publisher.rs" name = "subscriber" path = "src/subscriber.rs" +[[bin]] +name = "list_devices" +path = "src/list_devices.rs" + [dependencies] tokio = { version = "1", features = ["full", "parking_lot"] } livekit = { workspace = true, features = ["rustls-tls-native-roots"] } @@ -36,5 +40,6 @@ bytemuck = { version = "1.16", features = ["derive"] } [target.'cfg(target_os = "macos")'.dependencies] objc2 = { version = "0.6.0", features = ["relax-sign-encoding"] } +nokhwa-bindings-macos = "0.2" diff --git a/examples/local_video/README.md b/examples/local_video/README.md index 2d8e07896..b29a21bbf 100644 --- a/examples/local_video/README.md +++ b/examples/local_video/README.md @@ -1,7 +1,8 @@ # local_video -Two examples demonstrating capturing frames from a local camera video and publishing to LiveKit, and subscribing to render video in a window. +Three examples demonstrating capturing frames from a local camera video and publishing to LiveKit, listing camera capabilities, and subscribing to render video in a window. +- list_devices: enumerate available cameras and their capabilities - publisher: capture from a selected camera and publish a video track - subscriber: connect to a room, subscribe to video tracks, and display in a window @@ -28,6 +29,11 @@ Publisher usage: --api-secret YOUR_SECRET ``` +List devices usage: +``` + cargo run -p local_video --bin list_devices +``` + Publisher flags (in addition to the common connection flags above): - `--h265`: Use H.265/HEVC encoding if supported (falls back to H.264 on failure). - `--simulcast`: Publish simulcast video (multiple layers when the resolution is large enough). diff --git a/examples/local_video/src/list_devices.rs b/examples/local_video/src/list_devices.rs new file mode 100644 index 000000000..6280f6275 --- /dev/null +++ b/examples/local_video/src/list_devices.rs @@ -0,0 +1,114 @@ +use anyhow::Result; +use nokhwa::pixel_format::RgbFormat; +use nokhwa::utils::{ + ApiBackend, CameraFormat, CameraInfo, FrameFormat, RequestedFormat, RequestedFormatType, + Resolution, +}; +use nokhwa::Camera; +use std::collections::BTreeMap; + +#[cfg(target_os = "macos")] +use nokhwa_bindings_macos::AVCaptureDevice; + +fn main() -> Result<()> { + let cameras = nokhwa::query(ApiBackend::Auto)?; + if cameras.is_empty() { + println!("No cameras detected."); + return Ok(()); + } + + println!("Available cameras and capabilities:"); + for (idx, info) in cameras.iter().enumerate() { + println!(); + println!("{}. {}", idx, info.human_name()); + match enumerate_capabilities(info) { + Ok(formats) => print_capabilities(&formats), + Err(err) => println!(" Capabilities: unavailable ({})", err), + } + } + + Ok(()) +} + +#[cfg(target_os = "macos")] +fn enumerate_capabilities( + info: &CameraInfo, +) -> Result>>> { + let device = AVCaptureDevice::new(info.index())?; + let formats = device.supported_formats()?; + Ok(capabilities_from_formats(formats)) +} + +#[cfg(not(target_os = "macos"))] +fn enumerate_capabilities( + info: &CameraInfo, +) -> Result>>> { + let requested = RequestedFormat::new::(RequestedFormatType::None); + let mut camera = Camera::new(info.index().clone(), requested)?; + let mut capabilities = BTreeMap::new(); + if let Ok(mut fourccs) = camera.compatible_fourcc() { + fourccs.sort(); + for fourcc in fourccs { + let mut res_map = camera.compatible_list_by_resolution(fourcc)?; + let mut res_sorted = BTreeMap::new(); + for (res, mut fps_list) in res_map.drain() { + fps_list.sort(); + res_sorted.insert(res, fps_list); + } + capabilities.insert(fourcc, res_sorted); + } + } else { + let formats = camera.compatible_camera_formats()?; + capabilities = capabilities_from_formats(formats); + } + + Ok(capabilities) +} + +fn capabilities_from_formats( + formats: Vec, +) -> BTreeMap>> { + let mut capabilities = BTreeMap::new(); + for fmt in formats { + let res_map = capabilities + .entry(fmt.format()) + .or_insert_with(BTreeMap::new); + let fps_list = res_map.entry(fmt.resolution()).or_insert_with(Vec::new); + fps_list.push(fmt.frame_rate()); + } + for res_map in capabilities.values_mut() { + for fps_list in res_map.values_mut() { + fps_list.sort(); + fps_list.dedup(); + } + } + capabilities +} + +fn print_capabilities(capabilities: &BTreeMap>>) { + if capabilities.is_empty() { + println!(" Capabilities: none reported"); + return; + } + + println!(" Capabilities:"); + for (format, resolutions) in capabilities { + println!(" - Format: {}", format); + if resolutions.is_empty() { + println!(" (no resolutions reported)"); + continue; + } + for (resolution, fps_list) in resolutions { + let fps_text = if fps_list.is_empty() { + "unknown".to_string() + } else { + fps_list + .iter() + .map(|fps| fps.to_string()) + .collect::>() + .join(", ") + }; + println!(" {} @ {} fps", resolution, fps_text); + } + } +} From 1beec5fc71b6557fab5c9b860503db6e8d4ad468 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 20 Jan 2026 23:49:51 -0800 Subject: [PATCH 15/15] lint --- examples/local_video/src/list_devices.rs | 10 ++-------- examples/local_video/src/subscriber.rs | 25 ++++-------------------- 2 files changed, 6 insertions(+), 29 deletions(-) diff --git a/examples/local_video/src/list_devices.rs b/examples/local_video/src/list_devices.rs index 6280f6275..4bafc0c96 100644 --- a/examples/local_video/src/list_devices.rs +++ b/examples/local_video/src/list_devices.rs @@ -70,9 +70,7 @@ fn capabilities_from_formats( ) -> BTreeMap>> { let mut capabilities = BTreeMap::new(); for fmt in formats { - let res_map = capabilities - .entry(fmt.format()) - .or_insert_with(BTreeMap::new); + let res_map = capabilities.entry(fmt.format()).or_insert_with(BTreeMap::new); let fps_list = res_map.entry(fmt.resolution()).or_insert_with(Vec::new); fps_list.push(fmt.frame_rate()); } @@ -102,11 +100,7 @@ fn print_capabilities(capabilities: &BTreeMap>() - .join(", ") + fps_list.iter().map(|fps| fps.to_string()).collect::>().join(", ") }; println!(" {} @ {} fps", resolution, fps_text); } diff --git a/examples/local_video/src/subscriber.rs b/examples/local_video/src/subscriber.rs index be826edb0..e8abdbca4 100644 --- a/examples/local_video/src/subscriber.rs +++ b/examples/local_video/src/subscriber.rs @@ -133,11 +133,7 @@ async fn handle_track_subscribed( // If a participant filter is set, skip others if let Some(ref allow) = allowed_identity { if participant.identity().as_str() != allow { - debug!( - "Skipping track from '{}' (filter set to '{}')", - participant.identity(), - allow - ); + debug!("Skipping track from '{}' (filter set to '{}')", participant.identity(), allow); return; } } @@ -369,10 +365,7 @@ async fn handle_track_subscribed( }); } -fn clear_hud_and_simulcast( - shared: &Arc>, - simulcast: &Arc>, -) { +fn clear_hud_and_simulcast(shared: &Arc>, simulcast: &Arc>) { { let mut s = shared.lock(); s.codec.clear(); @@ -619,20 +612,10 @@ async fn run(args: Args, ctrl_c_received: Arc) -> Result<()> { .await; } RoomEvent::TrackUnsubscribed { publication, .. } => { - handle_track_unsubscribed( - publication, - &shared_clone, - &active_sid, - &simulcast, - ); + handle_track_unsubscribed(publication, &shared_clone, &active_sid, &simulcast); } RoomEvent::TrackUnpublished { publication, .. } => { - handle_track_unpublished( - publication, - &shared_clone, - &active_sid, - &simulcast, - ); + handle_track_unpublished(publication, &shared_clone, &active_sid, &simulcast); } _ => {} }