diff --git a/Cargo.lock b/Cargo.lock index b953c42..48f1dd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,7 +173,7 @@ dependencies = [ "log", "ndk", "ndk-context", - "ndk-sys", + "ndk-sys 0.6.0+11769913", "num_enum", "thiserror 2.0.20", ] @@ -249,6 +249,26 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "arboard" +version = "3.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0348a1c054491f4bfe6ab86a7b6ab1e44e45d899005de92f58b3df180b36ddaf" +dependencies = [ + "clipboard-win", + "image", + "log", + "objc2 0.6.4", + "objc2-app-kit 0.3.2", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-foundation 0.3.2", + "parking_lot", + "percent-encoding", + "windows-sys 0.60.2", + "x11rb", +] + [[package]] name = "arrayref" version = "0.3.9" @@ -273,6 +293,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" +[[package]] +name = "ash" +version = "0.38.0+1.3.281" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f" +dependencies = [ + "libloading", +] + [[package]] name = "asn1-rs" version = "0.6.2" @@ -565,6 +594,12 @@ dependencies = [ "wyz", ] +[[package]] +name = "block" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" + [[package]] name = "block-buffer" version = "0.10.4" @@ -683,6 +718,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.12.1" @@ -731,18 +772,43 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "calloop" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dbf9978365bac10f54d1d4b04f7ce4427e51f71d61f2fe15e3fed5166474df7" +dependencies = [ + "bitflags 2.13.1", + "polling", + "rustix 1.1.4", + "slab", + "tracing", +] + [[package]] name = "calloop-wayland-source" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a66a987056935f7efce4ab5668920b5d0dac4a7c99991a67395f13702ddd20" dependencies = [ - "calloop", + "calloop 0.13.0", "rustix 0.38.44", "wayland-backend", "wayland-client", ] +[[package]] +name = "calloop-wayland-source" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138efcf0940a02ebf0cc8d1eff41a1682a46b431630f4c52450d6265876021fa" +dependencies = [ + "calloop 0.14.4", + "rustix 1.1.4", + "wayland-backend", + "wayland-client", +] + [[package]] name = "camino" version = "1.2.5" @@ -997,6 +1063,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "clipboard-win" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde03770d3df201d4fb868f2c9c59e66a3e4e2bd06692a0fe701e7103c7e84d4" +dependencies = [ + "error-code", +] + [[package]] name = "cmake" version = "0.1.58" @@ -1012,6 +1087,16 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width 0.1.14", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -1291,6 +1376,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-bigint" version = "0.5.5" @@ -1812,7 +1903,7 @@ checksum = "521e380c0c8afb8d9a1e83a1822ee03556fc3e3e7dbc1fd30be14e37f9cb3f89" dependencies = [ "bit-set", "cssparser", - "foldhash", + "foldhash 0.2.0", "html5ever", "precomputed-hash", "selectors", @@ -1951,6 +2042,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ecolor" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc4feb366740ded31a004a0e4452fbf84e80ef432ecf8314c485210229672fd1" +dependencies = [ + "bytemuck", + "emath", +] + [[package]] name = "ed25519" version = "2.2.3" @@ -1998,6 +2099,59 @@ dependencies = [ "zeroize", ] +[[package]] +name = "egui" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dd34cec49ab55d85ebf70139cb1ccd29c977ef6b6ba4fe85489d6877ee9ef3" +dependencies = [ + "ahash", + "bitflags 2.13.1", + "emath", + "epaint", + "log", + "nohash-hasher", + "profiling", +] + +[[package]] +name = "egui-wgpu" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d319dfef570f699b6e9114e235e862a2ddcf75f0d1a061de9e1328d92146d820" +dependencies = [ + "ahash", + "bytemuck", + "document-features", + "egui", + "epaint", + "log", + "profiling", + "thiserror 1.0.69", + "type-map", + "web-time", + "wgpu", +] + +[[package]] +name = "egui-winit" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d9dfbb78fe4eb9c3a39ad528b90ee5915c252e77bbab9d4ebc576541ab67e13" +dependencies = [ + "ahash", + "arboard", + "bytemuck", + "egui", + "log", + "profiling", + "raw-window-handle", + "smithay-clipboard", + "web-time", + "webbrowser", + "winit", +] + [[package]] name = "either" version = "1.17.0" @@ -2047,6 +2201,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "emath" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e4cadcff7a5353ba72b7fea76bf2122b5ebdbc68e8155aa56dfdea90083fe1b" +dependencies = [ + "bytemuck", +] + [[package]] name = "embed-resource" version = "3.0.11" @@ -2094,6 +2257,30 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "epaint" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fcc0f5a7c613afd2dee5e4b30c3e6acafb8ad6f0edb06068811f708a67c562" +dependencies = [ + "ab_glyph", + "ahash", + "bytemuck", + "ecolor", + "emath", + "epaint_default_fonts", + "log", + "nohash-hasher", + "parking_lot", + "profiling", +] + +[[package]] +name = "epaint_default_fonts" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7e7a64c02cf7a5b51e745a9e45f60660a286f151c238b9d397b3e923f5082f" + [[package]] name = "equivalent" version = "1.0.2" @@ -2121,6 +2308,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "error-code" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" + [[package]] name = "event-listener" version = "5.4.2" @@ -2159,6 +2352,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "fax" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" + [[package]] name = "fdeflate" version = "0.3.7" @@ -2250,6 +2449,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foldhash" version = "0.2.0" @@ -2610,6 +2815,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "gl_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a95dfc23a2b4a9a2f5ab41d194f8bfda3cabec42af4e39f08c339eb2a0c124d" +dependencies = [ + "khronos_api", + "log", + "xml-rs", +] + [[package]] name = "glib" version = "0.18.5" @@ -2707,6 +2923,27 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" +[[package]] +name = "glow" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e5ea60d70410161c8bf5da3fdfeaa1c72ed2c15f8bbb9d19fe3a4fad085f08" +dependencies = [ + "js-sys", + "slotmap", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "glutin_wgl_sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4ee00b289aba7a9e5306d57c2d05499b2e5dc427f84ac708bd2c090212cf3e" +dependencies = [ + "gl_generator", +] + [[package]] name = "gobject-sys" version = "0.18.0" @@ -2729,6 +2966,57 @@ dependencies = [ "system-deps 7.0.8", ] +[[package]] +name = "gpu-alloc" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45cf04b2726f02df5508c6de726acdc90cdf97ac771a9a0ffd8ba10a6e696bf9" +dependencies = [ + "bitflags 2.13.1", + "gpu-alloc-types", +] + +[[package]] +name = "gpu-alloc-types" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2bbed164dd10ed526c2e4fe3e721ca4a71c61730e5aafac6844b417b3227058" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "gpu-allocator" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c151a2a5ef800297b4e79efa4f4bec035c5f51d5ae587287c9b952bdf734cacd" +dependencies = [ + "log", + "presser", + "thiserror 1.0.69", + "windows 0.58.0", +] + +[[package]] +name = "gpu-descriptor" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b89c83349105e3732062a895becfc71a8f921bb71ecbbdd8ff99263e3b53a0ca" +dependencies = [ + "bitflags 2.13.1", + "gpu-descriptor-types", + "hashbrown 0.15.5", +] + +[[package]] +name = "gpu-descriptor-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdf242682df893b86f33a73828fb09ca4b2d3bb6cc95249707fc684d27484b91" +dependencies = [ + "bitflags 2.13.1", +] + [[package]] name = "group" version = "0.13.0" @@ -2915,6 +3203,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hash32" version = "0.2.1" @@ -2939,6 +3238,15 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -2991,6 +3299,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hexf-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" + [[package]] name = "hkdf" version = "0.12.4" @@ -3298,6 +3612,20 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "moxcms", + "num-traits", + "png 0.18.1", + "tiff", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -3361,7 +3689,7 @@ dependencies = [ "dyn-clone", "fuzzy-matcher", "unicode-segmentation", - "unicode-width", + "unicode-width 0.2.2", ] [[package]] @@ -4054,6 +4382,23 @@ dependencies = [ "zeroize", ] +[[package]] +name = "khronos-egl" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aae1df220ece3c0ada96b8153459b67eebe9ae9212258bb0134ae60416fdf76" +dependencies = [ + "libc", + "libloading", + "pkg-config", +] + +[[package]] +name = "khronos_api" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" + [[package]] name = "kstring" version = "2.0.2" @@ -4190,6 +4535,15 @@ dependencies = [ "libc", ] +[[package]] +name = "malloc_buf" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" +dependencies = [ + "libc", +] + [[package]] name = "markup5ever" version = "0.38.0" @@ -4272,6 +4626,21 @@ dependencies = [ "autocfg", ] +[[package]] +name = "metal" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f569fb946490b5743ad69813cb19629130ce9374034abe31614a36402d18f99e" +dependencies = [ + "bitflags 2.13.1", + "block", + "core-graphics-types 0.1.3", + "foreign-types", + "log", + "objc", + "paste", +] + [[package]] name = "mime" version = "0.3.17" @@ -4306,6 +4675,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "muda" version = "0.19.3" @@ -4353,6 +4732,28 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "naga" +version = "24.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e380993072e52eef724eddfcde0ed013b0c023c3f0417336ed041aa9f076994e" +dependencies = [ + "arrayvec", + "bit-set", + "bitflags 2.13.1", + "cfg_aliases 0.2.2", + "codespan-reporting", + "hexf-parse", + "indexmap 2.14.0", + "log", + "rustc-hash 1.1.0", + "spirv", + "strum", + "termcolor", + "thiserror 2.0.20", + "unicode-xid", +] + [[package]] name = "ndk" version = "0.9.0" @@ -4362,7 +4763,7 @@ dependencies = [ "bitflags 2.13.1", "jni-sys 0.3.1", "log", - "ndk-sys", + "ndk-sys 0.6.0+11769913", "num_enum", "raw-window-handle", "thiserror 1.0.69", @@ -4374,6 +4775,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" +[[package]] +name = "ndk-sys" +version = "0.5.0+25.2.9519653" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691" +dependencies = [ + "jni-sys 0.3.1", +] + [[package]] name = "ndk-sys" version = "0.6.0+11769913" @@ -4414,6 +4824,12 @@ dependencies = [ "libc", ] +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + [[package]] name = "nom" version = "7.1.3" @@ -4521,6 +4937,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "objc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" +dependencies = [ + "malloc_buf", +] + [[package]] name = "objc-sys" version = "0.3.5" @@ -4573,6 +4998,7 @@ dependencies = [ "block2 0.6.2", "objc2 0.6.4", "objc2-core-foundation", + "objc2-core-graphics", "objc2-foundation 0.3.2", ] @@ -5058,6 +5484,15 @@ dependencies = [ "libredox", ] +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-stream" version = "0.2.0" @@ -5196,6 +5631,12 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pastey" version = "0.2.3" @@ -5554,6 +5995,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "pollster" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" + [[package]] name = "poly1305" version = "0.8.0" @@ -5654,6 +6101,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "presser" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" + [[package]] name = "primefield" version = "0.14.0" @@ -5757,6 +6210,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "profiling" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" + [[package]] name = "ptr_meta" version = "0.3.2" @@ -5777,6 +6236,18 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quick-xml" version = "0.41.0" @@ -5797,7 +6268,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp 0.5.14", - "rustc-hash", + "rustc-hash 2.1.2", "rustls", "socket2", "thiserror 2.0.20", @@ -5817,7 +6288,7 @@ dependencies = [ "lru-slab", "rand 0.9.5", "ring", - "rustc-hash", + "rustc-hash 2.1.2", "rustls", "rustls-pki-types", "slab", @@ -5945,6 +6416,12 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "range-alloc" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca45419789ae5a7899559e9512e58ca889e41f04f1f2445e9f4b290ceccd1d08" + [[package]] name = "raw-window-handle" version = "0.6.2" @@ -6197,40 +6674,15 @@ dependencies = [ ] [[package]] -name = "remotedesk-linux-terminal" +name = "remotedesk-native-gui" version = "0.2.19" dependencies = [ - "async-trait", - "base64 0.22.1", - "cpal", - "crossterm", - "ed25519-dalek 2.2.0", - "futures-util", - "inquire", - "ironrdp-rdpsnd", - "ironrdp-rdpsnd-native", - "keyring", - "rand 0.9.5", - "raw-window-handle", - "remotedesk-agent-runtime", - "remotedesk-credential-store", - "remotedesk-edge-service", - "remotedesk-protocol", - "reqwest 0.12.28", - "rustls", - "rustls-pemfile", - "serde", - "serde_json", - "sha2 0.10.9", - "softbuffer", - "tiny_http", - "tokio", - "tokio-rustls", - "tokio-tungstenite", - "url", - "windows 0.62.2", + "egui", + "egui-wgpu", + "egui-winit", + "pollster", + "wgpu", "winit", - "zeroize", ] [[package]] @@ -6290,19 +6742,32 @@ name = "remotedesk-windows-agent" version = "0.2.19" dependencies = [ "anyhow", + "base64 0.22.1", "clap", + "cpal", "flate2", + "getrandom 0.3.4", + "hmac 0.13.0", + "memmap2", + "opus2", "serde", "serde_json", + "sha2 0.11.0", "tokio", "windows 0.62.2", + "windows-sys 0.61.2", + "zeroize", ] [[package]] name = "remotedesk-windows-agent-viewer" version = "0.2.19" dependencies = [ + "cpal", "flate2", + "opus2", + "remotedesk-native-video", + "remotedesk-protocol", "serde", "serde_json", "softbuffer", @@ -6319,6 +6784,12 @@ dependencies = [ "bytecheck", ] +[[package]] +name = "renderdoc-sys" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b30a45b0cd0bcca8037f3d0dc3421eaf95327a17cad11964fb8179b4fc4832" + [[package]] name = "reqwest" version = "0.12.28" @@ -6663,7 +7134,7 @@ dependencies = [ "log", "rand 0.10.2", "rtc-shared", - "rustc-hash", + "rustc-hash 2.1.2", "slab", "thiserror 2.0.20", ] @@ -6766,6 +7237,12 @@ dependencies = [ "smallvec", ] +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.2" @@ -7021,7 +7498,7 @@ dependencies = [ "ab_glyph", "log", "memmap2", - "smithay-client-toolkit", + "smithay-client-toolkit 0.19.2", "tiny-skia", ] @@ -7099,7 +7576,7 @@ dependencies = [ "phf", "phf_codegen", "precomputed-hash", - "rustc-hash", + "rustc-hash 2.1.2", "servo_arc", "smallvec", ] @@ -7487,6 +7964,15 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +[[package]] +name = "slotmap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" +dependencies = [ + "version_check", +] + [[package]] name = "smallvec" version = "1.15.2" @@ -7500,8 +7986,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3457dea1f0eb631b4034d61d4d8c32074caa6cd1ab2d59f2327bd8461e2c0016" dependencies = [ "bitflags 2.13.1", - "calloop", - "calloop-wayland-source", + "calloop 0.13.0", + "calloop-wayland-source 0.3.0", "cursor-icon", "libc", "log", @@ -7518,6 +8004,44 @@ dependencies = [ "xkeysym", ] +[[package]] +name = "smithay-client-toolkit" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0512da38f5e2b31201a93524adb8d3136276fa4fe4aafab4e1f727a82b534cc0" +dependencies = [ + "bitflags 2.13.1", + "calloop 0.14.4", + "calloop-wayland-source 0.4.1", + "cursor-icon", + "libc", + "log", + "memmap2", + "rustix 1.1.4", + "thiserror 2.0.20", + "wayland-backend", + "wayland-client", + "wayland-csd-frame", + "wayland-cursor", + "wayland-protocols", + "wayland-protocols-experimental", + "wayland-protocols-misc", + "wayland-protocols-wlr", + "wayland-scanner", + "xkeysym", +] + +[[package]] +name = "smithay-clipboard" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71704c03f739f7745053bde45fa203a46c58d25bc5c4efba1d9a60e9dba81226" +dependencies = [ + "libc", + "smithay-client-toolkit 0.20.0", + "wayland-backend", +] + [[package]] name = "smol_str" version = "0.2.2" @@ -7604,6 +8128,15 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spirv" +version = "0.3.0+sdk-1.3.268.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eda41003dc44290527a59b13432d4a0379379fa074b70174882adfbdfd917844" +dependencies = [ + "bitflags 2.13.1", +] + [[package]] name = "spki" version = "0.7.3" @@ -7735,6 +8268,28 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.119", +] + [[package]] name = "substring" version = "1.4.5" @@ -7881,7 +8436,7 @@ dependencies = [ "libc", "log", "ndk", - "ndk-sys", + "ndk-sys 0.6.0+11769913", "objc2 0.6.4", "objc2-app-kit 0.3.2", "objc2-foundation 0.3.2", @@ -8159,6 +8714,15 @@ dependencies = [ "new_debug_unreachable", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -8208,6 +8772,20 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + [[package]] name = "time" version = "0.3.55" @@ -8661,6 +9239,15 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "type-map" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb30dbbd9036155e74adad6812e9898d03ec374946234fbcebd5dfc7b9187b90" +dependencies = [ + "rustc-hash 2.1.2", +] + [[package]] name = "typeid" version = "1.0.3" @@ -8743,12 +9330,24 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + [[package]] name = "unicode-width" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "universal-hash" version = "0.5.1" @@ -9048,6 +9647,32 @@ dependencies = [ "wayland-scanner", ] +[[package]] +name = "wayland-protocols-experimental" +version = "20250721.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40a1f863128dcaaec790d7b4b396cc9b9a7a079e878e18c47e6c2d2c5a8dcbb1" +dependencies = [ + "bitflags 2.13.1", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-misc" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e9567599ef23e09b8dad6e429e5738d4509dfc46b3b21f32841a304d16b29c8" +dependencies = [ + "bitflags 2.13.1", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + [[package]] name = "wayland-protocols-plasma" version = "0.3.12" @@ -9129,6 +9754,22 @@ dependencies = [ "string_cache_codegen", ] +[[package]] +name = "webbrowser" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62c35be770821a214dbc362fc26908c853e776c0004294d0b10b8a6bad582f94" +dependencies = [ + "jni 0.22.4", + "log", + "ndk-context", + "objc2 0.6.4", + "objc2-app-kit 0.3.2", + "objc2-foundation 0.3.2", + "url", + "web-sys", +] + [[package]] name = "webkit2gtk" version = "2.0.2" @@ -9210,8 +9851,8 @@ dependencies = [ "webview2-com-sys", "windows 0.61.3", "windows-core 0.61.2", - "windows-implement", - "windows-interface", + "windows-implement 0.60.2", + "windows-interface 0.59.3", ] [[package]] @@ -9236,6 +9877,121 @@ dependencies = [ "windows-core 0.61.2", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + +[[package]] +name = "wgpu" +version = "24.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b0b3436f0729f6cdf2e6e9201f3d39dc95813fad61d826c1ed07918b4539353" +dependencies = [ + "arrayvec", + "bitflags 2.13.1", + "cfg_aliases 0.2.2", + "document-features", + "js-sys", + "log", + "naga", + "parking_lot", + "profiling", + "raw-window-handle", + "smallvec", + "static_assertions", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "wgpu-core", + "wgpu-hal", + "wgpu-types", +] + +[[package]] +name = "wgpu-core" +version = "24.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f0aa306497a238d169b9dc70659105b4a096859a34894544ca81719242e1499" +dependencies = [ + "arrayvec", + "bit-vec 0.8.0", + "bitflags 2.13.1", + "cfg_aliases 0.2.2", + "document-features", + "indexmap 2.14.0", + "log", + "naga", + "once_cell", + "parking_lot", + "profiling", + "raw-window-handle", + "rustc-hash 1.1.0", + "smallvec", + "thiserror 2.0.20", + "wgpu-hal", + "wgpu-types", +] + +[[package]] +name = "wgpu-hal" +version = "24.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f112f464674ca69f3533248508ee30cb84c67cf06c25ff6800685f5e0294e259" +dependencies = [ + "android_system_properties", + "arrayvec", + "ash", + "bit-set", + "bitflags 2.13.1", + "block", + "bytemuck", + "cfg_aliases 0.2.2", + "core-graphics-types 0.1.3", + "glow", + "glutin_wgl_sys", + "gpu-alloc", + "gpu-allocator", + "gpu-descriptor", + "js-sys", + "khronos-egl", + "libc", + "libloading", + "log", + "metal", + "naga", + "ndk-sys 0.5.0+25.2.9519653", + "objc", + "once_cell", + "ordered-float", + "parking_lot", + "profiling", + "range-alloc", + "raw-window-handle", + "renderdoc-sys", + "rustc-hash 1.1.0", + "smallvec", + "thiserror 2.0.20", + "wasm-bindgen", + "web-sys", + "wgpu-types", + "windows 0.58.0", + "windows-core 0.58.0", +] + +[[package]] +name = "wgpu-types" +version = "24.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50ac044c0e76c03a0378e7786ac505d010a873665e2d51383dcff8dd227dc69c" +dependencies = [ + "bitflags 2.13.1", + "js-sys", + "log", + "web-sys", +] + [[package]] name = "whoami" version = "2.1.2" @@ -9301,6 +10057,16 @@ dependencies = [ "windows-version", ] +[[package]] +name = "windows" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" +dependencies = [ + "windows-core 0.58.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows" version = "0.61.3" @@ -9344,14 +10110,27 @@ dependencies = [ "windows-core 0.62.2", ] +[[package]] +name = "windows-core" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" +dependencies = [ + "windows-implement 0.58.0", + "windows-interface 0.58.0", + "windows-result 0.2.0", + "windows-strings 0.1.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ - "windows-implement", - "windows-interface", + "windows-implement 0.60.2", + "windows-interface 0.59.3", "windows-link 0.1.3", "windows-result 0.3.4", "windows-strings 0.4.2", @@ -9363,8 +10142,8 @@ version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ - "windows-implement", - "windows-interface", + "windows-implement 0.60.2", + "windows-interface 0.59.3", "windows-link 0.2.1", "windows-result 0.4.1", "windows-strings 0.5.1", @@ -9392,6 +10171,17 @@ dependencies = [ "windows-threading 0.2.1", ] +[[package]] +name = "windows-implement" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "windows-implement" version = "0.60.2" @@ -9403,6 +10193,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "windows-interface" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "windows-interface" version = "0.59.3" @@ -9457,6 +10258,15 @@ dependencies = [ "windows-strings 0.5.1", ] +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.3.4" @@ -9475,6 +10285,16 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result 0.2.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-strings" version = "0.4.2" @@ -9763,7 +10583,7 @@ dependencies = [ "bitflags 2.13.1", "block2 0.5.1", "bytemuck", - "calloop", + "calloop 0.13.0", "cfg_aliases 0.2.2", "concurrent-queue", "core-foundation 0.9.4", @@ -9785,7 +10605,7 @@ dependencies = [ "redox_syscall 0.4.1", "rustix 0.38.44", "sctk-adwaita", - "smithay-client-toolkit", + "smithay-client-toolkit 0.19.2", "smol_str", "tracing", "unicode-segmentation", @@ -10070,6 +10890,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" +[[package]] +name = "xml-rs" +version = "0.8.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7" + [[package]] name = "yasna" version = "0.6.0" @@ -10277,6 +11103,21 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" +[[package]] +name = "zune-core" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] + [[package]] name = "zvariant" version = "5.14.0" diff --git a/Cargo.toml b/Cargo.toml index 7f6940c..196f290 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,15 +5,14 @@ members = [ "agent/agent-runtime", "agent/windows-agent", "client/app-shell", + "client/native-gui", "client/crates/client-core", "client/helpers/control-service", "client/helpers/credential-store", - "client/helpers/linux-terminal", "client/helpers/native-video", + "client/helpers/windows-agent-viewer", "client/helpers/rdp-session", "client/helpers/rdp-viewer", - "client/helpers/windows-agent-viewer", - "edge/edge-service", "protocol", ] diff --git a/README.md b/README.md index 4555ff5..7986e0c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # RemoteDesk +> 架构决策（2026-08-14）：所有发布运行时统一使用 Rust。Agent 是服务端； +> 原生 Client 使用 winit、wgpu 和 egui；两端使用 str0m WebRTC。Go、 +> Hysteria2、Tauri、WebView、React 和软件兼容路径仅属于迁移期旧代码， +> 不属于目标架构。详见 [ADR 0001](docs/adr/0001-all-rust-runtime.md)。 + RemoteDesk 是一套面向个人工作站的 Windows 远程桌面客户端。项目不包含 VDI、桌面池或多租户调度，重点是从 Windows 控制端连接 Windows 和现代 Linux 被控端。 ## 产品定位 diff --git a/agent/agent-core/src/lib.rs b/agent/agent-core/src/lib.rs index 4ab95b0..f3906a0 100644 --- a/agent/agent-core/src/lib.rs +++ b/agent/agent-core/src/lib.rs @@ -703,7 +703,7 @@ mod tests { let selected = select_encoder( DesktopBackend::XorgShmCompatibility, - ZeroCopyPolicy::Compatibility, + ZeroCopyPolicy::RequiredEndToEnd, &capture, &candidates, ) @@ -788,7 +788,7 @@ mod tests { let selected = select_encoder( DesktopBackend::WaylandPipeWire, - ZeroCopyPolicy::Compatibility, + ZeroCopyPolicy::RequiredEndToEnd, &capture, &candidates, ) @@ -828,7 +828,7 @@ mod tests { let selected = select_encoder( DesktopBackend::WaylandPipeWire, - ZeroCopyPolicy::Compatibility, + ZeroCopyPolicy::RequiredEndToEnd, &capture, &candidates, ) @@ -936,7 +936,7 @@ mod tests { transfer_mode: TransferMode::SameAdapterZeroCopy, }; let mut pipeline = VideoPipeline::active( - ZeroCopyPolicy::Compatibility, + ZeroCopyPolicy::RequiredEndToEnd, Resolution::new(1920, 1080).unwrap(), zero_copy, ) diff --git a/agent/agent-core/src/multi_display.rs b/agent/agent-core/src/multi_display.rs index c8c1be7..e190543 100644 --- a/agent/agent-core/src/multi_display.rs +++ b/agent/agent-core/src/multi_display.rs @@ -989,7 +989,7 @@ mod tests { let plan = MultiDisplayCapturePlan::create( DesktopBackend::WaylandPipeWire, - ZeroCopyPolicy::Compatibility, + ZeroCopyPolicy::RequiredEndToEnd, &topology, &sources, &DisplaySelection::all(7), @@ -1071,7 +1071,7 @@ mod tests { let plan = MultiDisplayCapturePlan::create( DesktopBackend::WaylandPipeWire, - ZeroCopyPolicy::Compatibility, + ZeroCopyPolicy::RequiredEndToEnd, &topology, &sources, &DisplaySelection::all(7), @@ -1098,7 +1098,7 @@ mod tests { assert_eq!( MultiDisplayCapturePlan::create( DesktopBackend::WaylandPipeWire, - ZeroCopyPolicy::Compatibility, + ZeroCopyPolicy::RequiredEndToEnd, &topology, &sources, &DisplaySelection::all(7), diff --git a/agent/windows-agent/Cargo.toml b/agent/windows-agent/Cargo.toml index 2ffa099..c8c98f9 100644 --- a/agent/windows-agent/Cargo.toml +++ b/agent/windows-agent/Cargo.toml @@ -12,21 +12,43 @@ path = "src/main.rs" [dependencies] anyhow = "1.0" +base64 = "0.22" clap = { version = "4.5", features = ["derive"] } flate2 = "1.1" +getrandom = { version = "0.3", features = ["std"] } +hmac = "=0.13.0" +memmap2 = "0.9" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +sha2 = "=0.11.0" tokio = { version = "1.47", features = ["macros", "net", "rt-multi-thread", "io-util", "sync", "time"] } +zeroize = "1.8" windows = { version = "0.62.2", features = [ "Win32_Foundation", + "Win32_Graphics_Direct3D", "Win32_Graphics_Direct3D11", "Win32_Graphics_Dxgi", "Win32_Graphics_Dxgi_Common", "Win32_Graphics_Gdi", + "Win32_Media_MediaFoundation", + "Win32_System_Com", + "Win32_System_Ole", + "Win32_System_Variant", "Win32_UI_Input_KeyboardAndMouse", "Win32_UI_WindowsAndMessaging", ] } +[target.'cfg(windows)'.dependencies] +cpal = "0.17.3" +opus2 = { version = "0.4.0", features = ["bundled"] } +windows-sys = { version = "0.61.2", features = [ + "Win32_Foundation", + "Win32_Security", + "Win32_Security_Authorization", + "Win32_Storage_FileSystem", + "Win32_System_Threading", +] } + [lints.rust] unsafe_code = "allow" diff --git a/agent/windows-agent/src/audio.rs b/agent/windows-agent/src/audio.rs new file mode 100644 index 0000000..9896952 --- /dev/null +++ b/agent/windows-agent/src/audio.rs @@ -0,0 +1,471 @@ +//! Windows WASAPI loopback capture and fixed-duration Opus packetization. +//! +//! The capture callback is deliberately tiny: it only copies normalized f32 +//! samples into a bounded queue. Resampling, Opus encoding, timestamps, and +//! discontinuity marking stay on the media worker thread. + +#[cfg(windows)] +mod windows_audio { + use cpal::traits::{DeviceTrait as _, HostTrait as _, StreamTrait as _}; + use opus2::{Application, Bitrate, Channels, Encoder}; + use std::io; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::mpsc::{Receiver, SyncSender, TryRecvError, TrySendError, sync_channel}; + use std::time::Instant; + + const SAMPLE_RATE: u32 = 48_000; + const CHANNELS: usize = 2; + const FRAME_SAMPLES: usize = 960; + const MAX_OPUS_BYTES: usize = 4 * 1024; + + #[derive(Debug)] + pub(crate) struct CapturedAudioChunk { + pub(crate) samples: Vec, + pub(crate) capture_end_pts_100ns: u64, + } + + pub(crate) struct AudioCapture { + receiver: Receiver, + pub(crate) source_rate: u32, + pub(crate) source_channels: usize, + pub(crate) discontinuity: Arc, + _stream: cpal::Stream, + } + + impl AudioCapture { + pub(crate) fn new(session_started: Instant) -> io::Result { + let host = cpal::default_host(); + let device = host.default_output_device().ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + "WASAPI default output device unavailable", + ) + })?; + let supported = device + .default_output_config() + .map_err(|error| io::Error::other(error.to_string()))?; + let source_rate = supported.sample_rate(); + let source_channels = usize::from(supported.channels()); + if source_rate == 0 || source_channels == 0 || source_channels > 8 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "WASAPI loopback format is outside supported bounds", + )); + } + let config = supported.config(); + let (sender, receiver) = sync_channel(32); + let discontinuity = Arc::new(AtomicBool::new(false)); + let callback_discontinuity = Arc::clone(&discontinuity); + let error_discontinuity = Arc::clone(&discontinuity); + let error_callback = move |_error: cpal::StreamError| { + error_discontinuity.store(true, Ordering::Release); + }; + let stream = match supported.sample_format() { + cpal::SampleFormat::F32 => build_stream::( + &device, + &config, + sender, + callback_discontinuity, + session_started, + error_callback, + )?, + cpal::SampleFormat::I16 => build_stream::( + &device, + &config, + sender, + callback_discontinuity, + session_started, + error_callback, + )?, + cpal::SampleFormat::I32 => build_stream::( + &device, + &config, + sender, + callback_discontinuity, + session_started, + error_callback, + )?, + cpal::SampleFormat::U16 => build_stream::( + &device, + &config, + sender, + callback_discontinuity, + session_started, + error_callback, + )?, + cpal::SampleFormat::F64 => build_stream::( + &device, + &config, + sender, + callback_discontinuity, + session_started, + error_callback, + )?, + _ => { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "WASAPI loopback sample format is unsupported", + )); + } + }; + stream + .play() + .map_err(|error| io::Error::other(error.to_string()))?; + Ok(Self { + receiver, + source_rate, + source_channels, + discontinuity, + _stream: stream, + }) + } + + pub(crate) fn try_receive(&self) -> Result, TryRecvError> { + self.receiver.try_recv().map(Some) + } + + pub(crate) fn try_receive_latest( + &self, + ) -> Result, TryRecvError> { + let mut latest = self.receiver.try_recv()?; + loop { + match self.receiver.try_recv() { + Ok(chunk) => latest = chunk, + Err(TryRecvError::Empty) => return Ok(Some(latest)), + Err(TryRecvError::Disconnected) => return Err(TryRecvError::Disconnected), + } + } + } + } + + fn build_stream( + device: &cpal::Device, + config: &cpal::StreamConfig, + sender: SyncSender, + discontinuity: Arc, + session_started: Instant, + error_callback: E, + ) -> io::Result + where + T: cpal::SizedSample + cpal::Sample + Send + 'static, + f32: cpal::FromSample, + E: FnMut(cpal::StreamError) + Send + 'static, + { + let callback = move |data: &[T], _info: &cpal::InputCallbackInfo| { + let samples = data + .iter() + .map(|sample| sample.to_sample::()) + .collect(); + let capture_end_pts_100ns = + duration_to_100ns(Instant::now().saturating_duration_since(session_started)); + match sender.try_send(CapturedAudioChunk { + samples, + capture_end_pts_100ns, + }) { + Ok(()) => {} + Err(TrySendError::Full(_) | TrySendError::Disconnected(_)) => { + discontinuity.store(true, Ordering::Release); + } + } + }; + device + .build_input_stream(config, callback, error_callback, None) + .map_err(|error| io::Error::other(error.to_string())) + } + + pub(crate) struct AudioPipeline { + resampler: Resampler, + encoder: Encoder, + pending: Vec, + sample_count: u64, + sequence: u64, + discontinuity: bool, + timeline_base_pts_100ns: Option, + needs_reanchor: bool, + } + + pub(crate) struct EncodedAudioPacket { + pub(crate) sequence: u64, + pub(crate) pts_100ns: u64, + pub(crate) discontinuity: bool, + pub(crate) data: Vec, + } + + impl AudioPipeline { + pub(crate) fn new(source_rate: u32, source_channels: usize) -> io::Result { + let mut encoder = Encoder::new(SAMPLE_RATE, Channels::Stereo, Application::Audio) + .map_err(|error| io::Error::other(error.to_string()))?; + encoder + .set_bitrate(Bitrate::Bits(96_000)) + .map_err(|error| io::Error::other(error.to_string()))?; + encoder + .set_inband_fec(true) + .map_err(|error| io::Error::other(error.to_string()))?; + encoder + .set_packet_loss_perc(5) + .map_err(|error| io::Error::other(error.to_string()))?; + encoder + .set_dtx(true) + .map_err(|error| io::Error::other(error.to_string()))?; + Ok(Self { + resampler: Resampler::new(source_rate, source_channels)?, + encoder, + pending: Vec::with_capacity(FRAME_SAMPLES * CHANNELS * 2), + sample_count: 0, + sequence: 1, + discontinuity: false, + timeline_base_pts_100ns: None, + needs_reanchor: true, + }) + } + + pub(crate) fn mark_discontinuity(&mut self) { + self.discontinuity = true; + self.needs_reanchor = true; + self.pending.clear(); + self.resampler.reset(); + } + + pub(crate) fn push_source( + &mut self, + samples: &[f32], + capture_end_pts_100ns: u64, + ) -> io::Result> { + if !samples.len().is_multiple_of(self.resampler.source_channels) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "audio callback frame is incomplete", + )); + } + if self.needs_reanchor { + let source_frames = samples.len() / self.resampler.source_channels; + let source_duration_100ns = u64::try_from(source_frames) + .unwrap_or(u64::MAX) + .saturating_mul(10_000_000) + / u64::from(self.resampler.source_rate); + self.timeline_base_pts_100ns = + Some(capture_end_pts_100ns.saturating_sub(source_duration_100ns)); + self.sample_count = 0; + self.needs_reanchor = false; + } + self.pending.extend(self.resampler.push(samples)?); + let frame_width = FRAME_SAMPLES * CHANNELS; + let mut packets = Vec::new(); + while self.pending.len() >= frame_width { + let frame = self.pending.drain(..frame_width).collect::>(); + let data = self + .encoder + .encode_vec_float(&frame, MAX_OPUS_BYTES) + .map_err(|error| io::Error::other(error.to_string()))?; + let packet = EncodedAudioPacket { + sequence: self.sequence, + pts_100ns: self + .timeline_base_pts_100ns + .unwrap_or(capture_end_pts_100ns) + .saturating_add( + self.sample_count.saturating_mul(10_000_000) / u64::from(SAMPLE_RATE), + ), + discontinuity: self.discontinuity, + data, + }; + self.sequence = self.sequence.saturating_add(1); + self.sample_count = self.sample_count.saturating_add(FRAME_SAMPLES as u64); + self.discontinuity = false; + packets.push(packet); + } + Ok(packets) + } + } + + struct Resampler { + source_rate: u32, + source_channels: usize, + buffer: Vec, + position_numerator: u64, + } + + impl Resampler { + fn new(source_rate: u32, source_channels: usize) -> io::Result { + if source_rate == 0 || source_channels == 0 || source_channels > 8 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid source audio format", + )); + } + Ok(Self { + source_rate, + source_channels, + buffer: Vec::new(), + position_numerator: 0, + }) + } + + fn push(&mut self, samples: &[f32]) -> io::Result> { + if !samples.len().is_multiple_of(self.source_channels) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "audio callback frame is incomplete", + )); + } + self.buffer.extend_from_slice(samples); + let source_frames = self.buffer.len() / self.source_channels; + let source_frames_u64 = u64::try_from(source_frames).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidData, "audio callback is too large") + })?; + let mut output = Vec::new(); + while self.position_numerator / u64::from(SAMPLE_RATE) < source_frames_u64 { + let index_u64 = self.position_numerator / u64::from(SAMPLE_RATE); + let index = usize::try_from(index_u64).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidData, "audio position overflow") + })?; + let remainder = self.position_numerator % u64::from(SAMPLE_RATE); + if remainder != 0 && index + 1 >= source_frames { + break; + } + let fraction = f32::from(u16::try_from(remainder).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidData, "audio fraction overflow") + })?) / 48_000.0; + for channel in 0..CHANNELS { + let source_channel = if self.source_channels == 1 { + 0 + } else { + channel.min(self.source_channels - 1) + }; + let first = self.buffer[index * self.source_channels + source_channel]; + let second = if index + 1 < source_frames { + self.buffer[(index + 1) * self.source_channels + source_channel] + } else { + first + }; + output.push(first + (second - first) * fraction); + } + self.position_numerator = self + .position_numerator + .saturating_add(u64::from(self.source_rate)); + } + let consumed = usize::try_from(self.position_numerator / u64::from(SAMPLE_RATE)) + .unwrap_or(source_frames); + if consumed > 0 { + let remove = consumed.min(source_frames); + self.buffer.drain(..remove * self.source_channels); + self.position_numerator = self.position_numerator.saturating_sub( + u64::try_from(remove) + .unwrap_or(u64::MAX) + .saturating_mul(u64::from(SAMPLE_RATE)), + ); + } + Ok(output) + } + + fn reset(&mut self) { + self.buffer.clear(); + self.position_numerator = 0; + } + } + + fn duration_to_100ns(duration: std::time::Duration) -> u64 { + u64::try_from(duration.as_nanos() / 100).unwrap_or(u64::MAX) + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn resampler_preserves_48khz_stereo_frames() { + let input = vec![0.125_f32; FRAME_SAMPLES * 2]; + let output = Resampler::new(48_000, 2).unwrap().push(&input).unwrap(); + assert_eq!(output, input); + } + + #[test] + fn resampler_duplicates_mono_and_downsamples_96khz() { + let mono = vec![0.25_f32; 960]; + let mono_output = Resampler::new(48_000, 1).unwrap().push(&mono).unwrap(); + assert_eq!(mono_output.len(), 960 * 2); + assert!( + mono_output + .chunks_exact(2) + .all(|frame| frame == [0.25, 0.25]) + ); + + let stereo_96khz = vec![0.5_f32; 960 * 2]; + let downsampled = Resampler::new(96_000, 2) + .unwrap() + .push(&stereo_96khz) + .unwrap(); + assert_eq!(downsampled.len(), 480 * 2); + } + + #[test] + fn resampler_keeps_44100hz_ratio_across_callback_boundaries() { + let mut resampler = Resampler::new(44_100, 2).unwrap(); + let mut output_frames = 0; + for _ in 0..10 { + output_frames += resampler.push(&vec![0.0; 441 * 2]).unwrap().len() / 2; + } + assert!((4_799..=4_800).contains(&output_frames)); + } + + #[test] + fn opus_pipeline_emits_bounded_monotonic_twenty_ms_packets() { + let mut pipeline = AudioPipeline::new(48_000, 2).unwrap(); + let first = pipeline + .push_source(&vec![0.0; FRAME_SAMPLES * 2], 1_000_000) + .unwrap(); + let second = pipeline + .push_source(&vec![0.0; FRAME_SAMPLES * 2], 1_200_000) + .unwrap(); + assert_eq!(first.len(), 1); + assert_eq!(second.len(), 1); + assert_eq!(first[0].sequence, 1); + assert_eq!(first[0].pts_100ns, 800_000); + assert_eq!(second[0].sequence, 2); + assert_eq!(second[0].pts_100ns, 1_000_000); + assert!(!first[0].data.is_empty() && first[0].data.len() <= MAX_OPUS_BYTES); + } + + #[test] + fn discontinuity_is_applied_to_exactly_one_packet() { + let mut pipeline = AudioPipeline::new(48_000, 2).unwrap(); + pipeline.mark_discontinuity(); + let first = pipeline + .push_source(&vec![0.0; FRAME_SAMPLES * 2], 2_000_000) + .unwrap(); + let second = pipeline + .push_source(&vec![0.0; FRAME_SAMPLES * 2], 2_200_000) + .unwrap(); + assert!(first[0].discontinuity); + assert!(!second[0].discontinuity); + assert_eq!(first[0].pts_100ns, 1_800_000); + assert_eq!(second[0].pts_100ns, 2_000_000); + } + + #[test] + fn discontinuity_reanchors_pts_without_resetting_sequence() { + let mut pipeline = AudioPipeline::new(48_000, 2).unwrap(); + let first = pipeline + .push_source(&vec![0.0; FRAME_SAMPLES * 2], 200_000) + .unwrap(); + pipeline.mark_discontinuity(); + let resumed = pipeline + .push_source(&vec![0.0; FRAME_SAMPLES * 2], 5_200_000) + .unwrap(); + assert_eq!(first[0].sequence, 1); + assert_eq!(first[0].pts_100ns, 0); + assert_eq!(resumed[0].sequence, 2); + assert_eq!(resumed[0].pts_100ns, 5_000_000); + assert!(resumed[0].discontinuity); + } + } +} + +#[cfg(windows)] +pub(crate) use windows_audio::{AudioCapture, AudioPipeline, CapturedAudioChunk}; + +#[cfg(not(windows))] +pub(crate) struct AudioCapture; + +#[cfg(not(windows))] +pub(crate) struct AudioPipeline; diff --git a/agent/windows-agent/src/capability.rs b/agent/windows-agent/src/capability.rs new file mode 100644 index 0000000..45fa38e --- /dev/null +++ b/agent/windows-agent/src/capability.rs @@ -0,0 +1,166 @@ +//! Runtime capability probes. A registry/API presence check is not treated as +//! an active zero-copy encoder; it only reports whether Windows exposes an +//! approved hardware MFT that can be probed by the session worker. + +use std::sync::OnceLock; + +#[derive(Clone, Debug, Default)] +pub(crate) struct CapabilitySnapshot { + pub(crate) desktop_duplication: bool, + pub(crate) hardware_h264_mft: bool, + pub(crate) hardware_h264_encoder_verified: bool, + pub(crate) hardware_hevc_mft: bool, + pub(crate) hardware_av1_mft: bool, + pub(crate) hardware_pipeline_verified: bool, +} + +impl CapabilitySnapshot { + #[must_use] + pub(crate) const fn capture_backend(&self) -> &'static str { + if self.desktop_duplication { + "desktop-duplication-compatibility" + } else { + "gdi-bgra-zlib" + } + } + + #[must_use] + pub(crate) const fn pipeline_state(&self) -> &'static str { + if self.hardware_pipeline_verified { + "hardware_pipeline_verified" + } else if self.desktop_duplication + || self.hardware_h264_mft + || self.hardware_hevc_mft + || self.hardware_av1_mft + { + "hardware_probe_partial" + } else { + "hardware_probe_failed" + } + } + + #[must_use] + pub(crate) fn encoder_available(&self, codec: &str) -> bool { + match codec { + "h264" => self.hardware_h264_mft, + "hevc" => self.hardware_hevc_mft, + "av1" => self.hardware_av1_mft, + _ => false, + } + } +} + +static SNAPSHOT: OnceLock = OnceLock::new(); + +pub(crate) fn snapshot() -> &'static CapabilitySnapshot { + SNAPSHOT.get_or_init(probe) +} + +#[cfg(windows)] +pub(crate) fn probe() -> CapabilitySnapshot { + CapabilitySnapshot { + desktop_duplication: crate::capture::DdaCapture::new().is_ok(), + hardware_h264_mft: probe_hardware_encoder( + windows::Win32::Media::MediaFoundation::MFVideoFormat_H264, + ), + hardware_h264_encoder_verified: probe_hardware_h264_encoder(), + hardware_hevc_mft: probe_hardware_encoder( + windows::Win32::Media::MediaFoundation::MFVideoFormat_HEVC, + ), + hardware_av1_mft: probe_hardware_encoder( + windows::Win32::Media::MediaFoundation::MFVideoFormat_AV1, + ), + hardware_pipeline_verified: false, + } +} + +#[cfg(not(windows))] +pub(crate) const fn probe() -> CapabilitySnapshot { + CapabilitySnapshot { + desktop_duplication: false, + hardware_h264_mft: false, + hardware_h264_encoder_verified: false, + hardware_hevc_mft: false, + hardware_av1_mft: false, + hardware_pipeline_verified: false, + } +} + +#[cfg(windows)] +fn probe_hardware_h264_encoder() -> bool { + let Ok(capture) = crate::capture::DdaCapture::new() else { + return false; + }; + let (width, height) = capture.dimensions(); + if width < 2 || height < 2 || width % 2 != 0 || height % 2 != 0 { + return false; + } + let Ok(mut encoder) = crate::encoder::H264Encoder::new(width, height, 30, true) else { + return false; + }; + let Some(bytes) = usize::try_from(width) + .ok() + .and_then(|w| usize::try_from(height).ok().and_then(|h| w.checked_mul(h))) + .and_then(|pixels| pixels.checked_mul(4)) + else { + return false; + }; + let bgra = vec![0_u8; bytes]; + let Ok(nv12) = crate::encoder::bgra_to_nv12(width, height, &bgra) else { + return false; + }; + for _ in 0..3 { + if encoder.encode(&nv12).is_err() { + return false; + } + } + true +} + +#[cfg(windows)] +fn probe_hardware_encoder(codec: windows::core::GUID) -> bool { + use windows::Win32::Media::MediaFoundation::{ + MF_VERSION, MFSTARTUP_FULL, MFStartup, MFT_CATEGORY_VIDEO_ENCODER, MFT_ENUM_FLAG_HARDWARE, + MFT_ENUM_FLAG_SORTANDFILTER_APPROVED_ONLY, MFT_REGISTER_TYPE_INFO, MFTEnumEx, + MFVideoFormat_NV12, + }; + use windows::Win32::System::Com::{COINIT_MULTITHREADED, CoInitializeEx, CoTaskMemFree}; + + unsafe { + if CoInitializeEx(None, COINIT_MULTITHREADED).is_err() + || MFStartup(MF_VERSION, MFSTARTUP_FULL).is_err() + { + return false; + } + let input = MFT_REGISTER_TYPE_INFO { + guidMajorType: windows::Win32::Media::MediaFoundation::MFMediaType_Video, + guidSubtype: MFVideoFormat_NV12, + }; + let output = MFT_REGISTER_TYPE_INFO { + guidMajorType: windows::Win32::Media::MediaFoundation::MFMediaType_Video, + guidSubtype: codec, + }; + let mut activations: *mut Option = + std::ptr::null_mut(); + let mut count = 0_u32; + let result = MFTEnumEx( + MFT_CATEGORY_VIDEO_ENCODER, + MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_SORTANDFILTER_APPROVED_ONLY, + Some(&raw const input), + Some(&raw const output), + &raw mut activations, + &raw mut count, + ) + .is_ok() + && count > 0; + if !activations.is_null() { + let entries = std::slice::from_raw_parts_mut(activations, count as usize); + for entry in entries { + *entry = None; + } + CoTaskMemFree(Some(activations.cast())); + } + let _ = windows::Win32::Media::MediaFoundation::MFShutdown(); + result + } +} diff --git a/agent/windows-agent/src/capture.rs b/agent/windows-agent/src/capture.rs new file mode 100644 index 0000000..003d0da --- /dev/null +++ b/agent/windows-agent/src/capture.rs @@ -0,0 +1,302 @@ +//! Windows Desktop Duplication compatibility capture. +//! +//! The staging/readback path is deliberately reported as software/cpu_upload. +//! It is a compatibility path, not an end-to-end GPU zero-copy pipeline. An +//! IDD swap-chain consumer can replace this module without changing the agent +//! control protocol or frame envelope. + +#[cfg(windows)] +mod windows_capture { + use anyhow::Context as _; + use flate2::{Compression, write::ZlibEncoder}; + use std::io::Write as _; + use std::time::{Duration, Instant}; + use windows::Win32::Graphics::Direct3D::D3D_DRIVER_TYPE_UNKNOWN; + use windows::Win32::Graphics::Direct3D11::{ + D3D11_BIND_FLAG, D3D11_CPU_ACCESS_READ, D3D11_CREATE_DEVICE_FLAG, D3D11_MAP_READ, + D3D11_RESOURCE_MISC_FLAG, D3D11_TEXTURE2D_DESC, D3D11_USAGE_STAGING, D3D11CreateDevice, + ID3D11DeviceContext, ID3D11Texture2D, + }; + use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_SAMPLE_DESC}; + use windows::Win32::Graphics::Dxgi::{ + CreateDXGIFactory1, DXGI_ERROR_WAIT_TIMEOUT, DXGI_OUTDUPL_FRAME_INFO, IDXGIAdapter1, + IDXGIOutput1, IDXGIOutputDuplication, IDXGIResource, + }; + use windows::core::Interface as _; + + const FRAME_MAGIC: &[u8; 4] = b"RDWF"; + const FRAME_VERSION: u8 = 2; + const FRAME_HEADER_LEN: usize = 40; + const CODEC_BGRA_ZLIB: u8 = 1; + const MAX_DIMENSION: u32 = 16_384; + const MAX_RAW_FRAME_BYTES: usize = 256 * 1024 * 1024; + + pub struct EncodedFrame { + pub bytes: Vec, + pub capture_latency: Duration, + pub encode_latency: Duration, + } + + pub struct CapturedFrame { + pub width: u32, + pub height: u32, + pub pixels: Vec, + pub capture_latency: Duration, + } + + pub struct DdaCapture { + context: ID3D11DeviceContext, + duplication: IDXGIOutputDuplication, + staging: ID3D11Texture2D, + width: u32, + height: u32, + pixels: Vec, + } + + impl DdaCapture { + pub fn dimensions(&self) -> (u32, u32) { + (self.width, self.height) + } + + pub fn new() -> anyhow::Result { + unsafe { + let factory = CreateDXGIFactory1::() + .context("CreateDXGIFactory1 failed")?; + let mut selected: Option<( + IDXGIAdapter1, + windows::Win32::Graphics::Dxgi::IDXGIOutput, + )> = None; + for adapter_index in 0..32 { + let Ok(adapter) = factory.EnumAdapters1(adapter_index) else { + break; + }; + for output_index in 0..16 { + let Ok(output) = adapter.EnumOutputs(output_index) else { + break; + }; + let description = + output.GetDesc().context("IDXGIOutput::GetDesc failed")?; + if description.AttachedToDesktop.as_bool() { + selected = Some((adapter, output)); + break; + } + } + if selected.is_some() { + break; + } + } + let (adapter, output) = + selected.ok_or_else(|| anyhow::anyhow!("no attached DXGI output"))?; + let mut device = None; + let mut context = None; + D3D11CreateDevice( + &adapter, + D3D_DRIVER_TYPE_UNKNOWN, + windows::Win32::Foundation::HMODULE::default(), + D3D11_CREATE_DEVICE_FLAG(0), + None, + 7, + Some(&raw mut device), + None, + Some(&raw mut context), + ) + .context("D3D11CreateDevice failed")?; + let device = device.ok_or_else(|| anyhow::anyhow!("D3D11 returned no device"))?; + let context = + context.ok_or_else(|| anyhow::anyhow!("D3D11 returned no context"))?; + let output1: IDXGIOutput1 = + output.cast().context("DXGI output has no IDXGIOutput1")?; + let duplication = output1 + .DuplicateOutput(&device) + .context("DuplicateOutput failed")?; + let desc = duplication.GetDesc(); + let width = desc.ModeDesc.Width; + let height = desc.ModeDesc.Height; + validate_geometry(width, height)?; + let texture_desc = D3D11_TEXTURE2D_DESC { + Width: width, + Height: height, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_B8G8R8A8_UNORM, + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_STAGING, + BindFlags: D3D11_BIND_FLAG(0).0 as u32, + CPUAccessFlags: D3D11_CPU_ACCESS_READ.0 as u32, + MiscFlags: D3D11_RESOURCE_MISC_FLAG(0).0 as u32, + }; + let mut staging = None; + device + .CreateTexture2D(&texture_desc, None, Some(&raw mut staging)) + .context("CreateTexture2D staging resource failed")?; + let staging = + staging.ok_or_else(|| anyhow::anyhow!("D3D11 returned no staging texture"))?; + let bytes = usize::try_from(width) + .ok() + .and_then(|w| usize::try_from(height).ok().and_then(|h| w.checked_mul(h))) + .and_then(|pixels| pixels.checked_mul(4)) + .ok_or_else(|| anyhow::anyhow!("desktop frame size overflow"))?; + Ok(Self { + context, + duplication, + staging, + width, + height, + pixels: vec![0; bytes], + }) + } + } + + pub fn capture_frame(&mut self) -> anyhow::Result { + let captured = self.capture_bgra_frame()?; + let encode_started = Instant::now(); + let bytes = encode_frame( + captured.width, + captured.height, + &captured.pixels, + captured.capture_latency, + Duration::ZERO, + )?; + Ok(EncodedFrame { + bytes, + capture_latency: captured.capture_latency, + encode_latency: encode_started.elapsed(), + }) + } + + pub fn capture_bgra_frame(&mut self) -> anyhow::Result { + let capture_started = Instant::now(); + let mut info = DXGI_OUTDUPL_FRAME_INFO::default(); + let mut resource: Option = None; + unsafe { + if let Err(error) = + self.duplication + .AcquireNextFrame(100, &raw mut info, &raw mut resource) + { + if error.code() == DXGI_ERROR_WAIT_TIMEOUT.into() { + anyhow::bail!("dda_timeout"); + } + return Err(error).context("AcquireNextFrame failed"); + } + let mut mapped = false; + let result = (|| { + let resource = resource + .ok_or_else(|| anyhow::anyhow!("DDA returned no frame resource"))?; + let source: ID3D11Texture2D = resource + .cast() + .context("DDA resource is not a D3D11 texture")?; + self.context.CopyResource(&self.staging, &source); + let mut mapped_resource = + windows::Win32::Graphics::Direct3D11::D3D11_MAPPED_SUBRESOURCE::default(); + self.context + .Map( + &self.staging, + 0, + D3D11_MAP_READ, + 0, + Some(&raw mut mapped_resource), + ) + .context("Map staging texture failed")?; + mapped = true; + let row_bytes = usize::try_from(self.width).unwrap() * 4; + let pitch = usize::try_from(mapped_resource.RowPitch) + .context("invalid DDA row pitch")?; + anyhow::ensure!(pitch >= row_bytes, "DDA row pitch is shorter than a row"); + anyhow::ensure!( + !mapped_resource.pData.is_null(), + "DDA map returned a null pointer" + ); + let source_ptr = mapped_resource.pData.cast::(); + for row in 0..usize::try_from(self.height).unwrap() { + let source_row = + std::slice::from_raw_parts(source_ptr.add(row * pitch), row_bytes); + self.pixels[row * row_bytes..(row + 1) * row_bytes] + .copy_from_slice(source_row); + } + Ok::<(), anyhow::Error>(()) + })(); + if mapped { + self.context.Unmap(&self.staging, 0); + } + let _ = self.duplication.ReleaseFrame(); + result?; + } + Ok(CapturedFrame { + width: self.width, + height: self.height, + pixels: self.pixels.clone(), + capture_latency: capture_started.elapsed(), + }) + } + } + + fn validate_geometry(width: u32, height: u32) -> anyhow::Result<()> { + anyhow::ensure!( + width > 0 && height > 0 && width <= MAX_DIMENSION && height <= MAX_DIMENSION, + "unsupported DDA output {width}x{height}" + ); + let bytes = usize::try_from(width) + .ok() + .and_then(|w| usize::try_from(height).ok().and_then(|h| w.checked_mul(h))) + .and_then(|pixels| pixels.checked_mul(4)) + .ok_or_else(|| anyhow::anyhow!("DDA frame size overflow"))?; + anyhow::ensure!(bytes <= MAX_RAW_FRAME_BYTES, "DDA frame exceeds 256 MiB"); + Ok(()) + } + + fn encode_frame( + width: u32, + height: u32, + pixels: &[u8], + capture: Duration, + encode: Duration, + ) -> anyhow::Result> { + let mut compressor = ZlibEncoder::new(Vec::new(), Compression::fast()); + compressor.write_all(pixels)?; + let compressed = compressor.finish()?; + let raw_len = u32::try_from(pixels.len()).context("DDA raw frame is too large")?; + let compressed_len = + u32::try_from(compressed.len()).context("DDA compressed frame is too large")?; + let mut frame = Vec::with_capacity(FRAME_HEADER_LEN + compressed.len()); + frame.extend_from_slice(FRAME_MAGIC); + frame.push(FRAME_VERSION); + frame.push(CODEC_BGRA_ZLIB); + frame.extend_from_slice(&0_u16.to_le_bytes()); + frame.extend_from_slice(&width.to_le_bytes()); + frame.extend_from_slice(&height.to_le_bytes()); + frame.extend_from_slice(&raw_len.to_le_bytes()); + frame.extend_from_slice(&compressed_len.to_le_bytes()); + frame.extend_from_slice( + &u64::try_from(capture.as_micros()) + .unwrap_or(u64::MAX) + .to_le_bytes(), + ); + frame.extend_from_slice( + &u64::try_from(encode.as_micros()) + .unwrap_or(u64::MAX) + .to_le_bytes(), + ); + frame.extend_from_slice(&compressed); + Ok(frame) + } +} + +#[cfg(windows)] +pub(super) use windows_capture::DdaCapture; + +#[cfg(not(windows))] +pub(super) struct DdaCapture; + +#[cfg(not(windows))] +impl DdaCapture { + pub(super) fn dimensions(&self) -> (u32, u32) { + (0, 0) + } + + pub(super) fn new() -> anyhow::Result { + anyhow::bail!("desktop_duplication_windows_only") + } +} diff --git a/agent/windows-agent/src/encoder.rs b/agent/windows-agent/src/encoder.rs new file mode 100644 index 0000000..520b6a4 --- /dev/null +++ b/agent/windows-agent/src/encoder.rs @@ -0,0 +1,657 @@ +//! Windows H.264 encoding and CPU-side compatibility conversion. +//! +//! The encoder accepts an NV12 frame and returns one bounded Annex-B access +//! unit. The current Desktop Duplication path is a CPU readback, so callers +//! must report that upload/copy boundary separately; this module does not +//! claim end-to-end zero-copy merely because the MFT is hardware-backed. + +use std::time::Duration; + +const MAX_DIMENSION: u32 = 16_384; +const MAX_ACCESS_UNIT_BYTES: usize = 16 * 1024 * 1024; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct EncodedAccessUnit { + pub(crate) data: Vec, + pub(crate) pts_100ns: i64, + pub(crate) duration: Duration, + pub(crate) encode_latency: Duration, + pub(crate) keyframe: bool, + pub(crate) encoder: &'static str, + pub(crate) hardware: bool, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(crate) enum EncoderError { + InvalidConfiguration, + InvalidFrame, + RuntimeUnavailable, + EncoderUnavailable, + InputRejected, + OutputUnavailable, + InvalidOutput, + KeyFrameUnavailable, +} + +impl std::fmt::Display for EncoderError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(match self { + Self::InvalidConfiguration => "invalid_encoder_configuration", + Self::InvalidFrame => "invalid_nv12_frame", + Self::RuntimeUnavailable => "media_foundation_unavailable", + Self::EncoderUnavailable => "h264_encoder_unavailable", + Self::InputRejected => "h264_encoder_input_rejected", + Self::OutputUnavailable => "h264_encoder_output_unavailable", + Self::InvalidOutput => "invalid_h264_access_unit", + Self::KeyFrameUnavailable => "h264_keyframe_request_unavailable", + }) + } +} + +impl std::error::Error for EncoderError {} + +pub(crate) fn validate_configuration( + width: u32, + height: u32, + fps: u32, +) -> Result<(), EncoderError> { + if !(2..=MAX_DIMENSION).contains(&width) + || !(2..=MAX_DIMENSION).contains(&height) + || fps == 0 + || fps > 240 + || width % 2 != 0 + || height % 2 != 0 + { + return Err(EncoderError::InvalidConfiguration); + } + Ok(()) +} + +pub(crate) fn expected_nv12_bytes(width: u32, height: u32) -> Option { + usize::try_from(width) + .ok() + .and_then(|width| { + usize::try_from(height) + .ok() + .and_then(|height| width.checked_mul(height)) + }) + .and_then(|y| y.checked_add(y / 2)) +} + +/// Converts top-down BGRA8 into an 8-bit SDR NV12 frame. +pub(crate) fn bgra_to_nv12(width: u32, height: u32, bgra: &[u8]) -> Result, EncoderError> { + validate_configuration(width, height, 1)?; + let expected = expected_nv12_bytes(width, height).ok_or(EncoderError::InvalidFrame)?; + let bgra_len = usize::try_from(width) + .ok() + .and_then(|width| { + usize::try_from(height) + .ok() + .and_then(|height| width.checked_mul(height)) + }) + .and_then(|pixels| pixels.checked_mul(4)) + .ok_or(EncoderError::InvalidFrame)?; + if bgra.len() != bgra_len { + return Err(EncoderError::InvalidFrame); + } + let width = usize::try_from(width).map_err(|_| EncoderError::InvalidFrame)?; + let height = usize::try_from(height).map_err(|_| EncoderError::InvalidFrame)?; + let y_plane_len = width + .checked_mul(height) + .ok_or(EncoderError::InvalidFrame)?; + let mut output = vec![0_u8; expected]; + let (y_plane, uv_plane) = output.split_at_mut(y_plane_len); + + for y in 0..height { + for x in 0..width { + let source = (y * width + x) * 4; + let b = i32::from(bgra[source]); + let g = i32::from(bgra[source + 1]); + let r = i32::from(bgra[source + 2]); + y_plane[y * width + x] = clamp_u8((66 * r + 129 * g + 25 * b + 128) / 256 + 16); + } + } + for y in (0..height).step_by(2) { + for x in (0..width).step_by(2) { + let mut u = 0_i32; + let mut v = 0_i32; + for dy in 0..2 { + for dx in 0..2 { + let source = ((y + dy) * width + x + dx) * 4; + let b = i32::from(bgra[source]); + let g = i32::from(bgra[source + 1]); + let r = i32::from(bgra[source + 2]); + u += (-38 * r - 74 * g + 112 * b + 512) / 1024 + 128; + v += (112 * r - 94 * g - 18 * b + 512) / 1024 + 128; + } + } + let destination = (y / 2) * width + x; + uv_plane[destination] = clamp_u8(u / 4); + uv_plane[destination + 1] = clamp_u8(v / 4); + } + } + Ok(output) +} + +fn clamp_u8(value: i32) -> u8 { + value.clamp(0, 255) as u8 +} + +pub(crate) fn annex_b_contains_idr(data: &[u8]) -> bool { + let mut index = 0; + while index + 4 <= data.len() { + let start = if data[index..].starts_with(&[0, 0, 0, 1]) { + index + 4 + } else if index + 3 <= data.len() && data[index..].starts_with(&[0, 0, 1]) { + index + 3 + } else { + index += 1; + continue; + }; + if start < data.len() && data[start] & 0x1f == 5 { + return true; + } + index = start; + } + false +} + +/// Media Foundation may expose H.264 samples as byte-stream Annex-B or as +/// AVC length-prefixed NAL units. Normalize both forms before the access unit +/// crosses the process boundary. +pub(crate) fn normalize_h264_access_unit(data: &[u8]) -> Result, EncoderError> { + if data.is_empty() { + return Err(EncoderError::InvalidOutput); + } + if data.starts_with(&[0, 0, 0, 1]) || data.starts_with(&[0, 0, 1]) { + return Ok(data.to_vec()); + } + let mut offset = 0_usize; + let mut output = Vec::with_capacity(data.len() + 16); + while offset + 4 <= data.len() { + let length = u32::from_be_bytes(data[offset..offset + 4].try_into().unwrap()) as usize; + offset += 4; + if length == 0 || length > data.len().saturating_sub(offset) { + return Err(EncoderError::InvalidOutput); + } + output.extend_from_slice(&[0, 0, 0, 1]); + output.extend_from_slice(&data[offset..offset + length]); + offset += length; + } + if offset != data.len() || output.is_empty() { + return Err(EncoderError::InvalidOutput); + } + Ok(output) +} + +#[cfg(windows)] +mod windows_mft { + use super::{ + EncodedAccessUnit, EncoderError, MAX_ACCESS_UNIT_BYTES, annex_b_contains_idr, + validate_configuration, + }; + use core::mem::ManuallyDrop; + use std::collections::VecDeque; + use std::time::{Duration, Instant}; + use windows::Win32::Media::MediaFoundation::{ + CLSID_MSH264EncoderMFT, CODECAPI_AVEncVideoForceKeyFrame, ICodecAPI, IMFActivate, + IMFMediaType, IMFTransform, MF_E_NOTACCEPTING, MF_E_TRANSFORM_NEED_MORE_INPUT, + MF_MT_AVG_BITRATE, MF_MT_FRAME_RATE, MF_MT_FRAME_SIZE, MF_MT_INTERLACE_MODE, + MF_MT_MAJOR_TYPE, MF_MT_SUBTYPE, MF_VERSION, MFCreateMediaType, MFCreateMemoryBuffer, + MFCreateSample, MFMediaType_Video, MFSTARTUP_FULL, MFShutdown, MFStartup, + MFT_CATEGORY_VIDEO_ENCODER, MFT_ENUM_FLAG_HARDWARE, + MFT_ENUM_FLAG_SORTANDFILTER_APPROVED_ONLY, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, + MFT_MESSAGE_NOTIFY_START_OF_STREAM, MFT_OUTPUT_DATA_BUFFER, MFT_REGISTER_TYPE_INFO, + MFVideoFormat_H264, MFVideoFormat_NV12, MFVideoInterlace_Progressive, + }; + use windows::Win32::System::Com::{ + CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED, CoCreateInstance, CoInitializeEx, CoTaskMemFree, + }; + use windows::Win32::System::Variant::VARIANT; + use windows::core::Interface as _; + + const MAX_OUTPUTS_PER_INPUT: usize = 8; + const MAX_PENDING_ACCESS_UNITS: usize = 16; + + struct ComGuard; + impl Drop for ComGuard { + fn drop(&mut self) { + unsafe { windows::Win32::System::Com::CoUninitialize() }; + } + } + + struct MfGuard; + impl Drop for MfGuard { + fn drop(&mut self) { + unsafe { + let _ = MFShutdown(); + } + } + } + + pub(crate) struct H264Encoder { + transform: IMFTransform, + width: u32, + height: u32, + fps: u32, + next_pts_100ns: i64, + last_explicit_pts_100ns: Option, + encoder: &'static str, + hardware: bool, + pending: VecDeque, + _com: ComGuard, + _mf: MfGuard, + } + + impl H264Encoder { + pub(crate) fn new( + width: u32, + height: u32, + fps: u32, + hardware_only: bool, + ) -> Result { + validate_configuration(width, height, fps)?; + let com_result = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) }; + if com_result.is_err() && com_result != windows::Win32::Foundation::S_FALSE { + return Err(EncoderError::RuntimeUnavailable); + } + let com = ComGuard; + unsafe { MFStartup(MF_VERSION, MFSTARTUP_FULL) } + .map_err(|_| EncoderError::RuntimeUnavailable)?; + let mf = MfGuard; + let (transform, encoder, hardware) = if hardware_only { + activate_hardware_encoder()?.ok_or(EncoderError::EncoderUnavailable)? + } else { + let transform: IMFTransform = unsafe { + CoCreateInstance(&CLSID_MSH264EncoderMFT, None, CLSCTX_INPROC_SERVER) + } + .map_err(|_| EncoderError::EncoderUnavailable)?; + (transform, "mfh264enc", false) + }; + configure_transform(&transform, width, height, fps)?; + unsafe { transform.ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0) } + .map_err(|_| EncoderError::EncoderUnavailable)?; + unsafe { transform.ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0) } + .map_err(|_| EncoderError::EncoderUnavailable)?; + Ok(Self { + transform, + width, + height, + fps, + next_pts_100ns: 0, + last_explicit_pts_100ns: None, + encoder, + hardware, + pending: VecDeque::new(), + _com: com, + _mf: mf, + }) + } + + pub(crate) fn encode( + &mut self, + nv12: &[u8], + ) -> Result, EncoderError> { + let duration = Duration::from_nanos(1_000_000_000_u64 / u64::from(self.fps)); + let timestamp = self.next_pts_100ns; + self.next_pts_100ns = self + .next_pts_100ns + .saturating_add(i64::try_from(duration.as_nanos() / 100).unwrap_or(i64::MAX)); + self.encode_timestamped(nv12, timestamp, duration) + } + + pub(crate) fn encode_at( + &mut self, + nv12: &[u8], + requested_pts_100ns: i64, + ) -> Result, EncoderError> { + if requested_pts_100ns < 0 { + return Err(EncoderError::InvalidFrame); + } + let timestamp = self + .last_explicit_pts_100ns + .map_or(requested_pts_100ns, |last| { + requested_pts_100ns.max(last.saturating_add(1)) + }); + self.last_explicit_pts_100ns = Some(timestamp); + let duration = Duration::from_nanos(1_000_000_000_u64 / u64::from(self.fps)); + self.encode_timestamped(nv12, timestamp, duration) + } + + fn encode_timestamped( + &mut self, + nv12: &[u8], + timestamp: i64, + duration: Duration, + ) -> Result, EncoderError> { + let started = Instant::now(); + let expected = super::expected_nv12_bytes(self.width, self.height) + .ok_or(EncoderError::InvalidFrame)?; + if nv12.len() != expected { + return Err(EncoderError::InvalidFrame); + } + let sample = create_sample(nv12, timestamp, duration)?; + let input = unsafe { self.transform.ProcessInput(0, &sample, 0) }; + if let Err(error) = input { + if error.code() != MF_E_NOTACCEPTING { + return Err(EncoderError::InputRejected); + } + let outputs = self.drain_outputs(started)?; + self.queue_outputs(outputs)?; + unsafe { self.transform.ProcessInput(0, &sample, 0) } + .map_err(|_| EncoderError::InputRejected)?; + } + let outputs = self.drain_outputs(started)?; + self.queue_outputs(outputs)?; + Ok(self.pending.pop_front()) + } + + pub(crate) fn request_keyframe(&self) -> Result<(), EncoderError> { + let codec_api = self + .transform + .cast::() + .map_err(|_| EncoderError::KeyFrameUnavailable)?; + let value = VARIANT::from(true); + let force_keyframe = CODECAPI_AVEncVideoForceKeyFrame; + unsafe { codec_api.SetValue(&raw const force_keyframe, &raw const value) } + .map_err(|_| EncoderError::KeyFrameUnavailable) + } + + fn queue_outputs(&mut self, outputs: Vec) -> Result<(), EncoderError> { + if self.pending.len().saturating_add(outputs.len()) > MAX_PENDING_ACCESS_UNITS { + return Err(EncoderError::OutputUnavailable); + } + self.pending.extend(outputs); + Ok(()) + } + + fn drain_outputs( + &mut self, + started: Instant, + ) -> Result, EncoderError> { + let mut access_units = Vec::new(); + for _ in 0..MAX_OUTPUTS_PER_INPUT { + let mut output = MFT_OUTPUT_DATA_BUFFER::default(); + output.pSample = ManuallyDrop::new(None); + output.pEvents = ManuallyDrop::new(None); + let mut status = 0_u32; + let result = unsafe { + self.transform.ProcessOutput( + 0, + std::slice::from_mut(&mut output), + &raw mut status, + ) + }; + let sample = unsafe { ManuallyDrop::take(&mut output.pSample) }; + let _events = unsafe { ManuallyDrop::take(&mut output.pEvents) }; + match result { + Ok(()) => { + let sample = sample.ok_or(EncoderError::OutputUnavailable)?; + access_units.push(self.access_unit_from_sample(sample, started.elapsed())?); + } + Err(error) if error.code() == MF_E_TRANSFORM_NEED_MORE_INPUT => break, + Err(_) => return Err(EncoderError::OutputUnavailable), + } + } + Ok(access_units) + } + + fn access_unit_from_sample( + &self, + sample: windows::Win32::Media::MediaFoundation::IMFSample, + encode_latency: Duration, + ) -> Result { + let buffer_count = + unsafe { sample.GetBufferCount() }.map_err(|_| EncoderError::InvalidOutput)?; + if buffer_count != 1 { + return Err(EncoderError::InvalidOutput); + } + let buffer = + unsafe { sample.GetBufferByIndex(0) }.map_err(|_| EncoderError::InvalidOutput)?; + let current_length = + unsafe { buffer.GetCurrentLength() }.map_err(|_| EncoderError::InvalidOutput)?; + let mut data = vec![0_u8; MAX_ACCESS_UNIT_BYTES.min(current_length as usize)]; + let mut length = 0_u32; + let mut locked = core::ptr::null_mut(); + unsafe { buffer.Lock(&raw mut locked, None, Some(&raw mut length)) } + .map_err(|_| EncoderError::InvalidOutput)?; + let result = if locked.is_null() + || usize::try_from(length).unwrap_or(usize::MAX) > MAX_ACCESS_UNIT_BYTES + { + Err(EncoderError::InvalidOutput) + } else { + data.resize(length as usize, 0); + unsafe { + core::ptr::copy_nonoverlapping( + locked.cast::(), + data.as_mut_ptr(), + data.len(), + ) + }; + Ok(()) + }; + let _ = unsafe { buffer.Unlock() }; + result?; + if data.is_empty() { + return Err(EncoderError::InvalidOutput); + } + let data = super::normalize_h264_access_unit(&data)?; + let keyframe = annex_b_contains_idr(&data); + let pts_100ns = unsafe { sample.GetSampleTime() }.unwrap_or(0); + Ok(EncodedAccessUnit { + data, + pts_100ns, + duration: Duration::from_nanos(1_000_000_000 / u64::from(self.fps)), + encode_latency, + keyframe, + encoder: self.encoder, + hardware: self.hardware, + }) + } + } + + fn activate_hardware_encoder() + -> Result, EncoderError> { + let input = MFT_REGISTER_TYPE_INFO { + guidMajorType: MFMediaType_Video, + guidSubtype: MFVideoFormat_NV12, + }; + let output = MFT_REGISTER_TYPE_INFO { + guidMajorType: MFMediaType_Video, + guidSubtype: MFVideoFormat_H264, + }; + let mut activations: *mut Option = core::ptr::null_mut(); + let mut count = 0_u32; + let result = unsafe { + windows::Win32::Media::MediaFoundation::MFTEnumEx( + MFT_CATEGORY_VIDEO_ENCODER, + MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_SORTANDFILTER_APPROVED_ONLY, + Some(&raw const input), + Some(&raw const output), + &raw mut activations, + &raw mut count, + ) + }; + if result.is_err() || count == 0 { + free_activations(activations, count); + return Ok(None); + } + let transform = unsafe { + (*activations) + .as_ref() + .ok_or(EncoderError::EncoderUnavailable)? + .ActivateObject::() + }; + free_activations(activations, count); + Ok(Some(( + transform.map_err(|_| EncoderError::EncoderUnavailable)?, + "hardware-mft", + true, + ))) + } + + fn free_activations(pointer: *mut Option, count: u32) { + if pointer.is_null() { + return; + } + unsafe { + for item in std::slice::from_raw_parts_mut(pointer, count as usize) { + *item = None; + } + CoTaskMemFree(Some(pointer.cast())); + } + } + + fn configure_transform( + transform: &IMFTransform, + width: u32, + height: u32, + fps: u32, + ) -> Result<(), EncoderError> { + let input = media_type(MFVideoFormat_NV12, width, height, fps, None)?; + let output = media_type( + MFVideoFormat_H264, + width, + height, + fps, + Some(bitrate(width, height, fps)), + )?; + unsafe { transform.SetInputType(0, &input, 0) } + .map_err(|_| EncoderError::EncoderUnavailable)?; + unsafe { transform.SetOutputType(0, &output, 0) } + .map_err(|_| EncoderError::EncoderUnavailable)?; + Ok(()) + } + + fn media_type( + subtype: windows::core::GUID, + width: u32, + height: u32, + fps: u32, + bitrate_value: Option, + ) -> Result { + let media_type = + unsafe { MFCreateMediaType() }.map_err(|_| EncoderError::RuntimeUnavailable)?; + unsafe { media_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video) } + .map_err(|_| EncoderError::EncoderUnavailable)?; + unsafe { media_type.SetGUID(&MF_MT_SUBTYPE, &subtype) } + .map_err(|_| EncoderError::EncoderUnavailable)?; + unsafe { + media_type.SetUINT64( + &MF_MT_FRAME_SIZE, + (u64::from(width) << 32) | u64::from(height), + ) + } + .map_err(|_| EncoderError::EncoderUnavailable)?; + unsafe { media_type.SetUINT64(&MF_MT_FRAME_RATE, (u64::from(fps) << 32) | 1) } + .map_err(|_| EncoderError::EncoderUnavailable)?; + unsafe { + media_type.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32) + } + .map_err(|_| EncoderError::EncoderUnavailable)?; + if let Some(value) = bitrate_value { + unsafe { media_type.SetUINT32(&MF_MT_AVG_BITRATE, value) } + .map_err(|_| EncoderError::EncoderUnavailable)?; + } + Ok(media_type) + } + + fn bitrate(width: u32, height: u32, fps: u32) -> u32 { + let value = u64::from(width) + .saturating_mul(u64::from(height)) + .saturating_mul(u64::from(fps)) + .saturating_mul(2) + .saturating_div(100); + u32::try_from(value.clamp(2_000_000, 80_000_000)).unwrap_or(80_000_000) + } + + fn create_sample( + data: &[u8], + pts: i64, + duration: Duration, + ) -> Result { + let buffer = unsafe { + MFCreateMemoryBuffer(u32::try_from(data.len()).map_err(|_| EncoderError::InvalidFrame)?) + } + .map_err(|_| EncoderError::RuntimeUnavailable)?; + let mut destination = core::ptr::null_mut(); + unsafe { buffer.Lock(&raw mut destination, None, None) } + .map_err(|_| EncoderError::InvalidFrame)?; + if destination.is_null() { + let _ = unsafe { buffer.Unlock() }; + return Err(EncoderError::InvalidFrame); + } + unsafe { + core::ptr::copy_nonoverlapping(data.as_ptr(), destination.cast::(), data.len()); + } + unsafe { buffer.Unlock() }.map_err(|_| EncoderError::InvalidFrame)?; + unsafe { + buffer.SetCurrentLength( + u32::try_from(data.len()).map_err(|_| EncoderError::InvalidFrame)?, + ) + } + .map_err(|_| EncoderError::InvalidFrame)?; + let sample = unsafe { MFCreateSample() }.map_err(|_| EncoderError::RuntimeUnavailable)?; + unsafe { sample.AddBuffer(&buffer) }.map_err(|_| EncoderError::InvalidFrame)?; + unsafe { sample.SetSampleTime(pts) }.map_err(|_| EncoderError::InvalidFrame)?; + unsafe { + sample.SetSampleDuration(i64::try_from(duration.as_nanos() / 100).unwrap_or(i64::MAX)) + } + .map_err(|_| EncoderError::InvalidFrame)?; + Ok(sample) + } +} + +#[cfg(windows)] +#[allow(unused_imports)] +pub(crate) use windows_mft::H264Encoder; + +#[cfg(not(windows))] +pub(crate) struct H264Encoder; + +#[cfg(not(windows))] +impl H264Encoder { + pub(crate) fn new(_: u32, _: u32, _: u32, _: bool) -> Result { + Err(EncoderError::RuntimeUnavailable) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bgra_to_nv12_has_expected_layout() { + let bgra = vec![ + 0_u8, 0, 255, 255, 0, 255, 0, 255, 255, 0, 0, 255, 255, 255, 255, 255, + ]; + let nv12 = bgra_to_nv12(2, 2, &bgra).unwrap(); + assert_eq!(nv12.len(), 6); + assert!(nv12[..4].iter().all(|value| *value > 0)); + assert_eq!(nv12[4..].len(), 2); + } + + #[test] + fn odd_dimensions_and_wrong_lengths_fail_closed() { + assert_eq!( + validate_configuration(3, 2, 30), + Err(EncoderError::InvalidConfiguration) + ); + assert_eq!(bgra_to_nv12(2, 2, &[0; 3]), Err(EncoderError::InvalidFrame)); + } + + #[test] + fn idr_detection_accepts_annex_b_five() { + assert!(annex_b_contains_idr(&[0, 0, 0, 1, 5, 1])); + assert!(!annex_b_contains_idr(&[0, 0, 0, 1, 1, 1])); + } + + #[test] + fn avc_length_prefixed_output_is_normalized_to_annex_b() { + let avc = [0_u8, 0, 0, 2, 5, 1, 0, 0, 0, 1, 1]; + assert_eq!( + normalize_h264_access_unit(&avc).unwrap(), + [0, 0, 0, 1, 5, 1, 0, 0, 0, 1, 1] + ); + } +} diff --git a/agent/windows-agent/src/local_auth.rs b/agent/windows-agent/src/local_auth.rs new file mode 100644 index 0000000..65fc65a --- /dev/null +++ b/agent/windows-agent/src/local_auth.rs @@ -0,0 +1,186 @@ +use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; +use hmac::{Hmac, KeyInit as _, Mac as _}; +use serde::{Deserialize, Serialize}; +use sha2::Sha256; +use std::io; +use tokio::io::{ + AsyncBufReadExt as _, AsyncRead, AsyncReadExt as _, AsyncWrite, AsyncWriteExt as _, BufReader, +}; +use zeroize::{Zeroize as _, Zeroizing}; + +pub const AGENT_PIPE_TOKEN_ENV: &str = "REMOTEDESK_AGENT_PIPE_TOKEN"; +const AUTH_PROTOCOL_VERSION: u8 = 1; +const AUTH_KEY_BYTES: usize = 32; +const AUTH_NONCE_BYTES: usize = 32; +const AUTH_RESPONSE_MAX_BYTES: usize = 2 * 1024; +const AUTH_DOMAIN: &[u8] = b"RemoteDesk Windows Agent pipe auth v1\0"; + +type HmacSha256 = Hmac; + +pub struct PipeAuthenticator { + key: Zeroizing>, +} + +#[derive(Serialize)] +struct Challenge<'a> { + kind: &'a str, + protocol_version: u8, + nonce: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Authentication { + kind: String, + protocol_version: u8, + mac: String, +} + +impl PipeAuthenticator { + pub fn from_environment() -> io::Result { + let encoded = std::env::var(AGENT_PIPE_TOKEN_ENV).map_err(|_| { + io::Error::new( + io::ErrorKind::PermissionDenied, + format!("{AGENT_PIPE_TOKEN_ENV} is required for authenticated Named Pipe mode"), + ) + })?; + // The bootstrap is needed only while constructing the in-memory key. + unsafe { std::env::remove_var(AGENT_PIPE_TOKEN_ENV) }; + let mut encoded = Zeroizing::new(encoded); + let result = Self::from_encoded(&encoded); + encoded.zeroize(); + result + } + + fn from_encoded(encoded: &str) -> io::Result { + let key = Zeroizing::new(URL_SAFE_NO_PAD.decode(encoded).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "Agent Pipe token is not base64url", + ) + })?); + if key.len() != AUTH_KEY_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Agent Pipe token must decode to exactly 32 bytes", + )); + } + Ok(Self { key }) + } + + pub async fn authenticate(&self, stream: &mut S) -> io::Result<()> + where + S: AsyncRead + AsyncWrite + Unpin, + { + let mut nonce = [0_u8; AUTH_NONCE_BYTES]; + getrandom::fill(&mut nonce).map_err(io::Error::other)?; + let challenge = serde_json::to_vec(&Challenge { + kind: "agent_auth_challenge", + protocol_version: AUTH_PROTOCOL_VERSION, + nonce: URL_SAFE_NO_PAD.encode(nonce), + }) + .map_err(io::Error::other)?; + stream.write_all(&challenge).await?; + stream.write_all(b"\n").await?; + stream.flush().await?; + + let mut line = String::new(); + let mut reader = BufReader::new(stream).take((AUTH_RESPONSE_MAX_BYTES + 1) as u64); + let bytes = reader.read_line(&mut line).await?; + if bytes == 0 + || bytes > AUTH_RESPONSE_MAX_BYTES + || !line.ends_with('\n') + || !line.is_ascii() + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Agent Pipe authentication response is invalid", + )); + } + let authentication: Authentication = serde_json::from_str(&line) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid Agent Pipe auth"))?; + if authentication.kind != "agent_auth" + || authentication.protocol_version != AUTH_PROTOCOL_VERSION + { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "Agent Pipe authentication protocol mismatch", + )); + } + let supplied = URL_SAFE_NO_PAD.decode(authentication.mac).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidData, "invalid Agent Pipe auth MAC") + })?; + let mut mac = HmacSha256::new_from_slice(&self.key) + .map_err(|_| io::Error::other("invalid Agent Pipe HMAC key"))?; + mac.update(AUTH_DOMAIN); + mac.update(&nonce); + mac.verify_slice(&supplied).map_err(|_| { + io::Error::new( + io::ErrorKind::PermissionDenied, + "Agent Pipe authentication was rejected", + ) + }) + } +} + +#[cfg(test)] +fn authentication_mac(key: &[u8], nonce: &[u8]) -> io::Result> { + let mut mac = HmacSha256::new_from_slice(key) + .map_err(|_| io::Error::other("invalid Agent Pipe HMAC key"))?; + mac.update(AUTH_DOMAIN); + mac.update(nonce); + Ok(mac.finalize().into_bytes().to_vec()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::Value; + + #[test] + fn token_requires_exactly_32_base64url_bytes() { + assert!(PipeAuthenticator::from_encoded("not-base64!").is_err()); + assert!(PipeAuthenticator::from_encoded(&URL_SAFE_NO_PAD.encode([7_u8; 31])).is_err()); + assert!(PipeAuthenticator::from_encoded(&URL_SAFE_NO_PAD.encode([7_u8; 32])).is_ok()); + } + + #[test] + fn authentication_mac_matches_go_wire_vector() { + let key = (0_u8..32).collect::>(); + let nonce = (32_u8..64).collect::>(); + let mac = authentication_mac(&key, &nonce).unwrap(); + assert_eq!( + URL_SAFE_NO_PAD.encode(mac), + "KinBDOBvVhz0eRv37iahq8BO0z8QdiI0Qnx7HZmcxIk" + ); + } + + #[tokio::test] + async fn challenge_accepts_the_matching_response() { + let key = [9_u8; 32]; + let authenticator = PipeAuthenticator::from_encoded(&URL_SAFE_NO_PAD.encode(key)).unwrap(); + let (mut server, client) = tokio::io::duplex(4 * 1024); + let server_task = + tokio::spawn(async move { authenticator.authenticate(&mut server).await }); + let (reader, mut writer) = tokio::io::split(client); + let mut reader = BufReader::new(reader); + let mut line = String::new(); + reader.read_line(&mut line).await.unwrap(); + let challenge: Value = serde_json::from_str(&line).unwrap(); + let nonce = URL_SAFE_NO_PAD + .decode(challenge["nonce"].as_str().unwrap()) + .unwrap(); + let mac = authentication_mac(&key, &nonce).unwrap(); + let response = serde_json::json!({ + "kind": "agent_auth", + "protocol_version": 1, + "mac": URL_SAFE_NO_PAD.encode(mac) + }); + writer + .write_all(response.to_string().as_bytes()) + .await + .unwrap(); + writer.write_all(b"\n").await.unwrap(); + server_task.await.unwrap().unwrap(); + } +} diff --git a/agent/windows-agent/src/main.rs b/agent/windows-agent/src/main.rs index 7e1953a..25d6731 100644 --- a/agent/windows-agent/src/main.rs +++ b/agent/windows-agent/src/main.rs @@ -1,21 +1,40 @@ use clap::{Parser, Subcommand}; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use std::net::SocketAddr; +#[cfg(windows)] +use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::Duration; -use tokio::io::{AsyncBufReadExt as _, AsyncWriteExt as _, BufReader}; -use tokio::net::{TcpListener, TcpStream}; +use std::sync::mpsc as std_mpsc; +use std::sync::{Mutex, OnceLock}; +use std::time::{Duration, Instant}; +use tokio::io::{AsyncBufReadExt as _, AsyncRead, AsyncWrite, AsyncWriteExt as _, BufReader}; +use tokio::net::TcpListener; mod applications; +mod audio; +mod capability; +mod capture; +mod encoder; +mod local_auth; +mod media_ring; +#[cfg(windows)] +mod windows_security; -const PREFERRED_CAPTURE_BACKEND: &str = "dxgi-desktop-duplication-strict"; +const PREFERRED_CAPTURE_BACKEND: &str = "idd-headless-sdr"; const SOFTWARE_FALLBACK_BACKEND: &str = "gdi-bgra-zlib"; +const CURRENT_MEDIA_TRANSPORT: &str = "rdwf_tcp_compatibility"; +const TARGET_MEDIA_TRANSPORT: &str = "hysteria2_video_datagram_plus_encoded_ring"; const DEFAULT_FALLBACK_FPS: u8 = 10; const MAX_FALLBACK_FPS: u8 = 15; +const DEFAULT_VIDEO_CODEC: &str = "h264"; +const DEFAULT_CAPTURE_MODE: &str = "strict_hardware"; const NO_LATENCY_SAMPLE: u64 = u64::MAX; static CAPTURE_LATENCY_US: AtomicU64 = AtomicU64::new(NO_LATENCY_SAMPLE); static ENCODE_LATENCY_US: AtomicU64 = AtomicU64::new(NO_LATENCY_SAMPLE); static FRAME_PROCESSING_LATENCY_US: AtomicU64 = AtomicU64::new(NO_LATENCY_SAMPLE); +static ENCODED_GENERATION: AtomicU64 = AtomicU64::new(1); +static ENCODED_SESSIONS: OnceLock>> = OnceLock::new(); #[derive(Debug, Parser)] #[command( @@ -38,6 +57,12 @@ enum Command { #[arg(long, default_value = "0.0.0.0:39501")] listen: SocketAddr, }, + Pipe { + #[arg(long, default_value = r"\\.\pipe\RemoteDesk\windows-agent")] + name: String, + #[arg(long)] + allow_unauthenticated: bool, + }, } #[derive(Debug, Serialize)] @@ -46,34 +71,72 @@ struct Status { platform: &'static str, transport: &'static str, capture_backend: &'static str, + compatibility_capture_backend: &'static str, + video_codecs: [&'static str; 3], + audio_codec: &'static str, + media_transport: &'static str, input_backend: &'static str, session_runtime: &'static str, strict_gpu_pipeline: &'static str, software_fallback_backend: &'static str, software_fallback_available: bool, + media_pipeline_state: &'static str, + supported_capture_modes: [&'static str; 2], + desktop_duplication: bool, + hardware_h264_mft: bool, + hardware_h264_encoder_verified: bool, + hardware_hevc_mft: bool, + hardware_av1_mft: bool, + hardware_pipeline_verified: bool, + degraded: bool, } fn main() -> anyhow::Result<()> { let cli = Cli::parse(); match cli.command { Command::Status { json } => { + let capabilities = capability::snapshot(); let status = Status { product: "RemoteDesk Windows Agent", platform: "windows", transport: "native-agent", - capture_backend: PREFERRED_CAPTURE_BACKEND, + capture_backend: capabilities.capture_backend(), + compatibility_capture_backend: capabilities.capture_backend(), + video_codecs: ["h264", "hevc", "av1"], + audio_codec: "opus", + media_transport: CURRENT_MEDIA_TRANSPORT, input_backend: "windows-send-input", session_runtime: "native runtime not started", strict_gpu_pipeline: "not_implemented", software_fallback_backend: SOFTWARE_FALLBACK_BACKEND, software_fallback_available: cfg!(windows), + media_pipeline_state: capabilities.pipeline_state(), + supported_capture_modes: ["strict_hardware", "compatibility"], + desktop_duplication: capabilities.desktop_duplication, + hardware_h264_mft: capabilities.hardware_h264_mft, + hardware_h264_encoder_verified: capabilities.hardware_h264_encoder_verified, + hardware_hevc_mft: capabilities.hardware_hevc_mft, + hardware_av1_mft: capabilities.hardware_av1_mft, + hardware_pipeline_verified: capabilities.hardware_pipeline_verified, + degraded: !capabilities.hardware_pipeline_verified, }; if json { println!("{}", serde_json::to_string_pretty(&status)?); } else { println!("RemoteDesk Windows Agent"); println!("Transport: native Agent protocol"); - println!("Capture: strict DXGI Desktop Duplication / D3D11"); + println!("Capture: IDD Headless SDR (driver integration pending)"); + println!("Compatibility capture: {}", capabilities.capture_backend()); + println!( + "Video codecs: H.264={} HEVC={} AV1={}", + capabilities.hardware_h264_mft, + capabilities.hardware_hevc_mft, + capabilities.hardware_av1_mft + ); + println!("Audio: Opus (WASAPI capture pending)"); + println!( + "Media transport: {CURRENT_MEDIA_TRANSPORT} (target: {TARGET_MEDIA_TRANSPORT})" + ); println!("Input: SendInput"); println!("Session runtime: not started"); println!("Software fallback: GDI / BGRA / zlib available"); @@ -81,6 +144,10 @@ fn main() -> anyhow::Result<()> { } Command::Run => run_server("0.0.0.0:39501".parse()?)?, Command::Serve { listen } => run_server(listen)?, + Command::Pipe { + name, + allow_unauthenticated, + } => run_pipe_server(name, allow_unauthenticated)?, } Ok(()) } @@ -98,7 +165,7 @@ fn run_server(listen: SocketAddr) -> anyhow::Result<()> { loop { let (stream, peer) = listener.accept().await?; tokio::spawn(async move { - if let Err(error) = handle_client(stream).await { + if let Err(error) = handle_client(stream, false, peer.ip().is_loopback()).await { eprintln!("Windows Agent client {peer} failed: {error}"); } }); @@ -106,6 +173,59 @@ fn run_server(listen: SocketAddr) -> anyhow::Result<()> { }) } +#[cfg(windows)] +fn run_pipe_server(name: String, allow_unauthenticated: bool) -> anyhow::Result<()> { + let authenticator = if allow_unauthenticated { + eprintln!("WARNING: Agent Named Pipe authentication is disabled for local development"); + None + } else { + Some(Arc::new(local_auth::PipeAuthenticator::from_environment()?)) + }; + + let runtime = tokio::runtime::Runtime::new()?; + runtime.block_on(async move { + let mut server = windows_security::create_current_user_pipe(&name, true)?; + println!("RemoteDesk Windows Agent named pipe listening on {name}"); + loop { + server.connect().await?; + let connected = server; + server = windows_security::create_current_user_pipe(&name, false)?; + let authenticator = authenticator.clone(); + tokio::spawn(async move { + let mut connected = connected; + let authenticated = if let Some(authenticator) = authenticator { + match tokio::time::timeout( + Duration::from_secs(10), + authenticator.authenticate(&mut connected), + ) + .await + { + Ok(Ok(())) => true, + Ok(Err(error)) => { + eprintln!("Windows Agent named pipe authentication failed: {error}"); + return; + } + Err(_) => { + eprintln!("Windows Agent named pipe authentication timed out"); + return; + } + } + } else { + false + }; + if let Err(error) = handle_client(connected, authenticated, true).await { + eprintln!("Windows Agent named pipe client failed: {error}"); + } + }); + } + }) +} + +#[cfg(not(windows))] +fn run_pipe_server(_name: String, _allow_unauthenticated: bool) -> anyhow::Result<()> { + anyhow::bail!("named pipe control endpoint is Windows-only") +} + #[derive(Serialize)] struct Hello<'a> { kind: &'a str, @@ -113,24 +233,189 @@ struct Hello<'a> { protocol_minor: u16, capture_backend: &'a str, preferred_capture_backend: &'a str, + compatibility_capture_backend: &'a str, software_fallback_backend: &'a str, + video_codecs: [&'a str; 3], + audio_codec: &'a str, + media_transport: &'a str, + target_media_transport: &'a str, software_fallback_available: bool, input_backend: &'a str, authenticated: bool, + media_pipeline_state: &'a str, + supported_capture_modes: [&'a str; 2], + desktop_duplication: bool, + hardware_h264_mft: bool, + hardware_h264_encoder_verified: bool, + hardware_hevc_mft: bool, + hardware_av1_mft: bool, + hardware_pipeline_verified: bool, + degraded: bool, } -async fn handle_client(stream: TcpStream) -> anyhow::Result<()> { - let (reader, mut writer) = stream.into_split(); +struct EncodedSession { + session_id: String, + generation: u64, + stop: std_mpsc::Sender<()>, +} + +#[derive(Clone)] +struct EncodedSessionRoute { + generation: u64, + keyframe: std_mpsc::SyncSender<()>, + stop: std_mpsc::Sender<()>, +} + +impl Drop for EncodedSession { + fn drop(&mut self) { + let _ = self.stop.send(()); + if let Ok(mut sessions) = encoded_sessions().lock() + && sessions + .get(&self.session_id) + .is_some_and(|route| route.generation == self.generation) + { + sessions.remove(&self.session_id); + } + } +} + +fn encoded_sessions() -> &'static Mutex> { + ENCODED_SESSIONS.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn queue_keyframe_request(session_id: &str) -> bool { + if validate_session_id(session_id).is_err() { + return false; + } + let route = encoded_sessions() + .lock() + .ok() + .and_then(|sessions| sessions.get(session_id).cloned()); + route.is_some_and(|route| { + matches!( + route.keyframe.try_send(()), + Ok(()) | Err(std_mpsc::TrySendError::Full(())) + ) + }) +} + +#[derive(Debug)] +struct EncodedSessionInfo { + descriptor: media_ring::MediaRingDescriptor, + encoder_backend: &'static str, + hardware_encoder_selected: bool, +} + +fn requested_capture_mode(command: &AgentCommand) -> &str { + command + .capture_mode + .as_deref() + .unwrap_or(if command.allow_software_fallback { + "compatibility" + } else { + DEFAULT_CAPTURE_MODE + }) +} + +fn validate_capture_mode(mode: &str) -> anyhow::Result<()> { + anyhow::ensure!( + matches!(mode, "strict_hardware" | "compatibility"), + "unsupported capture mode: {mode}" + ); + Ok(()) +} + +fn validate_video_codec(codec: &str) -> anyhow::Result<()> { + anyhow::ensure!( + matches!(codec, "h264" | "hevc" | "av1"), + "unsupported video codec: {codec}" + ); + Ok(()) +} + +fn requested_video_codec(command: &AgentCommand) -> &str { + command + .video_codec + .as_deref() + .unwrap_or(DEFAULT_VIDEO_CODEC) +} + +fn software_desktop_response_for_backend( + mode: &str, + codec: &str, + fps: u8, + capture_backend: &str, +) -> serde_json::Value { + serde_json::json!({ + "kind": "desktop_opened", + "capture_backend": capture_backend, + "pixel_format": "bgra8", + "compression": "zlib", + "video_codec": "bgra-zlib", + "requested_video_codec": codec, + "capture_mode": mode, + "frames_per_second": fps, + "surface_type": "cpu_readback", + "cpu_map_count": 1, + "hardware_path_verified": false, + "degraded": true, + "media_transport": CURRENT_MEDIA_TRANSPORT, + "target_media_transport": TARGET_MEDIA_TRANSPORT + }) +} + +fn unavailable_media_response(mode: &str, codec: &str) -> serde_json::Value { + let error = if mode == "strict_hardware" { + "HARDWARE_CAPTURE_UNAVAILABLE" + } else { + "HARDWARE_ENCODER_UNAVAILABLE" + }; + serde_json::json!({ + "kind": "desktop_unavailable", + "error": error, + "capture_mode": mode, + "video_codec": codec, + "media_pipeline_state": "hardware_probe_failed", + "software_fallback_available": cfg!(windows), + "degraded": false + }) +} + +async fn handle_client( + stream: S, + authenticated: bool, + allow_local_media_control: bool, +) -> anyhow::Result<()> +where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + let capabilities = capability::snapshot(); + let mut encoded_session: Option = None; + let (reader, mut writer) = tokio::io::split(stream); let hello = serde_json::to_string(&Hello { kind: "windows_agent_hello", protocol_major: 1, protocol_minor: 0, - capture_backend: PREFERRED_CAPTURE_BACKEND, + capture_backend: capabilities.capture_backend(), preferred_capture_backend: PREFERRED_CAPTURE_BACKEND, + compatibility_capture_backend: capabilities.capture_backend(), software_fallback_backend: SOFTWARE_FALLBACK_BACKEND, + video_codecs: ["h264", "hevc", "av1"], + audio_codec: "opus", + media_transport: CURRENT_MEDIA_TRANSPORT, + target_media_transport: TARGET_MEDIA_TRANSPORT, software_fallback_available: cfg!(windows), input_backend: "windows-send-input", - authenticated: false, + authenticated, + media_pipeline_state: capabilities.pipeline_state(), + supported_capture_modes: ["strict_hardware", "compatibility"], + desktop_duplication: capabilities.desktop_duplication, + hardware_h264_mft: capabilities.hardware_h264_mft, + hardware_h264_encoder_verified: capabilities.hardware_h264_encoder_verified, + hardware_hevc_mft: capabilities.hardware_hevc_mft, + hardware_av1_mft: capabilities.hardware_av1_mft, + hardware_pipeline_verified: capabilities.hardware_pipeline_verified, + degraded: !capabilities.hardware_pipeline_verified, })?; writer.write_all(hello.as_bytes()).await?; writer.write_all(b"\n").await?; @@ -140,6 +425,183 @@ async fn handle_client(stream: TcpStream) -> anyhow::Result<()> { anyhow::bail!("agent command exceeds 16 KiB"); } let command: AgentCommand = serde_json::from_str(&line)?; + let capture_mode = requested_capture_mode(&command); + if let Err(error) = validate_capture_mode(capture_mode) { + let response = serde_json::json!({ + "kind": "command_failed", + "error": "INVALID_CAPTURE_MODE", + "detail": error.to_string() + }); + writer + .write_all(serde_json::to_string(&response)?.as_bytes()) + .await?; + writer.write_all(b"\n").await?; + continue; + } + let codec = requested_video_codec(&command); + if let Err(error) = validate_video_codec(codec) { + let response = serde_json::json!({ + "kind": "command_failed", + "error": "UNSUPPORTED_VIDEO_CODEC", + "detail": error.to_string() + }); + writer + .write_all(serde_json::to_string(&response)?.as_bytes()) + .await?; + writer.write_all(b"\n").await?; + continue; + } + if command.kind == "request_keyframe" { + let queued = command + .session_id + .as_deref() + .is_some_and(queue_keyframe_request); + let response = serde_json::json!({ + "kind": if queued { "keyframe_request_queued" } else { "command_failed" }, + "error": if queued { serde_json::Value::Null } else { serde_json::Value::String("NO_ENCODED_SESSION".to_owned()) } + }); + writer + .write_all(serde_json::to_string(&response)?.as_bytes()) + .await?; + writer.write_all(b"\n").await?; + continue; + } + if command.kind == "open_encoded_desktop" { + if !allow_local_media_control { + let response = serde_json::json!({ + "kind": "command_failed", + "error": "AUTHENTICATED_LOCAL_CONTROL_REQUIRED" + }); + writer + .write_all(serde_json::to_string(&response)?.as_bytes()) + .await?; + writer.write_all(b"\n").await?; + continue; + } + #[cfg(windows)] + { + let session_id = command + .session_id + .as_deref() + .ok_or_else(|| anyhow::anyhow!("session_id is required for encoded media"))?; + validate_session_id(session_id)?; + if capture_mode == "strict_hardware" { + let response = serde_json::json!({ + "kind": "desktop_unavailable", + "error": "HARDWARE_ZERO_COPY_UNVERIFIED", + "capture_mode": capture_mode, + "video_codec": codec, + "hardware_pipeline_verified": capabilities.hardware_pipeline_verified, + "degraded": false + }); + writer + .write_all(serde_json::to_string(&response)?.as_bytes()) + .await?; + writer.write_all(b"\n").await?; + continue; + } + if codec != "h264" { + let response = serde_json::json!({ + "kind": "desktop_unavailable", + "error": "ENCODED_SESSION_CODEC_NOT_IMPLEMENTED", + "video_codec": codec, + "supported_video_codec": "h264", + "degraded": false + }); + writer + .write_all(serde_json::to_string(&response)?.as_bytes()) + .await?; + writer.write_all(b"\n").await?; + continue; + } + if !command.allow_software_fallback && capture_mode != "compatibility" { + let response = serde_json::json!({ + "kind": "desktop_unavailable", + "error": "COMPATIBILITY_MODE_REQUIRED", + "degraded": false + }); + writer + .write_all(serde_json::to_string(&response)?.as_bytes()) + .await?; + writer.write_all(b"\n").await?; + continue; + } + if let Some(previous) = encoded_session.take() { + drop(previous); + } + match start_encoded_session( + session_id.to_owned(), + command.frames_per_second.unwrap_or(30).clamp(1, 120), + command.allow_software_fallback, + ) { + Ok((info, session)) => { + let response = serde_json::json!({ + "kind": "encoded_desktop_opened", + "capture_backend": "desktop-duplication-compatibility", + "video_codec": "h264", + "pixel_format": "nv12", + "media_transport": "shared_memory_encoded_access_units", + "audio_transport": "separate_transport", + "ring": info.descriptor, + "encoder_backend": info.encoder_backend, + "hardware_encoder_selected": info.hardware_encoder_selected, + "hardware_encoder_verified": false, + "hardware_pipeline_verified": false, + "hardware_path_verified": false, + "surface_type": "cpu_readback_then_encoder_upload", + "cpu_map_count": 1, + "degraded": true + }); + writer + .write_all(serde_json::to_string(&response)?.as_bytes()) + .await?; + writer.write_all(b"\n").await?; + encoded_session = Some(session); + } + Err(error) => { + let response = serde_json::json!({ + "kind": "desktop_unavailable", + "error": "ENCODED_SESSION_UNAVAILABLE", + "detail": error.to_string(), + "degraded": false + }); + writer + .write_all(serde_json::to_string(&response)?.as_bytes()) + .await?; + writer.write_all(b"\n").await?; + } + } + continue; + } + #[cfg(not(windows))] + { + writer + .write_all(b"{\"kind\":\"desktop_unavailable\",\"error\":\"windows_only\"}\n") + .await?; + continue; + } + } + if command.kind == "open_desktop" + && capture_mode == "strict_hardware" + && !capabilities.hardware_pipeline_verified + { + let response = serde_json::json!({ + "kind": "desktop_unavailable", + "error": "HARDWARE_PIPELINE_UNVERIFIED", + "capture_mode": capture_mode, + "video_codec": codec, + "media_pipeline_state": capabilities.pipeline_state(), + "desktop_duplication": capabilities.desktop_duplication, + "hardware_mft_present": capabilities.encoder_available(codec), + "hardware_pipeline_verified": capabilities.hardware_pipeline_verified, + "degraded": false + }); + writer + .write_all(serde_json::to_string(&response)?.as_bytes()) + .await?; + writer.write_all(b"\n").await?; + continue; + } if command.kind == "list_applications" { let response = match applications::load() { Ok(items) => serde_json::json!({ "kind": "applications", "applications": items }), @@ -155,10 +617,20 @@ async fn handle_client(stream: TcpStream) -> anyhow::Result<()> { #[cfg(windows)] match apply_input(&command) { Ok(()) => writer.write_all(b"{\"kind\":\"input_ok\"}\n").await?, - Err(error) => writer.write_all(serde_json::json!({"kind":"input_failed","error":error.to_string()}).to_string().as_bytes()).await?, + Err(error) => { + writer + .write_all( + serde_json::json!({"kind":"input_failed","error":error.to_string()}) + .to_string() + .as_bytes(), + ) + .await? + } } #[cfg(not(windows))] - writer.write_all(b"{\"kind\":\"input_failed\",\"error\":\"windows_only\"}\n").await?; + writer + .write_all(b"{\"kind\":\"input_failed\",\"error\":\"windows_only\"}\n") + .await?; } else if command.kind == "open_application" { if !command.allow_software_fallback { writer.write_all(b"{\"kind\":\"application_error\",\"error\":\"software_fallback_required\"}\n").await?; @@ -175,7 +647,16 @@ async fn handle_client(stream: TcpStream) -> anyhow::Result<()> { .frames_per_second .unwrap_or(DEFAULT_FALLBACK_FPS) .clamp(1, MAX_FALLBACK_FPS); - writer.write_all(format!("{{\"kind\":\"desktop_opened\",\"capture_backend\":\"{SOFTWARE_FALLBACK_BACKEND}\",\"pixel_format\":\"bgra8\",\"compression\":\"zlib\",\"frames_per_second\":{fps}}}\n").as_bytes()).await?; + let response = software_desktop_response_for_backend( + capture_mode, + codec, + fps, + capabilities.capture_backend(), + ); + writer + .write_all(serde_json::to_string(&response)?.as_bytes()) + .await?; + writer.write_all(b"\n").await?; stream_software_desktop(&mut writer, fps).await?; return Ok(()); } @@ -194,12 +675,9 @@ async fn handle_client(stream: TcpStream) -> anyhow::Result<()> { continue; } } else if command.kind == "open_desktop" { - if !command.allow_software_fallback { - let response = serde_json::to_string(&serde_json::json!({ - "kind": "desktop_unavailable", - "error": "strict_gpu_pipeline_unavailable", - "software_fallback_available": cfg!(windows) - }))?; + if capture_mode == "strict_hardware" || !command.allow_software_fallback { + let response = + serde_json::to_string(&unavailable_media_response(capture_mode, codec))?; writer.write_all(response.as_bytes()).await?; writer.write_all(b"\n").await?; continue; @@ -208,25 +686,38 @@ async fn handle_client(stream: TcpStream) -> anyhow::Result<()> { .frames_per_second .unwrap_or(DEFAULT_FALLBACK_FPS) .clamp(1, MAX_FALLBACK_FPS); + let response = software_desktop_response_for_backend( + capture_mode, + codec, + fps, + capabilities.capture_backend(), + ); writer - .write_all( - format!( - "{{\"kind\":\"desktop_opened\",\"capture_backend\":\"{SOFTWARE_FALLBACK_BACKEND}\",\"pixel_format\":\"bgra8\",\"compression\":\"zlib\",\"frames_per_second\":{fps}}}\n" - ) - .as_bytes(), - ) + .write_all(serde_json::to_string(&response)?.as_bytes()) .await?; + writer.write_all(b"\n").await?; stream_software_desktop(&mut writer, fps).await?; return Ok(()); } else if command.kind == "status" { let response = serde_json::to_string(&serde_json::json!({ "kind": "status", "ready": true, - "media": "strict_gpu_pipeline_unavailable", + "media": capabilities.pipeline_state(), "input": "not_started", + "media_transport": CURRENT_MEDIA_TRANSPORT, + "target_media_transport": TARGET_MEDIA_TRANSPORT, "preferred_capture_backend": PREFERRED_CAPTURE_BACKEND, + "compatibility_capture_backend": capabilities.capture_backend(), "software_fallback_backend": SOFTWARE_FALLBACK_BACKEND, "software_fallback_available": cfg!(windows), + "desktop_duplication": capabilities.desktop_duplication, + "hardware_h264_mft": capabilities.hardware_h264_mft, + "hardware_h264_encoder_verified": capabilities.hardware_h264_encoder_verified, + "hardware_hevc_mft": capabilities.hardware_hevc_mft, + "hardware_av1_mft": capabilities.hardware_av1_mft, + "hardware_pipeline_verified": capabilities.hardware_pipeline_verified, + "degraded": !capabilities.hardware_pipeline_verified, + "supported_capture_modes": ["strict_hardware", "compatibility"], "capture_latency_ms": latency_ms(&CAPTURE_LATENCY_US), "encode_latency_ms": latency_ms(&ENCODE_LATENCY_US), "frame_processing_latency_ms": latency_ms(&FRAME_PROCESSING_LATENCY_US) @@ -235,6 +726,7 @@ async fn handle_client(stream: TcpStream) -> anyhow::Result<()> { writer.write_all(b"\n").await?; } } + drop(encoded_session); Ok(()) } @@ -243,6 +735,366 @@ fn latency_ms(value: &AtomicU64) -> Option { (micros != NO_LATENCY_SAMPLE).then(|| Duration::from_micros(micros).as_secs_f64() * 1_000.0) } +fn validate_session_id(session_id: &str) -> anyhow::Result<()> { + anyhow::ensure!( + !session_id.is_empty() + && session_id.len() <= 128 + && session_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')), + "session_id is invalid" + ); + Ok(()) +} + +#[cfg(windows)] +fn start_encoded_session( + session_id: String, + fps: u8, + allow_software_fallback: bool, +) -> anyhow::Result<(EncodedSessionInfo, EncodedSession)> { + let generation = ENCODED_GENERATION.fetch_add(1, Ordering::Relaxed).max(1); + let session_started = Instant::now(); + let registry_session_id = session_id.clone(); + let (stop_sender, stop_receiver) = std_mpsc::channel(); + let (keyframe_sender, keyframe_receiver) = std_mpsc::sync_channel(1); + let (ready_sender, ready_receiver) = std_mpsc::sync_channel(1); + std::thread::Builder::new() + .name("windows-agent-encoded-media".to_owned()) + .spawn(move || { + // COM, DDA, and the MFT are thread-affine. Keep their complete + // lifetime on this worker instead of moving interface pointers. + let mut capture = match capture::DdaCapture::new() { + Ok(capture) => capture, + Err(error) => { + let _ = ready_sender.send(Err(error.to_string())); + return; + } + }; + let (width, height) = capture.dimensions(); + if width % 2 != 0 || height % 2 != 0 { + let _ = ready_sender.send(Err("DDA dimensions are not NV12 compatible".to_owned())); + return; + } + let (mut encoder, encoder_backend, hardware_path_verified) = + match encoder::H264Encoder::new(width, height, u32::from(fps), true) { + Ok(encoder) => (encoder, "hardware-mft", true), + Err(error) if allow_software_fallback => { + match encoder::H264Encoder::new(width, height, u32::from(fps), false) { + Ok(encoder) => (encoder, "software-mft", false), + Err(fallback) => { + let _ = ready_sender + .send(Err(format!("hardware: {error}; software: {fallback}"))); + return; + } + } + } + Err(error) => { + let _ = ready_sender + .send(Err(format!("hardware H.264 encoder unavailable: {error}"))); + return; + } + }; + let ring_path = std::env::temp_dir().join(format!( + "remotedesk-{session_id}-{}-{generation}.ring", + std::process::id() + )); + // Bound memory per session: enough queueing for bursty transport, + // without reserving a gigabyte for every desktop. + let mut ring = match media_ring::MediaRing::create(&ring_path, 16, 8 * 1024 * 1024) { + Ok(ring) => ring, + Err(error) => { + let _ = ready_sender.send(Err(error.to_string())); + return; + } + }; + let mut descriptor = + match ring.descriptor(&session_id, generation, std::process::id(), &ring_path) { + Ok(descriptor) => descriptor, + Err(error) => { + drop(ring); + let _ = std::fs::remove_file(&ring_path); + let _ = ready_sender.send(Err(error.to_string())); + return; + } + }; + let audio_path = std::env::temp_dir().join(format!( + "remotedesk-{session_id}-{}-{generation}.audio.ring", + std::process::id() + )); + let mut audio_worker = media_ring::MediaRing::create(&audio_path, 32, 8 * 1024) + .ok() + .and_then(|audio_ring| { + let audio_descriptor = audio_ring + .audio_descriptor(&session_id, generation, std::process::id(), &audio_path) + .ok()?; + match start_audio_worker(audio_ring, session_started) { + Ok(worker) => { + descriptor.audio_ring = Some(audio_descriptor); + Some(worker) + } + Err(error) => { + eprintln!("encoded media audio disabled: {error}"); + None + } + } + }); + if descriptor.audio_ring.is_none() { + let _ = std::fs::remove_file(&audio_path); + } + if ready_sender + .send(Ok(EncodedSessionInfo { + descriptor, + encoder_backend, + hardware_encoder_selected: hardware_path_verified, + })) + .is_err() + { + drop(ring); + if let Some(worker) = audio_worker.take() { + worker.stop(); + } + let _ = std::fs::remove_file(&audio_path); + let _ = std::fs::remove_file(&ring_path); + return; + } + let interval = Duration::from_secs_f64(1.0 / f64::from(fps)); + let mut sequence = 1_u64; + loop { + match stop_receiver.try_recv() { + Ok(()) | Err(std_mpsc::TryRecvError::Disconnected) => break, + Err(std_mpsc::TryRecvError::Empty) => {} + } + if keyframe_receiver.try_recv().is_ok() + && let Err(error) = encoder.request_keyframe() + { + eprintln!("encoded media keyframe request failed: {error}"); + } + let started = std::time::Instant::now(); + match capture.capture_bgra_frame() { + Ok(frame) => { + record_latency(&CAPTURE_LATENCY_US, frame.capture_latency); + let encoded = + encoder::bgra_to_nv12(frame.width, frame.height, &frame.pixels) + .map_err(|error| anyhow::anyhow!(error.to_string())) + .and_then(|nv12| { + let pts_100ns = + i64::try_from(session_pts_100ns(session_started)) + .unwrap_or(i64::MAX); + encoder + .encode_at(&nv12, pts_100ns) + .map_err(|error| anyhow::anyhow!(error.to_string())) + }); + match encoded { + Ok(Some(access_unit)) => { + record_latency(&ENCODE_LATENCY_US, access_unit.encode_latency); + let pts = u64::try_from(access_unit.pts_100ns).unwrap_or(0); + let kind = if access_unit.keyframe { 2 } else { 1 }; + let record = media_ring::MediaRecord { + sequence, + pts, + kind, + payload: access_unit.data, + }; + let _ = ring.push(&record); + sequence = sequence.saturating_add(1); + } + Ok(None) => {} + Err(error) => { + eprintln!("encoded media worker dropped frame: {error}"); + } + } + record_latency(&FRAME_PROCESSING_LATENCY_US, started.elapsed()); + } + Err(error) if error.to_string().contains("dda_timeout") => {} + Err(error) => { + eprintln!("encoded media capture stopped: {error}"); + break; + } + } + match stop_receiver.recv_timeout(interval.saturating_sub(started.elapsed())) { + Ok(()) | Err(std_mpsc::RecvTimeoutError::Disconnected) => break, + Err(std_mpsc::RecvTimeoutError::Timeout) => {} + } + } + drop(ring); + if let Some(worker) = audio_worker.take() { + worker.stop(); + } + for _ in 0..20 { + match std::fs::remove_file(&audio_path) { + Ok(()) => break, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => break, + Err(_) => std::thread::sleep(Duration::from_millis(10)), + } + } + for _ in 0..20 { + match std::fs::remove_file(&ring_path) { + Ok(()) => break, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => break, + Err(_) => std::thread::sleep(Duration::from_millis(10)), + } + } + })?; + let info = ready_receiver + .recv() + .map_err(|_| anyhow::anyhow!("encoded media worker exited during startup"))? + .map_err(|error| anyhow::anyhow!("encoded media worker unavailable: {error}"))?; + let previous = encoded_sessions() + .lock() + .map_err(|_| anyhow::anyhow!("encoded session registry is poisoned"))? + .insert( + registry_session_id.clone(), + EncodedSessionRoute { + generation, + keyframe: keyframe_sender.clone(), + stop: stop_sender.clone(), + }, + ); + if let Some(previous) = previous { + let _ = previous.stop.send(()); + } + Ok(( + info, + EncodedSession { + session_id: registry_session_id, + generation, + stop: stop_sender, + }, + )) +} + +#[cfg(windows)] +struct AudioWorker { + stop_sender: std_mpsc::Sender<()>, + handle: std::thread::JoinHandle<()>, +} + +#[cfg(windows)] +impl AudioWorker { + fn stop(self) { + let _ = self.stop_sender.send(()); + let _ = self.handle.join(); + } +} + +#[cfg(windows)] +fn start_audio_worker( + mut ring: media_ring::MediaRing, + session_started: Instant, +) -> Result { + let (stop_sender, stop_receiver) = std_mpsc::channel(); + let (ready_sender, ready_receiver) = std_mpsc::sync_channel(1); + let handle = std::thread::Builder::new() + .name("windows-agent-opus-capture".to_owned()) + .spawn(move || { + let capture = match audio::AudioCapture::new(session_started) { + Ok(capture) => capture, + Err(error) => { + let _ = ready_sender.send(Err(error.to_string())); + return; + } + }; + let mut pipeline = + match audio::AudioPipeline::new(capture.source_rate, capture.source_channels) { + Ok(pipeline) => pipeline, + Err(error) => { + let _ = ready_sender.send(Err(error.to_string())); + return; + } + }; + if ready_sender.send(Ok(())).is_err() { + return; + } + loop { + match stop_receiver.try_recv() { + Ok(()) | Err(std_mpsc::TryRecvError::Disconnected) => return, + Err(std_mpsc::TryRecvError::Empty) => {} + } + let callback_discontinuity = capture.discontinuity.swap(false, Ordering::AcqRel); + if callback_discontinuity { + pipeline.mark_discontinuity(); + } + let mut drained = false; + if callback_discontinuity { + match capture.try_receive_latest() { + Ok(Some(chunk)) => { + drained = true; + write_audio_chunk(&mut pipeline, &mut ring, &chunk); + } + Ok(None) | Err(std_mpsc::TryRecvError::Empty) => {} + Err(std_mpsc::TryRecvError::Disconnected) => return, + } + } + loop { + match capture.try_receive() { + Ok(Some(chunk)) => { + drained = true; + write_audio_chunk(&mut pipeline, &mut ring, &chunk); + } + Ok(None) | Err(std_mpsc::TryRecvError::Empty) => break, + Err(std_mpsc::TryRecvError::Disconnected) => return, + } + } + if !drained { + match stop_receiver.recv_timeout(Duration::from_millis(2)) { + Ok(()) | Err(std_mpsc::RecvTimeoutError::Disconnected) => return, + Err(std_mpsc::RecvTimeoutError::Timeout) => {} + } + } + } + }) + .map_err(|error| error.to_string())?; + match ready_receiver.recv() { + Ok(Ok(())) => Ok(AudioWorker { + stop_sender, + handle, + }), + Ok(Err(error)) => { + let _ = handle.join(); + Err(error) + } + Err(_) => { + let _ = handle.join(); + Err("audio worker exited during startup".to_owned()) + } + } +} + +#[cfg(windows)] +fn write_audio_chunk( + pipeline: &mut audio::AudioPipeline, + ring: &mut media_ring::MediaRing, + chunk: &audio::CapturedAudioChunk, +) { + match pipeline.push_source(&chunk.samples, chunk.capture_end_pts_100ns) { + Ok(packets) => { + for packet in packets { + let record = media_ring::MediaRecord { + sequence: packet.sequence, + pts: packet.pts_100ns, + kind: if packet.discontinuity { 2 } else { 1 }, + payload: packet.data, + }; + if !matches!(ring.push(&record), Ok(true)) { + pipeline.mark_discontinuity(); + } + } + } + Err(_) => pipeline.mark_discontinuity(), + } +} + +fn session_pts_100ns(session_started: Instant) -> u64 { + u64::try_from( + Instant::now() + .saturating_duration_since(session_started) + .as_nanos() + / 100, + ) + .unwrap_or(u64::MAX) +} + fn record_latency(value: &AtomicU64, elapsed: Duration) { let micros = u64::try_from(elapsed.as_micros()).unwrap_or(NO_LATENCY_SAMPLE - 1); value.store(micros, Ordering::Relaxed); @@ -253,6 +1105,12 @@ struct AgentCommand { kind: String, #[serde(default)] allow_software_fallback: bool, + #[serde(default)] + capture_mode: Option, + #[serde(default)] + video_codec: Option, + #[serde(default)] + session_id: Option, frames_per_second: Option, #[serde(default)] application: Option, @@ -274,12 +1132,31 @@ struct AgentCommand { #[cfg(windows)] fn apply_input(command: &AgentCommand) -> anyhow::Result<()> { - use windows::Win32::UI::Input::KeyboardAndMouse::{mouse_event, SendInput, INPUT, INPUT_0, INPUT_KEYBOARD, KEYBDINPUT, KEYBD_EVENT_FLAGS, KEYEVENTF_KEYUP, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_MOVE, MOUSEEVENTF_VIRTUALDESK, VIRTUAL_KEY}; + use windows::Win32::UI::Input::KeyboardAndMouse::{ + INPUT, INPUT_0, INPUT_KEYBOARD, KEYBD_EVENT_FLAGS, KEYBDINPUT, KEYEVENTF_KEYUP, + MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_MOVE, MOUSEEVENTF_VIRTUALDESK, SendInput, VIRTUAL_KEY, + mouse_event, + }; unsafe { match command.input_type.as_deref() { Some("key") => { - let flags = if command.down == Some(false) { KEYEVENTF_KEYUP } else { KEYBD_EVENT_FLAGS(0) }; - let input = INPUT { r#type: INPUT_KEYBOARD, Anonymous: INPUT_0 { ki: KEYBDINPUT { wVk: VIRTUAL_KEY(command.code.unwrap_or(0)), wScan: 0, dwFlags: flags, time: 0, dwExtraInfo: 0 } } }; + let flags = if command.down == Some(false) { + KEYEVENTF_KEYUP + } else { + KEYBD_EVENT_FLAGS(0) + }; + let input = INPUT { + r#type: INPUT_KEYBOARD, + Anonymous: INPUT_0 { + ki: KEYBDINPUT { + wVk: VIRTUAL_KEY(command.code.unwrap_or(0)), + wScan: 0, + dwFlags: flags, + time: 0, + dwExtraInfo: 0, + }, + }, + }; SendInput(&[input], std::mem::size_of::() as i32); } Some("mouse") => { @@ -288,12 +1165,24 @@ fn apply_input(command: &AgentCommand) -> anyhow::Result<()> { let mut flags = MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK | MOUSEEVENTF_MOVE; if let Some(action) = command.mouse_action.as_deref() { flags |= match action { - "left_down" => windows::Win32::UI::Input::KeyboardAndMouse::MOUSEEVENTF_LEFTDOWN, - "left_up" => windows::Win32::UI::Input::KeyboardAndMouse::MOUSEEVENTF_LEFTUP, - "right_down" => windows::Win32::UI::Input::KeyboardAndMouse::MOUSEEVENTF_RIGHTDOWN, - "right_up" => windows::Win32::UI::Input::KeyboardAndMouse::MOUSEEVENTF_RIGHTUP, - "middle_down" => windows::Win32::UI::Input::KeyboardAndMouse::MOUSEEVENTF_MIDDLEDOWN, - "middle_up" => windows::Win32::UI::Input::KeyboardAndMouse::MOUSEEVENTF_MIDDLEUP, + "left_down" => { + windows::Win32::UI::Input::KeyboardAndMouse::MOUSEEVENTF_LEFTDOWN + } + "left_up" => { + windows::Win32::UI::Input::KeyboardAndMouse::MOUSEEVENTF_LEFTUP + } + "right_down" => { + windows::Win32::UI::Input::KeyboardAndMouse::MOUSEEVENTF_RIGHTDOWN + } + "right_up" => { + windows::Win32::UI::Input::KeyboardAndMouse::MOUSEEVENTF_RIGHTUP + } + "middle_down" => { + windows::Win32::UI::Input::KeyboardAndMouse::MOUSEEVENTF_MIDDLEDOWN + } + "middle_up" => { + windows::Win32::UI::Input::KeyboardAndMouse::MOUSEEVENTF_MIDDLEUP + } "wheel" => windows::Win32::UI::Input::KeyboardAndMouse::MOUSEEVENTF_WHEEL, _ => MOUSEEVENTF_MOVE, }; @@ -307,28 +1196,15 @@ fn apply_input(command: &AgentCommand) -> anyhow::Result<()> { } #[cfg(windows)] -async fn stream_software_desktop( - writer: &mut tokio::net::tcp::OwnedWriteHalf, - fps: u8, -) -> anyhow::Result<()> { +async fn stream_software_desktop(writer: &mut W, fps: u8) -> anyhow::Result<()> +where + W: AsyncWrite + Unpin, +{ let (sender, mut receiver) = tokio::sync::mpsc::channel::>>(1); std::thread::Builder::new() - .name("gdi-bgra-zlib-capture".to_owned()) + .name("desktop-capture-compatibility".to_owned()) .spawn(move || { - let result = fallback::GdiCapture::new().and_then(|mut capture| { - let interval = Duration::from_secs_f64(1.0 / f64::from(fps)); - loop { - let started = std::time::Instant::now(); - let encoded = capture.capture_frame()?; - record_latency(&CAPTURE_LATENCY_US, encoded.capture_latency); - record_latency(&ENCODE_LATENCY_US, encoded.encode_latency); - record_latency(&FRAME_PROCESSING_LATENCY_US, started.elapsed()); - if sender.blocking_send(Ok(encoded.bytes)).is_err() { - return Ok(()); - } - std::thread::sleep(interval.saturating_sub(started.elapsed())); - } - }); + let result = capture_compatibility(fps, &sender); if let Err(error) = result { let _ = sender.blocking_send(Err(error)); } @@ -350,11 +1226,63 @@ async fn stream_software_desktop( Ok(()) } -#[cfg(not(windows))] -async fn stream_software_desktop( - _writer: &mut tokio::net::tcp::OwnedWriteHalf, - _fps: u8, +#[cfg(windows)] +fn capture_compatibility( + fps: u8, + sender: &tokio::sync::mpsc::Sender>>, ) -> anyhow::Result<()> { + let interval = Duration::from_secs_f64(1.0 / f64::from(fps)); + match capture::DdaCapture::new() { + Ok(mut capture) => loop { + let started = std::time::Instant::now(); + match capture.capture_frame() { + Ok(encoded) => { + record_latency(&CAPTURE_LATENCY_US, encoded.capture_latency); + record_latency(&ENCODE_LATENCY_US, encoded.encode_latency); + record_latency(&FRAME_PROCESSING_LATENCY_US, started.elapsed()); + if sender.blocking_send(Ok(encoded.bytes)).is_err() { + return Ok(()); + } + } + Err(error) if error.to_string().contains("dda_timeout") => {} + Err(error) => return Err(error), + } + std::thread::sleep(interval.saturating_sub(started.elapsed())); + }, + Err(dda_error) => { + eprintln!( + "Desktop Duplication unavailable; using GDI compatibility fallback: {dda_error}" + ); + let mut capture = fallback::GdiCapture::new()?; + let interval = Duration::from_secs_f64(1.0 / f64::from(fps)); + loop { + let started = std::time::Instant::now(); + let encoded = capture.capture_frame()?; + record_latency(&CAPTURE_LATENCY_US, encoded.capture_latency); + record_latency(&ENCODE_LATENCY_US, encoded.encode_latency); + record_latency(&FRAME_PROCESSING_LATENCY_US, started.elapsed()); + if sender.blocking_send(Ok(encoded.bytes)).is_err() { + return Ok(()); + } + std::thread::sleep(interval.saturating_sub(started.elapsed())); + } + } + } +} + +#[cfg(not(windows))] +fn capture_compatibility( + _fps: u8, + _sender: &tokio::sync::mpsc::Sender>>, +) -> anyhow::Result<()> { + anyhow::bail!("gdi_software_fallback_unavailable") +} + +#[cfg(not(windows))] +async fn stream_software_desktop(_writer: &mut W, _fps: u8) -> anyhow::Result<()> +where + W: AsyncWrite + Unpin, +{ anyhow::bail!("gdi_software_fallback_unavailable") } @@ -577,5 +1505,59 @@ mod tests { .unwrap(); assert!(enabled.allow_software_fallback); assert_eq!(enabled.frames_per_second, Some(12)); + assert_eq!(requested_capture_mode(&default), "strict_hardware"); + assert_eq!(requested_capture_mode(&enabled), "compatibility"); + } + + #[test] + fn explicit_capture_mode_overrides_legacy_fallback_flag() { + let command: AgentCommand = serde_json::from_str( + r#"{"kind":"open_desktop","allow_software_fallback":true,"capture_mode":"strict_hardware"}"#, + ) + .unwrap(); + assert_eq!(requested_capture_mode(&command), "strict_hardware"); + assert!(validate_capture_mode(requested_capture_mode(&command)).is_ok()); + } + + #[test] + fn unsupported_capture_mode_is_rejected() { + assert!(validate_capture_mode("idd_hdr10").is_err()); + let response = unavailable_media_response("strict_hardware", "hevc"); + assert_eq!(response["error"], "HARDWARE_CAPTURE_UNAVAILABLE"); + } + + #[test] + fn codec_and_degraded_response_are_explicit() { + assert!(validate_video_codec("h264").is_ok()); + assert!(validate_video_codec("vp9").is_err()); + let response = software_desktop_response_for_backend( + "compatibility", + "h264", + 10, + SOFTWARE_FALLBACK_BACKEND, + ); + assert_eq!(response["degraded"], true); + assert_eq!(response["hardware_path_verified"], false); + } + + #[test] + fn keyframe_requests_are_session_bound_and_coalesced() { + let session_id = format!("keyframe-test-{}", std::process::id()); + let (sender, receiver) = std_mpsc::sync_channel(1); + encoded_sessions().lock().unwrap().insert( + session_id.clone(), + EncodedSessionRoute { + generation: 9, + keyframe: sender, + stop: std_mpsc::channel().0, + }, + ); + assert!(queue_keyframe_request(&session_id)); + assert!(queue_keyframe_request(&session_id)); + assert_eq!(receiver.try_recv(), Ok(())); + assert!(receiver.try_recv().is_err()); + encoded_sessions().lock().unwrap().remove(&session_id); + assert!(!queue_keyframe_request(&session_id)); + assert!(!queue_keyframe_request("bad session")); } } diff --git a/agent/windows-agent/src/media_ring.rs b/agent/windows-agent/src/media_ring.rs new file mode 100644 index 0000000..6e2b651 --- /dev/null +++ b/agent/windows-agent/src/media_ring.rs @@ -0,0 +1,439 @@ +//! Bounded SPSC ring for encoded media IPC. +//! +//! The ring carries compressed access units only. It deliberately does not +//! expose a GPU texture or a raw BGRA/NV12 surface to the Go service. + +use memmap2::{MmapMut, MmapOptions}; +use serde::Serialize; +use std::fs::{File, OpenOptions}; +use std::io; +use std::mem::{align_of, size_of}; +use std::path::Path; +use std::sync::atomic::{AtomicU64, Ordering}; + +const MAGIC: [u8; 8] = *b"RDMRING1"; +const VERSION: u32 = 1; +const HEADER_BYTES: usize = size_of::(); +const SLOT_HEADER_BYTES: usize = size_of::(); +const MAX_SLOTS: u32 = 256; +const MAX_SLOT_BYTES: u32 = 16 * 1024 * 1024; + +#[repr(C, align(8))] +struct RingHeader { + magic: [u8; 8], + version: u32, + slot_count: u32, + slot_bytes: u32, + reserved: u32, + write_seq: AtomicU64, + read_seq: AtomicU64, +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct SlotHeader { + sequence: u64, + pts: u64, + kind: u32, + length: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MediaRecord { + pub sequence: u64, + pub pts: u64, + pub kind: u32, + pub payload: Vec, +} + +/// Control-plane description for a ring owned by one authenticated session. +/// The ring contains encoded access units only; it is never a raw-surface +/// transport. The creator must bind this descriptor to its session ACL before +/// sending it to another process. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct MediaRingDescriptor { + pub schema_version: u16, + pub session_id: String, + pub generation: u64, + pub owner_pid: u32, + pub ring_path: String, + pub slot_count: u32, + pub slot_bytes: u32, + pub video_transport: &'static str, + pub audio_transport: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub audio_ring: Option, +} + +/// Control-plane description for a separate Opus packet ring. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct AudioRingDescriptor { + pub schema_version: u16, + pub session_id: String, + pub generation: u64, + pub owner_pid: u32, + pub ring_path: String, + pub slot_count: u32, + pub slot_bytes: u32, + pub audio_transport: &'static str, +} + +impl AudioRingDescriptor { + pub fn validate(&self) -> io::Result<()> { + if self.schema_version != VERSION as u16 + || self.generation == 0 + || self.owner_pid == 0 + || self.session_id.is_empty() + || self.session_id.len() > 128 + || !self + .session_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) + { + return invalid_ring("audio ring descriptor identity is invalid"); + } + if !Path::new(&self.ring_path).is_absolute() { + return invalid_ring("audio ring path must be absolute"); + } + if self.audio_transport != "shared_memory_opus_packets" { + return invalid_ring("audio ring transport contract is invalid"); + } + validate_geometry(self.slot_count, self.slot_bytes) + } +} + +impl MediaRingDescriptor { + pub fn validate(&self) -> io::Result<()> { + if self.schema_version != VERSION as u16 + || self.generation == 0 + || self.owner_pid == 0 + || self.session_id.is_empty() + || self.session_id.len() > 128 + || !self + .session_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) + { + return invalid_ring("media ring descriptor identity is invalid"); + } + if !Path::new(&self.ring_path).is_absolute() { + return invalid_ring("media ring path must be absolute"); + } + if self.video_transport != "shared_memory_encoded_access_units" + || self.audio_transport != "separate_transport" + { + return invalid_ring("media ring transport contract is invalid"); + } + validate_geometry(self.slot_count, self.slot_bytes) + } +} + +pub struct MediaRing { + file: File, + map: MmapMut, + slot_count: u32, + slot_bytes: u32, +} + +impl MediaRing { + pub fn descriptor( + &self, + session_id: impl Into, + generation: u64, + owner_pid: u32, + ring_path: impl AsRef, + ) -> io::Result { + let descriptor = MediaRingDescriptor { + schema_version: VERSION as u16, + session_id: session_id.into(), + generation, + owner_pid, + ring_path: ring_path.as_ref().to_string_lossy().into_owned(), + slot_count: self.slot_count, + slot_bytes: self.slot_bytes, + video_transport: "shared_memory_encoded_access_units", + audio_transport: "separate_transport", + audio_ring: None, + }; + descriptor.validate()?; + Ok(descriptor) + } + + pub fn audio_descriptor( + &self, + session_id: impl Into, + generation: u64, + owner_pid: u32, + ring_path: impl AsRef, + ) -> io::Result { + let descriptor = AudioRingDescriptor { + schema_version: VERSION as u16, + session_id: session_id.into(), + generation, + owner_pid, + ring_path: ring_path.as_ref().to_string_lossy().into_owned(), + slot_count: self.slot_count, + slot_bytes: self.slot_bytes, + audio_transport: "shared_memory_opus_packets", + }; + descriptor.validate()?; + Ok(descriptor) + } + + pub fn create(path: impl AsRef, slot_count: u32, slot_bytes: u32) -> io::Result { + validate_geometry(slot_count, slot_bytes)?; + let path = path.as_ref(); + #[cfg(windows)] + let file = crate::windows_security::create_current_user_file(path)?; + #[cfg(not(windows))] + let file = OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(path)?; + let length = ring_bytes(slot_count, slot_bytes)?; + file.set_len(length as u64)?; + let mut map = unsafe { MmapOptions::new().len(length).map_mut(&file)? }; + unsafe { + std::ptr::write( + map.as_mut_ptr().cast::(), + RingHeader { + magic: MAGIC, + version: VERSION, + slot_count, + slot_bytes, + reserved: 0, + write_seq: AtomicU64::new(0), + read_seq: AtomicU64::new(0), + }, + ); + } + Ok(Self { + file, + map, + slot_count, + slot_bytes, + }) + } + + #[allow(dead_code)] + pub fn open(path: impl AsRef) -> io::Result { + let file = OpenOptions::new().read(true).write(true).open(path)?; + let length = usize::try_from(file.metadata()?.len()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "ring file is too large"))?; + if length < HEADER_BYTES { + return invalid_ring("ring file is shorter than its header"); + } + let map = unsafe { MmapOptions::new().len(length).map_mut(&file)? }; + let header = unsafe { &*map.as_ptr().cast::() }; + if header.magic != MAGIC || header.version != VERSION { + return invalid_ring("ring header magic or version is invalid"); + } + validate_geometry(header.slot_count, header.slot_bytes)?; + if ring_bytes(header.slot_count, header.slot_bytes)? != length { + return invalid_ring("ring file size does not match its header"); + } + Ok(Self { + file, + map, + slot_count: header.slot_count, + slot_bytes: header.slot_bytes, + }) + } + + /// Pushes one encoded access unit. A full ring returns false so the + /// caller can drop this video packet without blocking input or audio. + pub fn push(&mut self, record: &MediaRecord) -> io::Result { + if record.payload.is_empty() || record.payload.len() > self.slot_bytes as usize { + return invalid_ring("media record exceeds ring slot capacity"); + } + let write = self.header().write_seq.load(Ordering::Relaxed); + let read = self.header().read_seq.load(Ordering::Acquire); + if write.wrapping_sub(read) >= u64::from(self.slot_count) { + return Ok(false); + } + let slot = self.slot_mut(write); + slot.payload[..record.payload.len()].copy_from_slice(&record.payload); + *slot.header = SlotHeader { + sequence: record.sequence, + pts: record.pts, + kind: record.kind, + length: record.payload.len() as u32, + }; + self.header() + .write_seq + .store(write.wrapping_add(1), Ordering::Release); + Ok(true) + } + + #[allow(dead_code)] + pub fn pop(&mut self) -> io::Result> { + let read = self.header().read_seq.load(Ordering::Relaxed); + let write = self.header().write_seq.load(Ordering::Acquire); + if read == write { + return Ok(None); + } + let slot_bytes = self.slot_bytes as usize; + let slot = self.slot_mut(read); + let length = usize::try_from(slot.header.length).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidData, "ring record length overflow") + })?; + if length == 0 || length > slot_bytes { + return invalid_ring("ring record length is invalid"); + } + let record = MediaRecord { + sequence: slot.header.sequence, + pts: slot.header.pts, + kind: slot.header.kind, + payload: slot.payload[..length].to_vec(), + }; + self.header() + .read_seq + .store(read.wrapping_add(1), Ordering::Release); + Ok(Some(record)) + } + + fn header(&self) -> &RingHeader { + debug_assert_eq!(self.map.as_ptr().align_offset(align_of::()), 0); + unsafe { &*self.map.as_ptr().cast::() } + } + + fn slot_mut(&mut self, sequence: u64) -> Slot<'_> { + let offset = + HEADER_BYTES + (sequence as usize % self.slot_count as usize) * self.slot_stride(); + let pointer = unsafe { self.map.as_mut_ptr().add(offset) }; + Slot { + header: unsafe { &mut *pointer.cast::() }, + payload: unsafe { + std::slice::from_raw_parts_mut( + pointer.add(SLOT_HEADER_BYTES), + self.slot_bytes as usize, + ) + }, + } + } + + fn slot_stride(&self) -> usize { + SLOT_HEADER_BYTES + self.slot_bytes as usize + } +} + +struct Slot<'a> { + header: &'a mut SlotHeader, + payload: &'a mut [u8], +} + +fn validate_geometry(slot_count: u32, slot_bytes: u32) -> io::Result<()> { + if !(2..=MAX_SLOTS).contains(&slot_count) || !slot_count.is_power_of_two() { + return invalid_ring("slot count must be a power of two between 2 and 256"); + } + if slot_bytes == 0 || slot_bytes > MAX_SLOT_BYTES { + return invalid_ring("slot capacity is invalid"); + } + Ok(()) +} + +fn ring_bytes(slot_count: u32, slot_bytes: u32) -> io::Result { + let slots = usize::try_from(slot_count) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "slot count overflow"))?; + let capacity = usize::try_from(slot_bytes) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "slot capacity overflow"))?; + HEADER_BYTES + .checked_add( + slots + .checked_mul(SLOT_HEADER_BYTES.checked_add(capacity).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "ring size overflow") + })?) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "ring size overflow"))?, + ) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "ring size overflow")) +} + +fn invalid_ring(message: &str) -> io::Result { + Err(io::Error::new(io::ErrorKind::InvalidData, message)) +} + +impl Drop for MediaRing { + fn drop(&mut self) { + let _ = self.map.flush(); + let _ = self.file.sync_data(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn temp_path() -> std::path::PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + std::env::temp_dir().join(format!("remotedesk-media-ring-{nonce}.bin")) + } + + #[test] + fn producer_and_consumer_share_encoded_records() { + let path = temp_path(); + let mut producer = MediaRing::create(&path, 2, 64).unwrap(); + let mut consumer = MediaRing::open(&path).unwrap(); + let record = MediaRecord { + sequence: 7, + pts: 11, + kind: 1, + payload: b"encoded-access-unit".to_vec(), + }; + assert!(producer.push(&record).unwrap()); + assert_eq!(consumer.pop().unwrap(), Some(record)); + assert_eq!(consumer.pop().unwrap(), None); + drop(consumer); + drop(producer); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn full_ring_does_not_block() { + let path = temp_path(); + let mut ring = MediaRing::create(&path, 2, 8).unwrap(); + let record = MediaRecord { + sequence: 1, + pts: 0, + kind: 1, + payload: vec![1], + }; + assert!(ring.push(&record).unwrap()); + assert!(ring.push(&record).unwrap()); + assert!(!ring.push(&record).unwrap()); + drop(ring); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn descriptor_binds_session_and_encoded_only_transport() { + let path = temp_path(); + let ring = MediaRing::create(&path, 2, 64).unwrap(); + let descriptor = ring.descriptor("session-1", 3, 42, &path).unwrap(); + assert_eq!( + descriptor.video_transport, + "shared_memory_encoded_access_units" + ); + assert!(descriptor.validate().is_ok()); + drop(ring); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn descriptor_rejects_relative_or_unbound_identity() { + let path = temp_path(); + let ring = MediaRing::create(&path, 2, 64).unwrap(); + assert!(ring.descriptor("bad session", 1, 42, &path).is_err()); + assert!(ring.descriptor("session", 0, 42, &path).is_err()); + assert!( + ring.descriptor("session", 1, 42, "relative-ring.bin") + .is_err() + ); + drop(ring); + std::fs::remove_file(path).unwrap(); + } +} diff --git a/agent/windows-agent/src/windows_security.rs b/agent/windows-agent/src/windows_security.rs new file mode 100644 index 0000000..c1c720b --- /dev/null +++ b/agent/windows-agent/src/windows_security.rs @@ -0,0 +1,172 @@ +use std::ffi::{OsStr, c_void}; +use std::fs::File; +use std::io; +use std::mem; +use std::os::windows::ffi::OsStrExt as _; +use std::os::windows::io::{AsRawHandle as _, FromRawHandle as _, RawHandle}; +use std::path::Path; +use std::ptr; +use tokio::net::windows::named_pipe::{NamedPipeServer, PipeMode, ServerOptions}; +use windows_sys::Win32::Foundation::{ + ERROR_INSUFFICIENT_BUFFER, GENERIC_READ, GENERIC_WRITE, HANDLE, INVALID_HANDLE_VALUE, LocalFree, +}; +use windows_sys::Win32::Security::Authorization::{ + ConvertSidToStringSidW, ConvertStringSecurityDescriptorToSecurityDescriptorW, SDDL_REVISION_1, +}; +use windows_sys::Win32::Security::{ + GetTokenInformation, PSECURITY_DESCRIPTOR, SECURITY_ATTRIBUTES, TOKEN_QUERY, TOKEN_USER, + TokenUser, +}; +use windows_sys::Win32::Storage::FileSystem::{ + CREATE_NEW, CreateFileW, FILE_ATTRIBUTE_TEMPORARY, FILE_SHARE_DELETE, FILE_SHARE_READ, + FILE_SHARE_WRITE, +}; +use windows_sys::Win32::System::Threading::{GetCurrentProcess, OpenProcessToken}; + +struct CurrentUserSecurity { + descriptor: PSECURITY_DESCRIPTOR, +} + +impl CurrentUserSecurity { + fn new() -> io::Result { + let sid = current_user_sid()?; + let sddl = wide(OsStr::new(&format!("D:P(A;;GA;;;{sid})"))); + let mut descriptor: PSECURITY_DESCRIPTOR = ptr::null_mut(); + // SAFETY: the SDDL is terminated and descriptor is writable output storage. + if unsafe { + ConvertStringSecurityDescriptorToSecurityDescriptorW( + sddl.as_ptr(), + SDDL_REVISION_1, + &raw mut descriptor, + ptr::null_mut(), + ) + } == 0 + { + return Err(io::Error::last_os_error()); + } + Ok(Self { descriptor }) + } + + fn attributes(&mut self) -> io::Result { + Ok(SECURITY_ATTRIBUTES { + nLength: u32::try_from(mem::size_of::()) + .map_err(|_| io::Error::other("SECURITY_ATTRIBUTES size is not representable"))?, + lpSecurityDescriptor: self.descriptor.cast(), + bInheritHandle: 0, + }) + } +} + +impl Drop for CurrentUserSecurity { + fn drop(&mut self) { + if !self.descriptor.is_null() { + // SAFETY: the descriptor was allocated by the SDDL conversion API. + unsafe { LocalFree(self.descriptor.cast()) }; + } + } +} + +pub fn create_current_user_pipe(name: &str, first_instance: bool) -> io::Result { + let mut security = CurrentUserSecurity::new()?; + let mut attributes = security.attributes()?; + let mut options = ServerOptions::new(); + options + .pipe_mode(PipeMode::Byte) + .max_instances(16) + .reject_remote_clients(true) + .first_pipe_instance(first_instance); + // SAFETY: attributes and its descriptor remain alive for the complete creation call. + unsafe { + options.create_with_security_attributes_raw(name, (&raw mut attributes).cast::()) + } +} + +pub fn create_current_user_file(path: &Path) -> io::Result { + let mut security = CurrentUserSecurity::new()?; + let mut attributes = security.attributes()?; + let path = wide(path.as_os_str()); + // SAFETY: all pointers are valid for the duration of CreateFileW and the + // security descriptor is owned by `security` until the call returns. + let handle = unsafe { + CreateFileW( + path.as_ptr(), + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + &raw mut attributes, + CREATE_NEW, + FILE_ATTRIBUTE_TEMPORARY, + ptr::null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + return Err(io::Error::last_os_error()); + } + // SAFETY: CreateFileW returned a new owned kernel handle. + Ok(unsafe { File::from_raw_handle(handle as RawHandle) }) +} + +fn current_user_sid() -> io::Result { + let mut token: HANDLE = ptr::null_mut(); + // SAFETY: token points to writable storage and the process pseudo-handle is valid. + if unsafe { OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &raw mut token) } == 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: OpenProcessToken returned an owned token handle. + let token = unsafe { File::from_raw_handle(token as RawHandle) }; + let mut required = 0_u32; + // SAFETY: a null buffer with zero length is the documented size query. + unsafe { + GetTokenInformation( + token.as_raw_handle() as HANDLE, + TokenUser, + ptr::null_mut(), + 0, + &raw mut required, + ) + }; + let size_error = io::Error::last_os_error(); + if size_error.raw_os_error().map(i32::cast_unsigned) != Some(ERROR_INSUFFICIENT_BUFFER) + || required == 0 + { + return Err(size_error); + } + let words = (required as usize).div_ceil(mem::size_of::()); + let mut buffer = vec![0_usize; words]; + // SAFETY: buffer contains at least `required` writable bytes. + if unsafe { + GetTokenInformation( + token.as_raw_handle() as HANDLE, + TokenUser, + buffer.as_mut_ptr().cast::(), + required, + &raw mut required, + ) + } == 0 + { + return Err(io::Error::last_os_error()); + } + // SAFETY: TokenUser queries return a TOKEN_USER at the start of the buffer. + let token_user = unsafe { &*buffer.as_ptr().cast::() }; + let mut sid_string = ptr::null_mut(); + // SAFETY: the SID is owned by the token information buffer and output is writable. + if unsafe { ConvertSidToStringSidW(token_user.User.Sid, &raw mut sid_string) } == 0 { + return Err(io::Error::last_os_error()); + } + let mut length = 0_usize; + // SAFETY: ConvertSidToStringSidW returns a terminated LocalAlloc string. + unsafe { + while *sid_string.add(length) != 0 { + length += 1; + } + } + // SAFETY: the preceding scan established the initialized UTF-16 range. + let sid = String::from_utf16(unsafe { std::slice::from_raw_parts(sid_string, length) }) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "current SID is invalid")); + // SAFETY: the SID string was allocated by ConvertSidToStringSidW. + unsafe { LocalFree(sid_string.cast()) }; + sid +} + +fn wide(value: &OsStr) -> Vec { + value.encode_wide().chain([0]).collect() +} diff --git a/client/crates/client-core/src/display.rs b/client/crates/client-core/src/display.rs index 9ad2ca0..286e8b2 100644 --- a/client/crates/client-core/src/display.rs +++ b/client/crates/client-core/src/display.rs @@ -1065,7 +1065,7 @@ mod tests { ); request.gpu_policy = GpuPolicy { selection: GpuSelection::Manual(adapter("gpu-1")), - zero_copy_policy: ZeroCopyPolicy::Compatibility, + zero_copy_policy: ZeroCopyPolicy::RequiredEndToEnd, }; request.decoder_adapter = adapter("gpu-1"); let plan = planner.plan(request).unwrap(); diff --git a/client/crates/client-core/src/gpu.rs b/client/crates/client-core/src/gpu.rs index baf3838..051796d 100644 --- a/client/crates/client-core/src/gpu.rs +++ b/client/crates/client-core/src/gpu.rs @@ -220,7 +220,7 @@ mod tests { let plan = plan_d3d11_pipeline(PipelineRequest { policy: GpuPolicy { selection: GpuSelection::WindowDisplayAdapter, - zero_copy_policy: ZeroCopyPolicy::Compatibility, + zero_copy_policy: ZeroCopyPolicy::RequiredEndToEnd, }, window_display_adapter: adapter("render"), decoder_adapter: adapter("decode"), @@ -229,7 +229,7 @@ mod tests { .unwrap(); assert_eq!(plan.planned_memory_path, MemoryPathStatus::CrossAdapterCopy); - assert_eq!(plan.zero_copy_policy, ZeroCopyPolicy::Compatibility); + assert_eq!(plan.zero_copy_policy, ZeroCopyPolicy::RequiredEndToEnd); assert!(!plan.is_zero_copy_candidate()); } @@ -238,7 +238,7 @@ mod tests { let plan = plan_d3d11_pipeline(PipelineRequest { policy: GpuPolicy { selection: GpuSelection::Manual(adapter("manual-gpu")), - zero_copy_policy: ZeroCopyPolicy::Compatibility, + zero_copy_policy: ZeroCopyPolicy::RequiredEndToEnd, }, window_display_adapter: adapter("display-gpu"), decoder_adapter: adapter("manual-gpu"), diff --git a/client/helpers/native-video/src/lib.rs b/client/helpers/native-video/src/lib.rs new file mode 100644 index 0000000..058c972 --- /dev/null +++ b/client/helpers/native-video/src/lib.rs @@ -0,0 +1,2 @@ +#[cfg(windows)] +pub mod stream; diff --git a/client/helpers/native-video/src/main.rs b/client/helpers/native-video/src/main.rs index 2b6dc22..94829f6 100644 --- a/client/helpers/native-video/src/main.rs +++ b/client/helpers/native-video/src/main.rs @@ -112,7 +112,6 @@ fn parse_args(mut args: impl Iterator) -> Result zero_copy_policy = ZeroCopyPolicy::RequiredEndToEnd, - "--compatibility" => zero_copy_policy = ZeroCopyPolicy::Compatibility, "--help" | "-h" => return Err(usage().to_owned()), other => return Err(format!("unknown argument {other}")), } @@ -141,7 +140,7 @@ fn required_value(args: &mut impl Iterator, option: &str) -> Resu } fn usage() -> &'static str { - "usage: remotedesk-native-video --capabilities | --probe-d3d11 | --probe-h264-decoder | --probe-h264-file | --play-h264-file | --dry-run --window-adapter [--decode-adapter ] [--manual-adapter ] [--required-end-to-end|--compatibility]" + "usage: remotedesk-native-video --capabilities | --probe-d3d11 | --probe-h264-decoder | --probe-h264-file | --play-h264-file | --dry-run --window-adapter [--decode-adapter ] [--manual-adapter ]" } #[derive(serde::Serialize)] @@ -817,7 +816,7 @@ mod tests { "gpu-0", "--manual-adapter", "gpu-1", - "--compatibility", + "--required-end-to-end", ])) .unwrap(); @@ -841,12 +840,12 @@ mod tests { "--dry-run", "--window-adapter", "gpu-0", - "--compatibility", + "--required-end-to-end", ])) .unwrap(); assert_eq!( compatibility.zero_copy_policy, - ZeroCopyPolicy::Compatibility + ZeroCopyPolicy::RequiredEndToEnd ); } diff --git a/client/helpers/native-video/src/stream.rs b/client/helpers/native-video/src/stream.rs new file mode 100644 index 0000000..396be4f --- /dev/null +++ b/client/helpers/native-video/src/stream.rs @@ -0,0 +1,797 @@ +use core::mem::ManuallyDrop; +use std::sync::Arc; +use std::time::Instant; + +use raw_window_handle::{HasWindowHandle as _, RawWindowHandle}; +use windows::Win32::Foundation::{HWND, RECT}; +use windows::Win32::Graphics::Direct3D::{ + D3D_DRIVER_TYPE_HARDWARE, D3D_FEATURE_LEVEL_11_0, D3D_FEATURE_LEVEL_11_1, +}; +use windows::Win32::Graphics::Direct3D11::{ + D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_SDK_VERSION, + D3D11_TEX2D_VPIV, D3D11_TEX2D_VPOV, D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE, + D3D11_VIDEO_PROCESSOR_COLOR_SPACE, D3D11_VIDEO_PROCESSOR_CONTENT_DESC, + D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_INPUT, D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT, + D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC, D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC_0, + D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC, D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC_0, + D3D11_VIDEO_PROCESSOR_STREAM, D3D11_VIDEO_USAGE_PLAYBACK_NORMAL, + D3D11_VPIV_DIMENSION_TEXTURE2D, D3D11_VPOV_DIMENSION_TEXTURE2D, D3D11CreateDevice, + ID3D11Device, ID3D11DeviceContext, ID3D11RenderTargetView, ID3D11Texture2D, ID3D11VideoContext, + ID3D11VideoDevice, ID3D11VideoProcessor, ID3D11VideoProcessorEnumerator, + ID3D11VideoProcessorOutputView, +}; +use windows::Win32::Graphics::Dxgi::Common::{ + DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_NV12, DXGI_FORMAT_UNKNOWN, DXGI_MODE_DESC, + DXGI_RATIONAL, DXGI_SAMPLE_DESC, +}; +use windows::Win32::Graphics::Dxgi::{ + DXGI_MWA_NO_ALT_ENTER, DXGI_PRESENT, DXGI_SWAP_CHAIN_DESC, DXGI_SWAP_CHAIN_FLAG, + DXGI_SWAP_EFFECT_DISCARD, DXGI_USAGE_RENDER_TARGET_OUTPUT, IDXGIAdapter, IDXGIDevice, + IDXGIFactory, IDXGISwapChain, +}; +use windows::Win32::Media::MediaFoundation::{ + CLSID_MSH264DecoderMFT, IMFAttributes, IMFDXGIBuffer, IMFDXGIDeviceManager, IMFMediaType, + IMFSample, IMFTransform, MF_E_NO_MORE_TYPES, MF_E_NOTACCEPTING, MF_E_TRANSFORM_NEED_MORE_INPUT, + MF_E_TRANSFORM_STREAM_CHANGE, MF_MT_FRAME_SIZE, MF_MT_MAJOR_TYPE, MF_MT_SUBTYPE, + MF_MT_VIDEO_NOMINAL_RANGE, MF_MT_YUV_MATRIX, MF_SA_D3D11_AWARE, MF_VERSION, + MFCreateDXGIDeviceManager, MFCreateMediaType, MFCreateMemoryBuffer, MFCreateSample, + MFMediaType_Video, MFNominalRange_0_255, MFSTARTUP_FULL, MFStartup, + MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, MFT_MESSAGE_NOTIFY_START_OF_STREAM, + MFT_MESSAGE_SET_D3D_MANAGER, MFT_OUTPUT_DATA_BUFFER, MFT_OUTPUT_STREAM_CAN_PROVIDE_SAMPLES, + MFT_OUTPUT_STREAM_PROVIDES_SAMPLES, MFVideoFormat_H264, MFVideoFormat_H264_ES, + MFVideoFormat_NV12, MFVideoTransferMatrix_BT709, +}; +use windows::Win32::System::Com::{ + CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED, CoCreateInstance, CoInitializeEx, +}; +use windows::core::Interface as _; +use winit::window::Window; + +const MAX_OUTPUT_TYPES: u32 = 64; +const MAX_OUTPUTS_PER_INPUT: usize = 8; + +pub struct H264AccessUnit { + pub data: Vec, + pub duration: std::time::Duration, +} + +pub struct NativeH264Renderer { + decoder: StreamingDecoder, + renderer: VideoRenderer, +} + +pub struct NativeVideoPresentation { + pub decode_latency_us: u64, + pub presentation_latency_us: u64, + pub frames: usize, + pub width: u32, + pub height: u32, +} + +impl NativeH264Renderer { + pub fn new(window: Arc) -> Result { + let decoder = StreamingDecoder::new()?; + let renderer = VideoRenderer::new(window, decoder.device.clone(), decoder.context.clone())?; + Ok(Self { decoder, renderer }) + } + + pub fn decode_and_present( + &mut self, + access_unit: &H264AccessUnit, + ) -> Result, String> { + let decode_started = Instant::now(); + let frames = self.decoder.decode(access_unit)?; + let decode_latency_us = elapsed_microseconds(decode_started); + if frames.is_empty() { + return Ok(None); + } + let presentation_started = Instant::now(); + let frame_count = frames.len(); + let (width, height) = frames + .last() + .map(|frame| (frame.width, frame.height)) + .expect("non-empty decoded frame list"); + for frame in frames { + self.renderer.present(&frame)?; + } + Ok(Some(NativeVideoPresentation { + decode_latency_us, + presentation_latency_us: elapsed_microseconds(presentation_started), + frames: frame_count, + width, + height, + })) + } +} + +struct ComGuard; + +impl Drop for ComGuard { + fn drop(&mut self) { + unsafe { windows::Win32::System::Com::CoUninitialize() }; + } +} + +struct MediaFoundationGuard; + +impl Drop for MediaFoundationGuard { + fn drop(&mut self) { + let _ = unsafe { windows::Win32::Media::MediaFoundation::MFShutdown() }; + } +} + +struct StreamingDecoder { + transform: IMFTransform, + _input_type: IMFMediaType, + _manager: IMFDXGIDeviceManager, + _attributes: IMFAttributes, + device: ID3D11Device, + context: ID3D11DeviceContext, + next_timestamp_100ns: i64, + _media_foundation_guard: MediaFoundationGuard, + _com_guard: ComGuard, +} + +impl StreamingDecoder { + fn new() -> Result { + unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) } + .ok() + .map_err(|error| format!("unable to initialize COM for remote H.264: {error}"))?; + let com_guard = ComGuard; + unsafe { MFStartup(MF_VERSION, MFSTARTUP_FULL) } + .map_err(|error| format!("unable to start Media Foundation: {error}"))?; + let media_foundation_guard = MediaFoundationGuard; + let (device, context) = create_hardware_d3d11_device()?; + let mut reset_token = 0_u32; + let mut manager = None; + unsafe { MFCreateDXGIDeviceManager(&raw mut reset_token, &raw mut manager) } + .map_err(|error| format!("unable to create remote-video DXGI manager: {error}"))?; + let manager = + manager.ok_or_else(|| "Media Foundation returned no DXGI manager".to_owned())?; + unsafe { manager.ResetDevice(&device, reset_token) } + .map_err(|error| format!("unable to bind the remote-video D3D11 device: {error}"))?; + + let transform: IMFTransform = + unsafe { CoCreateInstance(&CLSID_MSH264DecoderMFT, None, CLSCTX_INPROC_SERVER) } + .map_err(|error| { + format!("unable to create the Windows H.264 decoder MFT: {error}") + })?; + let attributes = unsafe { transform.GetAttributes() } + .map_err(|error| format!("unable to read H.264 decoder attributes: {error}"))?; + let d3d11_aware_key = MF_SA_D3D11_AWARE; + if unsafe { attributes.GetUINT32(&d3d11_aware_key) }.unwrap_or_default() != 1 { + return Err("the Windows H.264 decoder is not D3D11-aware".to_owned()); + } + unsafe { transform.ProcessMessage(MFT_MESSAGE_SET_D3D_MANAGER, manager.as_raw() as usize) } + .map_err(|error| format!("unable to attach D3D11 to the H.264 decoder: {error}"))?; + + let input_type = select_h264_input(&transform)?; + select_nv12_output(&transform)?; + let output_info = unsafe { transform.GetOutputStreamInfo(0) }.map_err(|error| { + format!("unable to read H.264 decoder output requirements: {error}") + })?; + let sample_flags = u32::try_from( + (MFT_OUTPUT_STREAM_PROVIDES_SAMPLES.0 | MFT_OUTPUT_STREAM_CAN_PROVIDE_SAMPLES.0).max(0), + ) + .unwrap_or_default(); + if output_info.dwFlags & sample_flags == 0 { + return Err("the D3D11 H.264 decoder requires CPU-allocated output samples".to_owned()); + } + unsafe { transform.ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0) } + .map_err(|error| format!("unable to begin H.264 streaming: {error}"))?; + unsafe { transform.ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0) } + .map_err(|error| format!("unable to start H.264 input: {error}"))?; + Ok(Self { + transform, + _input_type: input_type, + _manager: manager, + _attributes: attributes, + device, + context, + next_timestamp_100ns: 0, + _media_foundation_guard: media_foundation_guard, + _com_guard: com_guard, + }) + } + + fn decode(&mut self, access_unit: &H264AccessUnit) -> Result, String> { + if access_unit.data.is_empty() || access_unit.data.len() > 16 * 1024 * 1024 { + return Err("remote H.264 access unit is outside the decoder bounds".to_owned()); + } + let mut frames = self.drain_output()?; + let input = create_input_sample( + &access_unit.data, + self.next_timestamp_100ns, + access_unit.duration, + )?; + let duration_100ns = i64::try_from(access_unit.duration.as_nanos() / 100) + .map_err(|_| "remote H.264 duration overflow".to_owned())? + .max(1); + self.next_timestamp_100ns = self.next_timestamp_100ns.saturating_add(duration_100ns); + match unsafe { self.transform.ProcessInput(0, &input, 0) } { + Ok(()) => {} + Err(error) if error.code() == MF_E_NOTACCEPTING => { + frames.extend(self.drain_output()?); + unsafe { self.transform.ProcessInput(0, &input, 0) } + .map_err(|retry| format!("H.264 decoder still rejects input: {retry}"))?; + } + Err(error) => return Err(format!("H.264 decoder rejected input: {error}")), + } + frames.extend(self.drain_output()?); + Ok(frames) + } + + fn drain_output(&mut self) -> Result, String> { + let mut frames = Vec::new(); + for _ in 0..MAX_OUTPUTS_PER_INPUT { + let mut output = MFT_OUTPUT_DATA_BUFFER::default(); + output.pSample = ManuallyDrop::new(None); + let mut status = 0_u32; + let result = unsafe { + self.transform + .ProcessOutput(0, std::slice::from_mut(&mut output), &raw mut status) + }; + let sample = unsafe { ManuallyDrop::take(&mut output.pSample) }; + let _events = unsafe { ManuallyDrop::take(&mut output.pEvents) }; + match result { + Ok(()) => { + let sample = sample.ok_or_else(|| { + "H.264 decoder returned success without a DXGI sample".to_owned() + })?; + frames.push(decoded_frame(sample, &self.transform, &self.device)?); + } + Err(error) if error.code() == MF_E_TRANSFORM_NEED_MORE_INPUT => break, + Err(error) if error.code() == MF_E_TRANSFORM_STREAM_CHANGE => { + select_nv12_output(&self.transform)?; + } + Err(error) => return Err(format!("H.264 decoder output failed: {error}")), + } + } + Ok(frames) + } +} + +fn select_h264_input(transform: &IMFTransform) -> Result { + let mut last_error = None; + for subtype in [MFVideoFormat_H264_ES, MFVideoFormat_H264] { + let media_type = unsafe { MFCreateMediaType() } + .map_err(|error| format!("unable to create the H.264 input media type: {error}"))?; + let major_type_key = MF_MT_MAJOR_TYPE; + let video_type = MFMediaType_Video; + unsafe { media_type.SetGUID(&major_type_key, &video_type) } + .map_err(|error| format!("unable to set the H.264 input major type: {error}"))?; + let subtype_key = MF_MT_SUBTYPE; + unsafe { media_type.SetGUID(&subtype_key, &subtype) } + .map_err(|error| format!("unable to set the H.264 input subtype: {error}"))?; + match unsafe { transform.SetInputType(0, &media_type, 0) } { + Ok(()) => return Ok(media_type), + Err(error) => last_error = Some(error), + } + } + Err(format!( + "Windows H.264 decoder rejected Annex-B input: {}", + last_error + .map(|error| error.to_string()) + .unwrap_or_else(|| "no supported input subtype".to_owned()) + )) +} + +fn create_input_sample( + data: &[u8], + timestamp_100ns: i64, + duration: std::time::Duration, +) -> Result { + let length = u32::try_from(data.len()).map_err(|_| "H.264 input is too large".to_owned())?; + let buffer = unsafe { MFCreateMemoryBuffer(length) } + .map_err(|error| format!("unable to allocate H.264 input buffer: {error}"))?; + let mut destination = core::ptr::null_mut(); + unsafe { buffer.Lock(&raw mut destination, None, None) } + .map_err(|error| format!("unable to lock H.264 input buffer: {error}"))?; + if destination.is_null() { + let _ = unsafe { buffer.Unlock() }; + return Err("Media Foundation returned a null H.264 input buffer".to_owned()); + } + unsafe { core::ptr::copy_nonoverlapping(data.as_ptr(), destination, data.len()) }; + unsafe { buffer.Unlock() } + .map_err(|error| format!("unable to unlock H.264 input buffer: {error}"))?; + unsafe { buffer.SetCurrentLength(length) } + .map_err(|error| format!("unable to commit H.264 input bytes: {error}"))?; + let sample = unsafe { MFCreateSample() } + .map_err(|error| format!("unable to create H.264 input sample: {error}"))?; + unsafe { sample.AddBuffer(&buffer) } + .map_err(|error| format!("unable to attach H.264 input bytes: {error}"))?; + unsafe { sample.SetSampleTime(timestamp_100ns) } + .map_err(|error| format!("unable to set H.264 sample time: {error}"))?; + let duration_100ns = i64::try_from(duration.as_nanos() / 100) + .map_err(|_| "H.264 sample duration overflow".to_owned())? + .max(1); + unsafe { sample.SetSampleDuration(duration_100ns) } + .map_err(|error| format!("unable to set H.264 sample duration: {error}"))?; + Ok(sample) +} + +fn select_nv12_output(transform: &IMFTransform) -> Result<(), String> { + for index in 0..MAX_OUTPUT_TYPES { + let media_type = match unsafe { transform.GetOutputAvailableType(0, index) } { + Ok(media_type) => media_type, + Err(error) if error.code() == MF_E_NO_MORE_TYPES => break, + Err(error) => { + return Err(format!( + "unable to enumerate decoder output {index}: {error}" + )); + } + }; + let subtype_key = MF_MT_SUBTYPE; + let subtype = unsafe { media_type.GetGUID(&subtype_key) }; + if subtype.is_ok_and(|value| value == MFVideoFormat_NV12) + && unsafe { transform.SetOutputType(0, &media_type, 0) }.is_ok() + { + return Ok(()); + } + } + Err("the Windows H.264 decoder exposes no NV12 output".to_owned()) +} + +struct DecodedFrame { + _sample: IMFSample, + texture: ID3D11Texture2D, + subresource: u32, + mip_levels: u32, + width: u32, + height: u32, + input_color_space: D3D11_VIDEO_PROCESSOR_COLOR_SPACE, +} + +fn decoded_frame( + sample: IMFSample, + transform: &IMFTransform, + expected_device: &ID3D11Device, +) -> Result { + let buffer_count = unsafe { sample.GetBufferCount() } + .map_err(|error| format!("unable to count decoded H.264 buffers: {error}"))?; + if buffer_count != 1 { + return Err("decoded H.264 frame must contain one DXGI buffer".to_owned()); + } + let buffer = unsafe { sample.GetBufferByIndex(0) } + .map_err(|error| format!("unable to read decoded H.264 buffer: {error}"))?; + let dxgi_buffer = buffer + .cast::() + .map_err(|_| "H.264 decoder returned a CPU media buffer".to_owned())?; + let mut raw_texture = core::ptr::null_mut(); + unsafe { dxgi_buffer.GetResource(&ID3D11Texture2D::IID, &raw mut raw_texture) } + .map_err(|error| format!("decoded H.264 buffer has no D3D11 texture: {error}"))?; + if raw_texture.is_null() { + return Err("decoded H.264 buffer returned a null texture".to_owned()); + } + let texture = unsafe { ID3D11Texture2D::from_raw(raw_texture) }; + let mut descriptor = windows::Win32::Graphics::Direct3D11::D3D11_TEXTURE2D_DESC::default(); + unsafe { texture.GetDesc(&raw mut descriptor) }; + if descriptor.Format != DXGI_FORMAT_NV12 || descriptor.Width == 0 || descriptor.Height == 0 { + return Err("decoded H.264 texture is not NV12".to_owned()); + } + let texture_device = unsafe { texture.GetDevice() } + .map_err(|error| format!("unable to read decoded texture device: {error}"))?; + if texture_device != *expected_device { + return Err("decoded H.264 texture belongs to another D3D11 device".to_owned()); + } + let subresource = unsafe { dxgi_buffer.GetSubresourceIndex() } + .map_err(|error| format!("unable to read decoded H.264 subresource: {error}"))?; + let subresources = descriptor.MipLevels.saturating_mul(descriptor.ArraySize); + if descriptor.MipLevels == 0 || subresources == 0 || subresource >= subresources { + return Err("decoded H.264 subresource is outside its texture".to_owned()); + } + let media_type = unsafe { transform.GetOutputCurrentType(0) } + .map_err(|error| format!("unable to read current H.264 output type: {error}"))?; + let (width, height) = + media_type_frame_size(&media_type).unwrap_or((descriptor.Width, descriptor.Height)); + if width > descriptor.Width || height > descriptor.Height { + return Err("visible H.264 frame exceeds its NV12 texture".to_owned()); + } + Ok(DecodedFrame { + _sample: sample, + texture, + subresource, + mip_levels: descriptor.MipLevels, + width, + height, + input_color_space: media_type_color_space(&media_type, height), + }) +} + +fn create_hardware_d3d11_device() -> Result<(ID3D11Device, ID3D11DeviceContext), String> { + let levels = [D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0]; + let mut device = None; + let mut context = None; + unsafe { + D3D11CreateDevice( + None::<&IDXGIAdapter>, + D3D_DRIVER_TYPE_HARDWARE, + windows::Win32::Foundation::HMODULE::default(), + D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT, + Some(&levels), + D3D11_SDK_VERSION, + Some(&raw mut device), + None, + Some(&raw mut context), + ) + } + .map_err(|error| format!("unable to create remote-video D3D11 device: {error}"))?; + Ok(( + device.ok_or_else(|| "D3D11 returned no remote-video device".to_owned())?, + context.ok_or_else(|| "D3D11 returned no remote-video context".to_owned())?, + )) +} + +struct VideoRenderer { + window: Arc, + device: ID3D11Device, + context: ID3D11DeviceContext, + video_device: ID3D11VideoDevice, + video_context: ID3D11VideoContext, + swap_chain: IDXGISwapChain, + pipeline: Option, + output_size: (u32, u32), + output_frame: u32, +} + +struct ProcessorPipeline { + input_size: (u32, u32), + output_size: (u32, u32), + enumerator: ID3D11VideoProcessorEnumerator, + processor: ID3D11VideoProcessor, + output_view: ID3D11VideoProcessorOutputView, + render_target: ID3D11RenderTargetView, +} + +impl VideoRenderer { + fn new( + window: Arc, + device: ID3D11Device, + context: ID3D11DeviceContext, + ) -> Result { + let RawWindowHandle::Win32(handle) = window + .window_handle() + .map_err(|error| format!("unable to read remote-video window handle: {error}"))? + .as_raw() + else { + return Err("remote H.264 presentation requires a Win32 window".to_owned()); + }; + let hwnd = HWND(handle.hwnd.get() as *mut core::ffi::c_void); + let size = window.inner_size(); + let width = size.width.max(1); + let height = size.height.max(1); + let descriptor = swap_chain_descriptor(hwnd, width, height); + let dxgi_device: IDXGIDevice = device + .cast() + .map_err(|error| format!("remote-video device has no DXGI interface: {error}"))?; + let adapter = unsafe { dxgi_device.GetAdapter() } + .map_err(|error| format!("unable to read remote-video adapter: {error}"))?; + let factory: IDXGIFactory = unsafe { adapter.GetParent() } + .map_err(|error| format!("unable to read remote-video DXGI factory: {error}"))?; + unsafe { factory.MakeWindowAssociation(hwnd, DXGI_MWA_NO_ALT_ENTER) } + .map_err(|error| format!("unable to configure remote-video window: {error}"))?; + let mut swap_chain = None; + unsafe { factory.CreateSwapChain(&device, &raw const descriptor, &raw mut swap_chain) } + .ok() + .map_err(|error| format!("unable to create remote-video swap chain: {error}"))?; + Ok(Self { + window, + video_device: device + .cast() + .map_err(|error| format!("D3D11 device has no video interface: {error}"))?, + video_context: context + .cast() + .map_err(|error| format!("D3D11 context has no video interface: {error}"))?, + device, + context, + swap_chain: swap_chain.ok_or_else(|| "DXGI returned no swap chain".to_owned())?, + pipeline: None, + output_size: (width, height), + output_frame: 0, + }) + } + + fn present(&mut self, frame: &DecodedFrame) -> Result<(), String> { + let size = self.window.inner_size(); + if size.width == 0 || size.height == 0 { + return Ok(()); + } + self.resize(size.width, size.height)?; + self.ensure_pipeline(frame.width, frame.height)?; + let pipeline = self + .pipeline + .as_ref() + .ok_or_else(|| "video processor is missing".to_owned())?; + let input_descriptor = D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC { + FourCC: 0, + ViewDimension: D3D11_VPIV_DIMENSION_TEXTURE2D, + Anonymous: D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC_0 { + Texture2D: D3D11_TEX2D_VPIV { + MipSlice: frame.subresource % frame.mip_levels, + ArraySlice: frame.subresource / frame.mip_levels, + }, + }, + }; + let mut input_view = None; + unsafe { + self.video_device.CreateVideoProcessorInputView( + &frame.texture, + &pipeline.enumerator, + &raw const input_descriptor, + Some(&raw mut input_view), + ) + } + .map_err(|error| format!("unable to create NV12 input view: {error}"))?; + let source = RECT { + left: 0, + top: 0, + right: i32::try_from(frame.width).map_err(|_| "video width overflow")?, + bottom: i32::try_from(frame.height).map_err(|_| "video height overflow")?, + }; + let destination = aspect_fit_rect(frame.width, frame.height, size.width, size.height)?; + let target = RECT { + left: 0, + top: 0, + right: i32::try_from(size.width).map_err(|_| "window width overflow")?, + bottom: i32::try_from(size.height).map_err(|_| "window height overflow")?, + }; + unsafe { + self.context + .ClearRenderTargetView(&pipeline.render_target, &[0.0, 0.0, 0.0, 1.0]); + self.video_context.VideoProcessorSetOutputTargetRect( + &pipeline.processor, + true, + Some(&raw const target), + ); + self.video_context.VideoProcessorSetStreamSourceRect( + &pipeline.processor, + 0, + true, + Some(&raw const source), + ); + self.video_context.VideoProcessorSetStreamDestRect( + &pipeline.processor, + 0, + true, + Some(&raw const destination), + ); + self.video_context.VideoProcessorSetStreamColorSpace( + &pipeline.processor, + 0, + &raw const frame.input_color_space, + ); + } + let mut stream = D3D11_VIDEO_PROCESSOR_STREAM { + Enable: true.into(), + pInputSurface: ManuallyDrop::new(input_view), + ..Default::default() + }; + let result = unsafe { + self.video_context.VideoProcessorBlt( + &pipeline.processor, + &pipeline.output_view, + self.output_frame, + std::slice::from_ref(&stream), + ) + }; + let _input_view = unsafe { ManuallyDrop::take(&mut stream.pInputSurface) }; + result.map_err(|error| format!("unable to process remote NV12 frame: {error}"))?; + unsafe { self.swap_chain.Present(1, DXGI_PRESENT(0)) } + .ok() + .map_err(|error| format!("unable to present remote H.264 frame: {error}"))?; + self.output_frame = self.output_frame.wrapping_add(1); + Ok(()) + } + + fn resize(&mut self, width: u32, height: u32) -> Result<(), String> { + if self.output_size == (width, height) { + return Ok(()); + } + self.pipeline = None; + unsafe { + self.swap_chain.ResizeBuffers( + 0, + width, + height, + DXGI_FORMAT_UNKNOWN, + DXGI_SWAP_CHAIN_FLAG(0), + ) + } + .map_err(|error| format!("unable to resize remote-video swap chain: {error}"))?; + self.output_size = (width, height); + Ok(()) + } + + fn ensure_pipeline(&mut self, input_width: u32, input_height: u32) -> Result<(), String> { + if self.pipeline.as_ref().is_some_and(|pipeline| { + pipeline.input_size == (input_width, input_height) + && pipeline.output_size == self.output_size + }) { + return Ok(()); + } + let content = D3D11_VIDEO_PROCESSOR_CONTENT_DESC { + InputFrameFormat: D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE, + InputFrameRate: DXGI_RATIONAL { + Numerator: 60, + Denominator: 1, + }, + InputWidth: input_width, + InputHeight: input_height, + OutputFrameRate: DXGI_RATIONAL { + Numerator: 60, + Denominator: 1, + }, + OutputWidth: self.output_size.0, + OutputHeight: self.output_size.1, + Usage: D3D11_VIDEO_USAGE_PLAYBACK_NORMAL, + }; + let enumerator = unsafe { + self.video_device + .CreateVideoProcessorEnumerator(&raw const content) + } + .map_err(|error| format!("unable to create video processor enumerator: {error}"))?; + let input_support = unsafe { enumerator.CheckVideoProcessorFormat(DXGI_FORMAT_NV12) } + .map_err(|error| format!("unable to verify NV12 processor input: {error}"))?; + let output_support = + unsafe { enumerator.CheckVideoProcessorFormat(DXGI_FORMAT_B8G8R8A8_UNORM) } + .map_err(|error| format!("unable to verify BGRA processor output: {error}"))?; + let required_input = u32::try_from(D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_INPUT.0) + .map_err(|_| "invalid D3D11 input support flag".to_owned())?; + let required_output = u32::try_from(D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT.0) + .map_err(|_| "invalid D3D11 output support flag".to_owned())?; + if input_support & required_input == 0 || output_support & required_output == 0 { + return Err("D3D11 VideoProcessor lacks NV12/BGRA support".to_owned()); + } + let processor = unsafe { self.video_device.CreateVideoProcessor(&enumerator, 0) } + .map_err(|error| format!("unable to create D3D11 video processor: {error}"))?; + let back_buffer: ID3D11Texture2D = unsafe { self.swap_chain.GetBuffer(0) } + .map_err(|error| format!("unable to read remote-video back buffer: {error}"))?; + let mut render_target = None; + unsafe { + self.device + .CreateRenderTargetView(&back_buffer, None, Some(&raw mut render_target)) + } + .map_err(|error| format!("unable to create remote-video render target: {error}"))?; + let render_target = + render_target.ok_or_else(|| "D3D11 returned no render target view".to_owned())?; + let output_descriptor = D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC { + ViewDimension: D3D11_VPOV_DIMENSION_TEXTURE2D, + Anonymous: D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC_0 { + Texture2D: D3D11_TEX2D_VPOV { MipSlice: 0 }, + }, + }; + let mut output_view = None; + unsafe { + self.video_device.CreateVideoProcessorOutputView( + &back_buffer, + &enumerator, + &raw const output_descriptor, + Some(&raw mut output_view), + ) + } + .map_err(|error| format!("unable to create processor output view: {error}"))?; + self.pipeline = Some(ProcessorPipeline { + input_size: (input_width, input_height), + output_size: self.output_size, + enumerator, + processor, + output_view: output_view + .ok_or_else(|| "D3D11 returned no processor output view".to_owned())?, + render_target, + }); + Ok(()) + } +} + +fn swap_chain_descriptor(hwnd: HWND, width: u32, height: u32) -> DXGI_SWAP_CHAIN_DESC { + DXGI_SWAP_CHAIN_DESC { + BufferDesc: DXGI_MODE_DESC { + Width: width, + Height: height, + RefreshRate: DXGI_RATIONAL { + Numerator: 0, + Denominator: 1, + }, + Format: DXGI_FORMAT_B8G8R8A8_UNORM, + ..Default::default() + }, + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + BufferUsage: DXGI_USAGE_RENDER_TARGET_OUTPUT, + BufferCount: 2, + OutputWindow: hwnd, + Windowed: true.into(), + SwapEffect: DXGI_SWAP_EFFECT_DISCARD, + Flags: 0, + } +} + +fn aspect_fit_rect( + source_width: u32, + source_height: u32, + target_width: u32, + target_height: u32, +) -> Result { + if source_width == 0 || source_height == 0 || target_width == 0 || target_height == 0 { + return Err("video dimensions must be non-zero".to_owned()); + } + let source_aspect = f64::from(source_width) / f64::from(source_height); + let target_aspect = f64::from(target_width) / f64::from(target_height); + let (width, height) = if source_aspect > target_aspect { + ( + target_width, + (f64::from(target_width) / source_aspect).round() as u32, + ) + } else { + ( + (f64::from(target_height) * source_aspect).round() as u32, + target_height, + ) + }; + let left = (target_width - width) / 2; + let top = (target_height - height) / 2; + Ok(RECT { + left: i32::try_from(left).map_err(|_| "video left overflow")?, + top: i32::try_from(top).map_err(|_| "video top overflow")?, + right: i32::try_from(left + width).map_err(|_| "video right overflow")?, + bottom: i32::try_from(top + height).map_err(|_| "video bottom overflow")?, + }) +} + +fn media_type_frame_size(media_type: &IMFMediaType) -> Option<(u32, u32)> { + let frame_size_key = MF_MT_FRAME_SIZE; + let packed = unsafe { media_type.GetUINT64(&frame_size_key) }.ok()?; + let width = u32::try_from(packed >> 32).ok()?; + let height = u32::try_from(packed & u64::from(u32::MAX)).ok()?; + (width > 0 && height > 0).then_some((width, height)) +} + +fn media_type_color_space( + media_type: &IMFMediaType, + visible_height: u32, +) -> D3D11_VIDEO_PROCESSOR_COLOR_SPACE { + let bt709_value = u32::try_from(MFVideoTransferMatrix_BT709.0).unwrap_or(1); + let full_range_value = u32::try_from(MFNominalRange_0_255.0).unwrap_or(1); + let yuv_matrix_key = MF_MT_YUV_MATRIX; + let bt709 = unsafe { media_type.GetUINT32(&yuv_matrix_key) } + .map_or(visible_height >= 720, |value| value == bt709_value); + let nominal_range_key = MF_MT_VIDEO_NOMINAL_RANGE; + let full_range = unsafe { media_type.GetUINT32(&nominal_range_key) } + .is_ok_and(|value| value == full_range_value); + D3D11_VIDEO_PROCESSOR_COLOR_SPACE { + _bitfield: (u32::from(bt709) << 2) | ((if full_range { 2 } else { 1 }) << 4), + } +} + +fn elapsed_microseconds(started: Instant) -> u64 { + started.elapsed().as_micros().try_into().unwrap_or(u64::MAX) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn aspect_fit_centers_letterbox_and_pillarbox_output() { + assert_eq!( + aspect_fit_rect(1_920, 1_080, 1_000, 1_000).unwrap(), + RECT { + left: 0, + top: 218, + right: 1_000, + bottom: 781 + } + ); + assert_eq!( + aspect_fit_rect(1_000, 1_000, 1_920, 1_080).unwrap(), + RECT { + left: 420, + top: 0, + right: 1_500, + bottom: 1_080 + } + ); + } +} diff --git a/client/helpers/windows-agent-viewer/Cargo.toml b/client/helpers/windows-agent-viewer/Cargo.toml index 3e0d2d8..62ae382 100644 --- a/client/helpers/windows-agent-viewer/Cargo.toml +++ b/client/helpers/windows-agent-viewer/Cargo.toml @@ -7,12 +7,16 @@ rust-version.workspace = true [dependencies] flate2 = "1.1" +remotedesk-native-video = { path = "../native-video" } +remotedesk-protocol = { path = "../../../protocol" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" softbuffer = "0.4" winit = "0.30" [target.'cfg(windows)'.dependencies] +cpal = "0.17.3" +opus2 = { version = "0.4.0", features = ["bundled"] } windows = { version = "0.62.2", features = ["Win32_Foundation", "Win32_UI_WindowsAndMessaging"] } [lints.rust] diff --git a/client/helpers/windows-agent-viewer/src/audio.rs b/client/helpers/windows-agent-viewer/src/audio.rs new file mode 100644 index 0000000..177435f --- /dev/null +++ b/client/helpers/windows-agent-viewer/src/audio.rs @@ -0,0 +1,633 @@ +//! Bounded RDA1 jitter buffering, Opus decoding, and WASAPI playback. +//! +//! Audio is independent from the video event mailbox. A slow or unavailable +//! audio device only disables this worker; it never terminates video or input. + +use remotedesk_protocol::AudioPacket; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +const AUDIO_TARGET_BUFFER: Duration = Duration::from_millis(60); +const VIDEO_LATE_TOLERANCE_100NS: u64 = 800_000; +const AUDIO_SYNC_DEADBAND_100NS: i64 = 50_000; +const AUDIO_RATE_LIMIT_PPM: i32 = 5_000; + +/// Shared media timeline used to make the video path follow audio playout. +#[derive(Debug)] +pub(crate) struct MediaClock { + base_audio_pts: Option, + base_local_time: Option, + last_audio_pts: Option, + source_rate_adjustment_ppm: i32, + buffer_rate_adjustment_ppm: i32, +} + +impl MediaClock { + pub(crate) fn new() -> Self { + Self { + base_audio_pts: None, + base_local_time: None, + last_audio_pts: None, + source_rate_adjustment_ppm: 0, + buffer_rate_adjustment_ppm: 0, + } + } + + pub(crate) fn observe_audio(&mut self, pts: u64, duration_ms: u16) { + if self.base_audio_pts.is_none() { + self.base_audio_pts = Some(pts); + self.base_local_time = Some(Instant::now() + AUDIO_TARGET_BUFFER); + } + let is_forward = self.last_audio_pts.is_none_or(|last_pts| pts > last_pts); + if is_forward { + if let Some(last_pts) = self.last_audio_pts { + let delta = pts.saturating_sub(last_pts); + let expected_duration = u64::from(duration_ms.max(1)).saturating_mul(10_000); + let error = i64::try_from(delta).unwrap_or(i64::MAX) + - i64::try_from(expected_duration).unwrap_or(i64::MAX); + if error.abs() <= AUDIO_SYNC_DEADBAND_100NS { + self.source_rate_adjustment_ppm = + self.source_rate_adjustment_ppm.saturating_mul(7) / 8; + } else { + let target = (error.saturating_mul(1_000_000) + / i64::try_from(expected_duration.max(1)).unwrap_or(1)) + .clamp( + -i64::from(AUDIO_RATE_LIMIT_PPM), + i64::from(AUDIO_RATE_LIMIT_PPM), + ); + let current = i64::from(self.source_rate_adjustment_ppm); + self.source_rate_adjustment_ppm = + i32::try_from(current + (target - current) / 8).unwrap_or(0); + } + } + self.last_audio_pts = Some(pts); + } + } + + pub(crate) fn reset_audio(&mut self, pts: u64) { + self.base_audio_pts = Some(pts); + self.base_local_time = Some(Instant::now() + AUDIO_TARGET_BUFFER); + self.last_audio_pts = Some(pts); + self.source_rate_adjustment_ppm = 0; + self.buffer_rate_adjustment_ppm = 0; + } + + pub(crate) fn update_buffer_rate_adjustment(&mut self, buffered_frames: usize) { + const TARGET_AUDIO_FRAMES: i64 = 2_880; + let error = i64::try_from(buffered_frames).unwrap_or(i64::MAX) - TARGET_AUDIO_FRAMES; + self.buffer_rate_adjustment_ppm = error.saturating_mul(2).clamp( + -i64::from(AUDIO_RATE_LIMIT_PPM), + i64::from(AUDIO_RATE_LIMIT_PPM), + ) as i32; + } + + pub(crate) fn rate_adjustment_ppm(&self) -> i32 { + self.source_rate_adjustment_ppm + .saturating_add(self.buffer_rate_adjustment_ppm) + .clamp(-AUDIO_RATE_LIMIT_PPM, AUDIO_RATE_LIMIT_PPM) + } + + fn audio_playout_pts(&self) -> Option { + let (Some(base_pts), Some(base_time)) = (self.base_audio_pts, self.base_local_time) else { + return None; + }; + let elapsed = Instant::now().saturating_duration_since(base_time); + let elapsed_100ns = elapsed.as_nanos().min(u128::from(u64::MAX) * 100) / 100; + base_pts.checked_add(u64::try_from(elapsed_100ns).unwrap_or(u64::MAX)) + } + + /// Returns false only for video frames already too far behind audio. + pub(crate) fn should_present_video(&self, video_pts: u64) -> bool { + self.audio_playout_pts().is_none_or(|audio_pts| { + video_pts.saturating_add(VIDEO_LATE_TOLERANCE_100NS) >= audio_pts + }) + } +} + +#[cfg(windows)] +mod windows_audio { + use super::*; + use cpal::SizedSample; + use cpal::traits::{DeviceTrait as _, HostTrait as _, StreamTrait as _}; + use opus2::{Channels, Decoder}; + use std::collections::{BTreeMap, VecDeque}; + use std::sync::atomic::{AtomicI32, Ordering}; + use std::sync::mpsc::Receiver; + + const MAX_JITTER_PACKETS: usize = 64; + const MAX_AUDIO_FRAMES: usize = 48_000; + const MAX_DECODE_SAMPLES_PER_CHANNEL: usize = 5_760; + + struct PlaybackBuffer { + frames: VecDeque<[f32; 2]>, + } + + impl PlaybackBuffer { + fn new() -> Self { + Self { + frames: VecDeque::with_capacity(MAX_AUDIO_FRAMES), + } + } + + fn push(&mut self, samples: &[f32]) { + if samples.len() / 2 > MAX_AUDIO_FRAMES { + return; + } + if self.frames.len() + samples.len() / 2 > MAX_AUDIO_FRAMES { + self.frames.clear(); + } + self.frames + .extend(samples.chunks_exact(2).map(|frame| [frame[0], frame[1]])); + } + + fn pop(&mut self) -> [f32; 2] { + self.frames.pop_front().unwrap_or([0.0; 2]) + } + + fn clear(&mut self) { + self.frames.clear(); + } + + fn len(&self) -> usize { + self.frames.len() + } + } + + struct JitterDecoder { + decoder: Decoder, + packets: BTreeMap, + expected: Option, + generation: u64, + stream_id: u32, + primed: bool, + last_decoded_sequence: Option, + } + + impl JitterDecoder { + fn new(packet: &AudioPacket) -> Result { + Ok(Self { + decoder: Decoder::new(48_000, Channels::Stereo) + .map_err(|error| format!("Opus decoder initialization failed: {error}"))?, + packets: BTreeMap::new(), + expected: Some(packet.sequence), + generation: packet.generation, + stream_id: packet.stream_id, + primed: false, + last_decoded_sequence: None, + }) + } + + fn reset(&mut self, packet: &AudioPacket) -> Result<(), String> { + self.decoder = Decoder::new(48_000, Channels::Stereo) + .map_err(|error| format!("Opus decoder reset failed: {error}"))?; + self.packets.clear(); + self.expected = Some(packet.sequence); + self.generation = packet.generation; + self.stream_id = packet.stream_id; + self.primed = false; + self.last_decoded_sequence = None; + Ok(()) + } + + fn push( + &mut self, + packet: AudioPacket, + output: &mut PlaybackBuffer, + clock: &Arc>, + rate_adjustment_ppm: &Arc, + ) -> Result<(), String> { + let reset = packet.stream_id != self.stream_id + || packet.generation != self.generation + || packet + .flags + .contains(remotedesk_protocol::AudioPacketFlags::DISCONTINUITY); + if reset { + self.reset(&packet)?; + } + let mut clock_guard = clock + .lock() + .map_err(|_| "audio clock lock poisoned".to_owned())?; + if reset { + output.clear(); + clock_guard.reset_audio(packet.pts); + rate_adjustment_ppm.store(0, Ordering::Release); + } + drop(clock_guard); + if self.packets.len() >= MAX_JITTER_PACKETS { + self.packets.pop_first(); + } + self.packets.entry(packet.sequence).or_insert(packet); + if !self.primed { + if self.packets.len() < 3 { + return Ok(()); + } + self.primed = true; + } + self.drain(output, clock)?; + let mut clock = clock + .lock() + .map_err(|_| "audio clock lock poisoned".to_owned())?; + clock.update_buffer_rate_adjustment(output.len()); + rate_adjustment_ppm.store(clock.rate_adjustment_ppm(), Ordering::Release); + Ok(()) + } + + fn drain( + &mut self, + output: &mut PlaybackBuffer, + clock: &Arc>, + ) -> Result<(), String> { + let Some(mut expected) = self.expected else { + return Ok(()); + }; + let mut missing_budget = 3; + loop { + if let Some(packet) = self.packets.remove(&expected) { + self.decode(&packet.payload, false, output)?; + let mut clock = clock + .lock() + .map_err(|_| "audio clock lock poisoned".to_owned())?; + if self + .last_decoded_sequence + .is_some_and(|sequence| sequence.saturating_add(1) == packet.sequence) + { + clock.observe_audio(packet.pts, packet.duration_ms); + } + self.last_decoded_sequence = Some(packet.sequence); + expected = expected.saturating_add(1); + missing_budget = 3; + continue; + } + let Some((&next, _)) = self.packets.first_key_value() else { + break; + }; + if next <= expected { + self.packets.remove(&next); + continue; + } + if missing_budget == 0 { + expected = next; + missing_budget = 3; + continue; + } + // The packet after the gap may contain in-band FEC for one + // missing packet. Fall back to Opus PLC if FEC is unavailable. + let fec_payload = (next == expected.saturating_add(1)) + .then(|| self.packets.get(&next).map(|packet| packet.payload.clone())) + .flatten(); + if fec_payload + .as_deref() + .is_none_or(|payload| self.decode(payload, true, output).is_err()) + { + self.decode(&[], false, output)?; + } + expected = expected.saturating_add(1); + missing_budget -= 1; + } + self.expected = Some(expected); + Ok(()) + } + + fn decode( + &mut self, + payload: &[u8], + fec: bool, + output: &mut PlaybackBuffer, + ) -> Result<(), String> { + let mut pcm = vec![0_i16; MAX_DECODE_SAMPLES_PER_CHANNEL * 2]; + let samples = self + .decoder + .decode(payload, &mut pcm, fec) + .map_err(|error| format!("Opus decode failed: {error}"))?; + let interleaved = pcm + .iter() + .take(samples.saturating_mul(2)) + .map(|sample| f32::from(*sample) / f32::from(i16::MAX)) + .collect::>(); + output.push(&interleaved); + Ok(()) + } + } + + pub(crate) fn start(receiver: Receiver, clock: Arc>) { + let _ = std::thread::Builder::new() + .name("windows-agent-opus-playback".to_owned()) + .spawn(move || { + if let Err(error) = run(receiver, clock) { + eprintln!("Windows Agent audio disabled: {error}"); + } + }); + } + + fn run(receiver: Receiver, clock: Arc>) -> Result<(), String> { + let host = cpal::default_host(); + let device = host + .default_output_device() + .ok_or_else(|| "WASAPI default output device unavailable".to_owned())?; + let supported = device + .default_output_config() + .map_err(|error| format!("WASAPI output configuration failed: {error}"))?; + let config = supported.config(); + let channels = usize::from(config.channels); + let output_rate = config.sample_rate; + if channels == 0 || channels > 8 { + return Err("WASAPI output channel count is unsupported".to_owned()); + } + let buffer = Arc::new(Mutex::new(PlaybackBuffer::new())); + let rate_adjustment_ppm = Arc::new(AtomicI32::new(0)); + let stream = match supported.sample_format() { + cpal::SampleFormat::I8 => build_output::( + &device, + &config, + channels, + output_rate, + Arc::clone(&buffer), + Arc::clone(&rate_adjustment_ppm), + )?, + cpal::SampleFormat::I16 => build_output::( + &device, + &config, + channels, + output_rate, + Arc::clone(&buffer), + Arc::clone(&rate_adjustment_ppm), + )?, + cpal::SampleFormat::I24 => build_output::( + &device, + &config, + channels, + output_rate, + Arc::clone(&buffer), + Arc::clone(&rate_adjustment_ppm), + )?, + cpal::SampleFormat::I32 => build_output::( + &device, + &config, + channels, + output_rate, + Arc::clone(&buffer), + Arc::clone(&rate_adjustment_ppm), + )?, + cpal::SampleFormat::I64 => build_output::( + &device, + &config, + channels, + output_rate, + Arc::clone(&buffer), + Arc::clone(&rate_adjustment_ppm), + )?, + cpal::SampleFormat::U8 => build_output::( + &device, + &config, + channels, + output_rate, + Arc::clone(&buffer), + Arc::clone(&rate_adjustment_ppm), + )?, + cpal::SampleFormat::U16 => build_output::( + &device, + &config, + channels, + output_rate, + Arc::clone(&buffer), + Arc::clone(&rate_adjustment_ppm), + )?, + cpal::SampleFormat::U24 => build_output::( + &device, + &config, + channels, + output_rate, + Arc::clone(&buffer), + Arc::clone(&rate_adjustment_ppm), + )?, + cpal::SampleFormat::U32 => build_output::( + &device, + &config, + channels, + output_rate, + Arc::clone(&buffer), + Arc::clone(&rate_adjustment_ppm), + )?, + cpal::SampleFormat::U64 => build_output::( + &device, + &config, + channels, + output_rate, + Arc::clone(&buffer), + Arc::clone(&rate_adjustment_ppm), + )?, + cpal::SampleFormat::F32 => build_output::( + &device, + &config, + channels, + output_rate, + Arc::clone(&buffer), + Arc::clone(&rate_adjustment_ppm), + )?, + cpal::SampleFormat::F64 => build_output::( + &device, + &config, + channels, + output_rate, + Arc::clone(&buffer), + Arc::clone(&rate_adjustment_ppm), + )?, + _ => return Err("WASAPI output sample format is unsupported".to_owned()), + }; + stream + .play() + .map_err(|error| format!("WASAPI output start failed: {error}"))?; + + let first = receiver + .recv() + .map_err(|_| "audio stream closed".to_owned())?; + let mut decoder = JitterDecoder::new(&first)?; + { + let mut playback = buffer + .lock() + .map_err(|_| "audio buffer lock poisoned".to_owned())?; + decoder.push(first, &mut playback, &clock, &rate_adjustment_ppm)?; + } + loop { + match receiver.recv() { + Ok(packet) => { + let mut playback = buffer + .lock() + .map_err(|_| "audio buffer lock poisoned".to_owned())?; + decoder.push(packet, &mut playback, &clock, &rate_adjustment_ppm)?; + } + Err(_) => return Ok(()), + } + } + } + + fn build_output( + device: &cpal::Device, + config: &cpal::StreamConfig, + channels: usize, + output_rate: u32, + buffer: Arc>, + rate_adjustment_ppm: Arc, + ) -> Result + where + T: SizedSample + cpal::FromSample, + { + let step = 48_000.0 / f64::from(output_rate.max(1)); + let mut phase = 1.0; + let mut current = [0.0_f32; 2]; + device + .build_output_stream( + config, + move |output: &mut [T], _: &cpal::OutputCallbackInfo| { + if let Ok(mut buffer) = buffer.try_lock() { + for frame in output.chunks_mut(channels) { + while phase >= 1.0 { + current = buffer.pop(); + phase -= 1.0; + } + let adjustment = f64::from(rate_adjustment_ppm.load(Ordering::Acquire)) + / 1_000_000.0; + phase += step * (1.0 + adjustment); + for (channel, value) in frame.iter_mut().enumerate() { + let sample = match channel { + 0 => current[0], + 1 => current[1], + _ => (current[0] + current[1]) * 0.5, + }; + *value = T::from_sample(sample); + } + } + } else { + for value in output { + *value = T::from_sample(0.0); + } + } + }, + |error| eprintln!("WASAPI output stream error: {error}"), + None, + ) + .map_err(|error| format!("WASAPI output stream creation failed: {error}")) + } + + #[cfg(test)] + mod tests { + use super::*; + use opus2::{Application, Encoder}; + + fn packet(encoder: &mut Encoder, sequence: u64) -> remotedesk_protocol::AudioPacket { + let payload = encoder + .encode_vec_float(&vec![0.0; 960 * 2], 4 * 1024) + .unwrap(); + remotedesk_protocol::AudioPacket { + stream_id: 1, + generation: 1, + sequence, + pts: (sequence - 1) * 200_000, + duration_ms: 20, + flags: remotedesk_protocol::AudioPacketFlags::from_bits(0).unwrap(), + payload, + } + } + + #[test] + fn jitter_startup_reorders_three_packets_before_decode() { + let mut encoder = Encoder::new(48_000, Channels::Stereo, Application::Audio).unwrap(); + let first = packet(&mut encoder, 1); + let second = packet(&mut encoder, 2); + let third = packet(&mut encoder, 3); + let clock = Arc::new(Mutex::new(MediaClock::new())); + let rate_adjustment_ppm = Arc::new(AtomicI32::new(0)); + let mut output = PlaybackBuffer::new(); + let mut jitter = JitterDecoder::new(&first).unwrap(); + + jitter + .push(first, &mut output, &clock, &rate_adjustment_ppm) + .unwrap(); + jitter + .push(third, &mut output, &clock, &rate_adjustment_ppm) + .unwrap(); + assert!(output.frames.is_empty()); + jitter + .push(second, &mut output, &clock, &rate_adjustment_ppm) + .unwrap(); + + assert_eq!(output.frames.len(), 3 * 960); + assert_eq!(jitter.expected, Some(4)); + assert!(jitter.packets.is_empty()); + } + } +} + +#[cfg(windows)] +pub(crate) use windows_audio::start; + +#[cfg(not(windows))] +pub(crate) fn start( + _receiver: std::sync::mpsc::Receiver, + _clock: Arc>, +) { +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn media_clock_is_unlocked_before_audio_and_accepts_on_time_video() { + let mut clock = MediaClock::new(); + assert!(clock.should_present_video(1)); + clock.observe_audio(1_000_000, 20); + assert!(clock.should_present_video(1_000_000)); + } + + #[test] + fn media_clock_keeps_nominal_audio_rate_near_zero() { + let mut clock = MediaClock::new(); + clock.observe_audio(1_000_000, 20); + for sequence in 1..32 { + clock.observe_audio(1_000_000 + sequence * 200_000, 20); + } + assert_eq!(clock.rate_adjustment_ppm(), 0); + } + + #[test] + fn media_clock_bounds_drift_correction_and_decays_in_deadband() { + let mut clock = MediaClock::new(); + clock.observe_audio(1_000_000, 20); + for sequence in 1..128 { + clock.observe_audio(1_000_000 + sequence * 300_000, 30); + } + assert!(clock.rate_adjustment_ppm().abs() <= AUDIO_RATE_LIMIT_PPM); + let corrected = clock.rate_adjustment_ppm(); + for sequence in 128..160 { + clock.observe_audio( + 1_000_000 + 127 * 300_000 + (sequence - 127) * 200_000, + 20, + ); + } + assert!(clock.rate_adjustment_ppm().abs() < corrected.abs()); + } + + #[test] + fn media_clock_reset_clears_drift_and_buffer_correction() { + let mut clock = MediaClock::new(); + clock.observe_audio(1_000_000, 20); + clock.observe_audio(1_300_000, 20); + clock.update_buffer_rate_adjustment(48_000); + assert_ne!(clock.rate_adjustment_ppm(), 0); + + clock.reset_audio(9_000_000); + assert_eq!(clock.rate_adjustment_ppm(), 0); + } + + #[test] + fn media_clock_buffer_feedback_is_bounded_and_zero_at_target() { + let mut clock = MediaClock::new(); + clock.update_buffer_rate_adjustment(2_880); + assert_eq!(clock.rate_adjustment_ppm(), 0); + clock.update_buffer_rate_adjustment(48_000); + assert_eq!(clock.rate_adjustment_ppm(), AUDIO_RATE_LIMIT_PPM); + clock.update_buffer_rate_adjustment(0); + assert_eq!(clock.rate_adjustment_ppm(), -AUDIO_RATE_LIMIT_PPM); + } +} diff --git a/client/helpers/windows-agent-viewer/src/main.rs b/client/helpers/windows-agent-viewer/src/main.rs index 3c54de3..4fc4518 100644 --- a/client/helpers/windows-agent-viewer/src/main.rs +++ b/client/helpers/windows-agent-viewer/src/main.rs @@ -1,13 +1,21 @@ use flate2::read::ZlibDecoder; +mod audio; +#[cfg(windows)] +use remotedesk_native_video::stream::{H264AccessUnit, NativeH264Renderer}; +use remotedesk_protocol::{ + AUDIO_DATAGRAM_MAX_BYTES, VIDEO_DATAGRAM_MAX_BYTES, VideoAccessUnit, VideoReassembler, + decode_audio_datagram, +}; use serde::Serialize; use std::env; use std::fs; use std::io::{BufRead as _, BufReader, Read as _, Write as _}; -use std::net::{TcpStream, ToSocketAddrs}; +use std::net::{SocketAddr, TcpStream, ToSocketAddrs, UdpSocket}; use std::num::NonZeroU32; use std::path::PathBuf; use std::process::ExitCode; -use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, mpsc}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use winit::application::ApplicationHandler; use winit::dpi::PhysicalSize; @@ -22,6 +30,7 @@ const DEFAULT_PORT: u16 = 39_501; const MAX_DIMENSION: u32 = 16_384; const MAX_RAW_BYTES: usize = 256 * 1024 * 1024; const MAX_COMPRESSED_BYTES: usize = 256 * 1024 * 1024; +const INPUT_QUEUE_CAPACITY: usize = 256; #[derive(Debug)] struct Args { @@ -31,6 +40,7 @@ struct Args { fullscreen: bool, mode: String, application: Option, + media_udp: Option, } #[derive(Debug)] @@ -47,9 +57,15 @@ struct Frame { enum ViewerEvent { Connected, Frame(Frame), + Encoded(EncodedViewerFrame), Failed(String), } +struct EncodedViewerFrame { + access_unit: VideoAccessUnit, + pending: Arc, +} + fn main() -> ExitCode { match run() { Ok(()) => ExitCode::SUCCESS, @@ -68,16 +84,50 @@ fn run() -> Result<(), String> { .map_err(|error| error.to_string())?; let proxy = event_loop.create_proxy(); let target = normalize_target(&args.target)?; + let input = InputChannel::start(target.clone()); #[cfg(windows)] - windows_keyboard_hook::install(target.clone()); + windows_keyboard_hook::install(input.clone()); + let frame_target = target.clone(); let fps = args.fps; let mode = args.mode.clone(); let application = args.application.clone(); - std::thread::Builder::new() - .name("windows-agent-rdwf".to_owned()) - .spawn(move || receive_frames(&target, fps, &mode, application.as_deref(), &proxy)) - .map_err(|error| error.to_string())?; - let mut app = ViewerApp::new(&event_loop, args.fullscreen, diagnostics, args.target.clone())?; + let encoded_media = args.media_udp.is_some(); + if let Some(bind) = args.media_udp { + let (audio_sender, audio_receiver) = mpsc::sync_channel(64); + let media_clock = Arc::new(Mutex::new(audio::MediaClock::new())); + audio::start(audio_receiver, Arc::clone(&media_clock)); + let media_input = input.clone(); + let media_session_id = args.session_id.clone(); + let media_clock_for_receiver = Arc::clone(&media_clock); + std::thread::Builder::new() + .name("windows-agent-rdv1".to_owned()) + .spawn(move || { + receive_encoded_media( + bind, + &proxy, + &media_input, + &media_session_id, + audio_sender, + media_clock_for_receiver, + ); + }) + .map_err(|error| error.to_string())?; + } else { + std::thread::Builder::new() + .name("windows-agent-rdwf".to_owned()) + .spawn(move || { + receive_frames(&frame_target, fps, &mode, application.as_deref(), &proxy); + }) + .map_err(|error| error.to_string())?; + } + let mut app = ViewerApp::new( + &event_loop, + args.fullscreen, + diagnostics, + input, + encoded_media, + fps, + )?; event_loop .run_app(&mut app) .map_err(|error| error.to_string()) @@ -90,6 +140,7 @@ fn parse_args(mut args: impl Iterator) -> Result { let mut fullscreen = false; let mut mode = "desktop".to_owned(); let mut application = None; + let mut media_udp = None; while let Some(option) = args.next() { match option.as_str() { "--target" => target = args.next(), @@ -98,16 +149,31 @@ fn parse_args(mut args: impl Iterator) -> Result { fps = args .next() .and_then(|value| value.parse().ok()) - .filter(|value| (1..=15).contains(value)) - .ok_or_else(|| "--fps must be between 1 and 15".to_owned())?; + .filter(|value| (1..=120).contains(value)) + .ok_or_else(|| "--fps must be between 1 and 120".to_owned())?; } "--fullscreen" => fullscreen = true, "--mode" => mode = args.next().ok_or_else(|| "--mode is required".to_owned())?, "--application" => application = args.next(), + "--media-udp" => { + let value = args + .next() + .ok_or_else(|| "--media-udp is required".to_owned())?; + let address = value + .parse::() + .map_err(|_| "--media-udp must be an IP socket address".to_owned())?; + if !address.ip().is_loopback() { + return Err("--media-udp must bind to loopback".to_owned()); + } + media_udp = Some(address); + } _ => return Err(format!("unknown option {option}")), } } let target = target.ok_or_else(|| "--target is required".to_owned())?; + if media_udp.is_none() && fps > 15 { + return Err("--fps above 15 requires --media-udp".to_owned()); + } let session_id = session_id.filter(|value| { !value.is_empty() && value.len() <= 64 @@ -125,9 +191,112 @@ fn parse_args(mut args: impl Iterator) -> Result { _ => return Err("--mode must be desktop or application".to_owned()), }, application, + media_udp, }) } +fn receive_encoded_media( + bind: SocketAddr, + proxy: &winit::event_loop::EventLoopProxy, + input: &InputChannel, + session_id: &str, + audio_sender: mpsc::SyncSender, + media_clock: Arc>, +) { + if let Err(error) = + receive_encoded_media_inner(bind, proxy, input, session_id, audio_sender, media_clock) + { + let _ = proxy.send_event(ViewerEvent::Failed(error)); + } +} + +fn receive_encoded_media_inner( + bind: SocketAddr, + proxy: &winit::event_loop::EventLoopProxy, + input: &InputChannel, + session_id: &str, + audio_sender: mpsc::SyncSender, + media_clock: Arc>, +) -> Result<(), String> { + let socket = + UdpSocket::bind(bind).map_err(|error| format!("media UDP bind failed: {error}"))?; + let started = Instant::now(); + let mut reassembler = VideoReassembler::new(); + let mut buffer = vec![0_u8; AUDIO_DATAGRAM_MAX_BYTES.max(VIDEO_DATAGRAM_MAX_BYTES) + 1]; + let mut connected = false; + let mut pinned_source = None; + let pending = Arc::new(AtomicBool::new(false)); + let mut last_keyframe_request = None; + loop { + let (length, source) = socket + .recv_from(&mut buffer) + .map_err(|error| format!("RDV1 receive failed: {error}"))?; + if !source.ip().is_loopback() + || length > buffer.len() - 1 + || pinned_source.is_some_and(|pinned| pinned != source) + { + continue; + } + if length >= 4 && &buffer[..4] == b"RDA1" { + if let Ok(packet) = decode_audio_datagram(&buffer[..length]) { + pinned_source.get_or_insert(source); + let _ = audio_sender.try_send(packet); + } + continue; + } + if length < 4 || &buffer[..4] != b"RDV1" || length > VIDEO_DATAGRAM_MAX_BYTES { + continue; + } + let access_unit = match reassembler.push_datagram(&buffer[..length], started.elapsed()) { + Ok(access_unit) => { + pinned_source.get_or_insert(source); + access_unit + } + Err(_) => continue, + }; + if reassembler.keyframe_required() + && last_keyframe_request + .is_none_or(|last: Instant| last.elapsed() >= Duration::from_millis(250)) + { + input.send_priority( + serde_json::json!({"kind":"request_keyframe","session_id":session_id}), + ); + last_keyframe_request = Some(Instant::now()); + } + let Some(access_unit) = access_unit else { + continue; + }; + if media_clock + .lock() + .is_ok_and(|clock| !clock.should_present_video(access_unit.pts)) + { + continue; + } + if !connected { + proxy + .send_event(ViewerEvent::Connected) + .map_err(|_| "viewer window closed".to_owned())?; + connected = true; + } + if pending + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + continue; + } + if proxy + .send_event(ViewerEvent::Encoded(EncodedViewerFrame { + access_unit, + pending: Arc::clone(&pending), + })) + .is_err() + { + pending.store(false, Ordering::Release); + return Err("viewer window closed".to_owned()); + } + } +} + fn normalize_target(target: &str) -> Result { if target.is_empty() || target.len() > 512 || target.contains(char::is_whitespace) { return Err("Windows Agent target is invalid".to_owned()); @@ -186,9 +355,9 @@ fn receive_frames_inner( } let command = if mode == "application" { let id = application.ok_or_else(|| "应用模式未选择应用".to_owned())?; - serde_json::json!({ "kind": "open_application", "application": id, "allow_software_fallback": true, "frames_per_second": fps }) + serde_json::json!({ "kind": "open_application", "application": id, "frames_per_second": fps }) } else { - serde_json::json!({ "kind": "open_desktop", "allow_software_fallback": true, "frames_per_second": fps }) + serde_json::json!({ "kind": "open_desktop", "frames_per_second": fps }) }; writeln!( reader.get_mut(), @@ -354,12 +523,16 @@ struct ViewerApp { surface: Option>>, frame: Option, fullscreen: bool, - target: String, + input: InputChannel, cursor_position: (i32, i32), modifiers: ModifiersState, control_down: bool, alt_down: bool, diagnostics: Diagnostics, + encoded_media: bool, + encoded_frame_duration: Duration, + #[cfg(windows)] + native_renderer: Option, } impl ViewerApp { @@ -367,7 +540,9 @@ impl ViewerApp { event_loop: &EventLoop, fullscreen: bool, diagnostics: Diagnostics, - target: String, + input: InputChannel, + encoded_media: bool, + fps: u8, ) -> Result { Ok(Self { context: softbuffer::Context::new(event_loop.owned_display_handle()) @@ -376,12 +551,16 @@ impl ViewerApp { surface: None, frame: None, fullscreen, - target, + input, cursor_position: (0, 0), modifiers: ModifiersState::empty(), control_down: false, alt_down: false, diagnostics, + encoded_media, + encoded_frame_duration: Duration::from_secs_f64(1.0 / f64::from(fps.max(1))), + #[cfg(windows)] + native_renderer: None, }) } @@ -441,7 +620,28 @@ impl ApplicationHandler for ViewerApp { match event_loop.create_window(attributes) { Ok(window) => { let window = Arc::new(window); - self.surface = softbuffer::Surface::new(&self.context, Arc::clone(&window)).ok(); + if self.encoded_media { + #[cfg(windows)] + match NativeH264Renderer::new(Arc::clone(&window)) { + Ok(renderer) => self.native_renderer = Some(renderer), + Err(error) => { + self.diagnostics.failed(&error); + eprintln!("{error}"); + event_loop.exit(); + return; + } + } + #[cfg(not(windows))] + { + self.diagnostics + .failed("native H.264 rendering is Windows-only"); + event_loop.exit(); + return; + } + } else { + self.surface = + softbuffer::Surface::new(&self.context, Arc::clone(&window)).ok(); + } self.window = Some(window); } Err(_) => event_loop.exit(), @@ -466,10 +666,14 @@ impl ApplicationHandler for ViewerApp { WindowEvent::CursorMoved { position, .. } => { if let Some(window) = &self.window { let size = window.inner_size(); - let x = (position.x.max(0.0).min(size.width as f64) * 65535.0 / size.width.max(1) as f64) as i32; - let y = (position.y.max(0.0).min(size.height as f64) * 65535.0 / size.height.max(1) as f64) as i32; + let x = (position.x.max(0.0).min(size.width as f64) * 65535.0 + / size.width.max(1) as f64) as i32; + let y = (position.y.max(0.0).min(size.height as f64) * 65535.0 + / size.height.max(1) as f64) as i32; self.cursor_position = (x, y); - send_input(&self.target, serde_json::json!({"kind":"input","input_type":"mouse","x":x,"y":y})); + self.input.send_motion( + serde_json::json!({"kind":"input","input_type":"mouse","x":x,"y":y}), + ); } } WindowEvent::MouseInput { state, button, .. } => { @@ -482,27 +686,43 @@ impl ApplicationHandler for ViewerApp { (MouseButton::Middle, ElementState::Released) => "middle_up", _ => return, }; - send_input(&self.target, serde_json::json!({"kind":"input","input_type":"mouse","mouse_action":action,"x":self.cursor_position.0,"y":self.cursor_position.1})); + self.input.send_priority(serde_json::json!({"kind":"input","input_type":"mouse","mouse_action":action,"x":self.cursor_position.0,"y":self.cursor_position.1})); } WindowEvent::MouseWheel { delta, .. } => { - let value = match delta { MouseScrollDelta::LineDelta(_, y) => (y * 120.0) as i32, MouseScrollDelta::PixelDelta(p) => p.y as i32 }; - send_input(&self.target, serde_json::json!({"kind":"input","input_type":"mouse","mouse_action":"wheel","buttons":value,"x":self.cursor_position.0,"y":self.cursor_position.1})); + let value = match delta { + MouseScrollDelta::LineDelta(_, y) => (y * 120.0) as i32, + MouseScrollDelta::PixelDelta(p) => p.y as i32, + }; + self.input.send_priority(serde_json::json!({"kind":"input","input_type":"mouse","mouse_action":"wheel","buttons":value,"x":self.cursor_position.0,"y":self.cursor_position.1})); } WindowEvent::ModifiersChanged(modifiers) => self.modifiers = modifiers.state(), WindowEvent::KeyboardInput { event, .. } => { if matches!(event.logical_key, Key::Named(NamedKey::Super)) { - let vk = if matches!(event.physical_key, PhysicalKey::Code(KeyCode::SuperRight)) { 0x5C } else { 0x5B }; - send_input(&self.target, serde_json::json!({"kind":"input","input_type":"key","code":vk,"down":event.state == ElementState::Pressed})); + let vk = if matches!(event.physical_key, PhysicalKey::Code(KeyCode::SuperRight)) + { + 0x5C + } else { + 0x5B + }; + self.input.send_priority(serde_json::json!({"kind":"input","input_type":"key","code":vk,"down":event.state == ElementState::Pressed})); return; } - let control = matches!(event.physical_key, PhysicalKey::Code(KeyCode::ControlLeft | KeyCode::ControlRight)); - let alt = matches!(event.physical_key, PhysicalKey::Code(KeyCode::AltLeft | KeyCode::AltRight)); + let control = matches!( + event.physical_key, + PhysicalKey::Code(KeyCode::ControlLeft | KeyCode::ControlRight) + ); + let alt = matches!( + event.physical_key, + PhysicalKey::Code(KeyCode::AltLeft | KeyCode::AltRight) + ); if event.state == ElementState::Pressed { self.control_down |= control; self.alt_down |= alt; if let PhysicalKey::Code(code) = event.physical_key { let vk = virtual_key(code); - if vk != 0 { send_input(&self.target, serde_json::json!({"kind":"input","input_type":"key","code":vk,"down":true})); } + if vk != 0 { + self.input.send_priority(serde_json::json!({"kind":"input","input_type":"key","code":vk,"down":true})); + } } if matches!(event.physical_key, PhysicalKey::Code(KeyCode::KeyF)) && (self.control_down || self.modifiers.control_key()) @@ -510,16 +730,25 @@ impl ApplicationHandler for ViewerApp { { self.fullscreen = !self.fullscreen; if let Some(window) = &self.window { - window.set_fullscreen(self.fullscreen.then(|| Fullscreen::Borderless(window.current_monitor()))); + window.set_fullscreen( + self.fullscreen + .then(|| Fullscreen::Borderless(window.current_monitor())), + ); } } } else { if let PhysicalKey::Code(code) = event.physical_key { let vk = virtual_key(code); - if vk != 0 { send_input(&self.target, serde_json::json!({"kind":"input","input_type":"key","code":vk,"down":false})); } + if vk != 0 { + self.input.send_priority(serde_json::json!({"kind":"input","input_type":"key","code":vk,"down":false})); + } + } + if control { + self.control_down = false; + } + if alt { + self.alt_down = false; } - if control { self.control_down = false; } - if alt { self.alt_down = false; } } } _ => {} @@ -536,6 +765,43 @@ impl ApplicationHandler for ViewerApp { window.request_redraw(); } } + ViewerEvent::Encoded(encoded) => { + encoded.pending.store(false, Ordering::Release); + #[cfg(windows)] + { + let Some(renderer) = self.native_renderer.as_mut() else { + return; + }; + let input = H264AccessUnit { + data: encoded.access_unit.payload, + duration: self.encoded_frame_duration, + }; + match renderer.decode_and_present(&input) { + Ok(Some(presentation)) => { + self.diagnostics.native_frame( + presentation.width, + presentation.height, + presentation.frames, + presentation.decode_latency_us, + presentation.presentation_latency_us, + ); + } + Ok(None) => {} + Err(error) => { + self.diagnostics.failed(&error); + eprintln!("{error}"); + event_loop.exit(); + } + } + } + #[cfg(not(windows))] + { + let _ = encoded.access_unit; + self.diagnostics + .failed("native H.264 rendering is Windows-only"); + event_loop.exit(); + } + } ViewerEvent::Failed(error) => { eprintln!("{error}"); self.diagnostics.failed(&error); @@ -551,16 +817,42 @@ impl ApplicationHandler for ViewerApp { fn virtual_key(code: KeyCode) -> u16 { match code { - KeyCode::KeyA => 0x41, KeyCode::KeyB => 0x42, KeyCode::KeyC => 0x43, KeyCode::KeyD => 0x44, - KeyCode::KeyE => 0x45, KeyCode::KeyF => 0x46, KeyCode::KeyG => 0x47, KeyCode::KeyH => 0x48, - KeyCode::KeyI => 0x49, KeyCode::KeyJ => 0x4A, KeyCode::KeyK => 0x4B, KeyCode::KeyL => 0x4C, - KeyCode::KeyM => 0x4D, KeyCode::KeyN => 0x4E, KeyCode::KeyO => 0x4F, KeyCode::KeyP => 0x50, - KeyCode::KeyQ => 0x51, KeyCode::KeyR => 0x52, KeyCode::KeyS => 0x53, KeyCode::KeyT => 0x54, - KeyCode::KeyU => 0x55, KeyCode::KeyV => 0x56, KeyCode::KeyW => 0x57, KeyCode::KeyX => 0x58, - KeyCode::KeyY => 0x59, KeyCode::KeyZ => 0x5A, - KeyCode::Digit0 => 0x30, KeyCode::Digit1 => 0x31, KeyCode::Digit2 => 0x32, KeyCode::Digit3 => 0x33, - KeyCode::Digit4 => 0x34, KeyCode::Digit5 => 0x35, KeyCode::Digit6 => 0x36, KeyCode::Digit7 => 0x37, - KeyCode::Digit8 => 0x38, KeyCode::Digit9 => 0x39, + KeyCode::KeyA => 0x41, + KeyCode::KeyB => 0x42, + KeyCode::KeyC => 0x43, + KeyCode::KeyD => 0x44, + KeyCode::KeyE => 0x45, + KeyCode::KeyF => 0x46, + KeyCode::KeyG => 0x47, + KeyCode::KeyH => 0x48, + KeyCode::KeyI => 0x49, + KeyCode::KeyJ => 0x4A, + KeyCode::KeyK => 0x4B, + KeyCode::KeyL => 0x4C, + KeyCode::KeyM => 0x4D, + KeyCode::KeyN => 0x4E, + KeyCode::KeyO => 0x4F, + KeyCode::KeyP => 0x50, + KeyCode::KeyQ => 0x51, + KeyCode::KeyR => 0x52, + KeyCode::KeyS => 0x53, + KeyCode::KeyT => 0x54, + KeyCode::KeyU => 0x55, + KeyCode::KeyV => 0x56, + KeyCode::KeyW => 0x57, + KeyCode::KeyX => 0x58, + KeyCode::KeyY => 0x59, + KeyCode::KeyZ => 0x5A, + KeyCode::Digit0 => 0x30, + KeyCode::Digit1 => 0x31, + KeyCode::Digit2 => 0x32, + KeyCode::Digit3 => 0x33, + KeyCode::Digit4 => 0x34, + KeyCode::Digit5 => 0x35, + KeyCode::Digit6 => 0x36, + KeyCode::Digit7 => 0x37, + KeyCode::Digit8 => 0x38, + KeyCode::Digit9 => 0x39, KeyCode::ControlLeft | KeyCode::ControlRight => 0x11, KeyCode::AltLeft | KeyCode::AltRight => 0x12, KeyCode::SuperLeft => 0x5B, @@ -572,26 +864,106 @@ fn virtual_key(code: KeyCode) -> u16 { } } -fn send_input(target: &str, command: serde_json::Value) { - let Ok(mut stream) = TcpStream::connect_timeout( - &match target.to_socket_addrs().ok().and_then(|mut addrs| addrs.next()) { - Some(address) => address, - None => return, - }, - Duration::from_secs(2), - ) else { return }; - let _ = stream.set_nodelay(true); - let _ = stream.set_read_timeout(Some(Duration::from_secs(2))); - let Ok(clone) = stream.try_clone() else { return }; - let mut reader = BufReader::with_capacity(1, clone); - let Ok(hello) = read_json_line(&mut reader) else { return }; - if hello.get("kind").and_then(serde_json::Value::as_str) != Some("windows_agent_hello") { return; } - if writeln!(stream, "{}", command).is_err() { return; } - if let Ok(response) = read_json_line(&mut reader) { - if response.get("kind").and_then(serde_json::Value::as_str) == Some("input_failed") { - eprintln!("Windows Agent input failed: {}", response.get("error").and_then(serde_json::Value::as_str).unwrap_or("unknown")); +#[derive(Clone)] +struct InputChannel { + priority: mpsc::SyncSender, + latest_motion: Arc>>, +} + +impl InputChannel { + fn start(target: String) -> Self { + let (priority, receiver) = mpsc::sync_channel(INPUT_QUEUE_CAPACITY); + let latest_motion = Arc::new(Mutex::new(None)); + let worker_motion = Arc::clone(&latest_motion); + let _ = std::thread::Builder::new() + .name("windows-agent-input".to_owned()) + .spawn(move || input_worker(&target, &receiver, &worker_motion)); + Self { + priority, + latest_motion, } } + + fn send_priority(&self, command: serde_json::Value) { + if let Err(mpsc::TrySendError::Full(_)) = self.priority.try_send(command) { + eprintln!("Windows Agent input queue is full"); + } + } + + fn send_motion(&self, command: serde_json::Value) { + if let Ok(mut latest) = self.latest_motion.lock() { + *latest = Some(command); + } + } +} + +fn input_worker( + target: &str, + priority: &mpsc::Receiver, + latest_motion: &Mutex>, +) { + let mut stream = None; + loop { + let command = match priority.recv_timeout(Duration::from_millis(4)) { + Ok(command) => Some(command), + Err(mpsc::RecvTimeoutError::Timeout) => latest_motion + .lock() + .ok() + .and_then(|mut latest| latest.take()), + Err(mpsc::RecvTimeoutError::Disconnected) => return, + }; + let Some(command) = command else { + continue; + }; + if stream.is_none() { + stream = connect_input_channel(target).ok(); + } + let Some(active) = stream.as_mut() else { + continue; + }; + if writeln!(active, "{command}").is_err() { + stream = None; + } + } +} + +fn connect_input_channel(target: &str) -> Result { + let address = target + .to_socket_addrs() + .map_err(|error| error.to_string())? + .next() + .ok_or_else(|| "Windows Agent input target did not resolve".to_owned())?; + let stream = TcpStream::connect_timeout(&address, Duration::from_secs(2)) + .map_err(|error| error.to_string())?; + stream + .set_nodelay(true) + .map_err(|error| error.to_string())?; + stream + .set_write_timeout(Some(Duration::from_secs(2))) + .map_err(|error| error.to_string())?; + let reader_stream = stream.try_clone().map_err(|error| error.to_string())?; + let mut reader = BufReader::with_capacity(1024, reader_stream); + let hello = read_json_line(&mut reader)?; + if hello.get("kind").and_then(serde_json::Value::as_str) != Some("windows_agent_hello") { + return Err("input endpoint is not a Windows Agent".to_owned()); + } + let _ = std::thread::Builder::new() + .name("windows-agent-input-ack".to_owned()) + .spawn(move || { + while let Ok(response) = read_json_line(&mut reader) { + if response.get("kind").and_then(serde_json::Value::as_str) == Some("input_failed") + { + eprintln!( + "Windows Agent input failed: {}", + response + .get("error") + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown") + ); + } + } + }); + Ok(stream) } #[derive(Serialize)] @@ -628,6 +1000,7 @@ struct Diagnostics { decode: Option, presentation: Option, error: Option, + renderer: &'static str, } impl Diagnostics { @@ -651,6 +1024,7 @@ impl Diagnostics { decode: None, presentation: None, error: None, + renderer: "softbuffer-bgra", }; value.write()?; Ok(value) @@ -680,6 +1054,32 @@ impl Diagnostics { self.presentation = Some(elapsed.as_secs_f64() * 1_000.0); let _ = self.write(); } + fn native_frame( + &mut self, + width: u32, + height: u32, + frames: usize, + decode_latency_us: u64, + presentation_latency_us: u64, + ) { + self.state = "connected"; + let frames = u64::try_from(frames).unwrap_or(u64::MAX); + self.frame_count = self.frame_count.saturating_add(frames); + self.fps_frames = self + .fps_frames + .saturating_add(u32::try_from(frames).unwrap_or(u32::MAX)); + self.size = (width, height); + self.decode = Some(micros_ms(decode_latency_us)); + self.presentation = Some(micros_ms(presentation_latency_us)); + self.renderer = "media-foundation-d3d11"; + let elapsed = self.fps_started.elapsed(); + if elapsed >= Duration::from_secs(1) { + self.fps = Some(f64::from(self.fps_frames) / elapsed.as_secs_f64()); + self.fps_frames = 0; + self.fps_started = Instant::now(); + } + let _ = self.write(); + } fn failed(&mut self, error: &str) { self.state = "failed"; self.error = Some(error.chars().take(128).collect()); @@ -709,7 +1109,7 @@ impl Diagnostics { frame_processing_latency_ms: self.processing, decode_latency_ms: self.decode, presentation_latency_ms: self.presentation, - renderer: "softbuffer-bgra", + renderer: self.renderer, error_code: self.error.as_deref(), }; let data = serde_json::to_vec(&snapshot).map_err(|error| error.to_string())?; @@ -732,4 +1132,59 @@ mod tests { assert_eq!(normalize_target("10.0.0.2").unwrap(), "10.0.0.2:39501"); assert_eq!(normalize_target("10.0.0.2:4000").unwrap(), "10.0.0.2:4000"); } + + #[test] + fn encoded_media_requires_loopback_and_allows_120_fps() { + let args = parse_args( + [ + "--target", + "10.0.0.2", + "--session-id", + "session-1", + "--fps", + "120", + "--media-udp", + "127.0.0.1:39502", + ] + .into_iter() + .map(str::to_owned), + ) + .unwrap(); + assert_eq!(args.fps, 120); + assert_eq!(args.media_udp.unwrap().to_string(), "127.0.0.1:39502"); + assert!( + parse_args( + [ + "--target", + "10.0.0.2", + "--session-id", + "session-1", + "--media-udp", + "0.0.0.0:39502", + ] + .into_iter() + .map(str::to_owned), + ) + .is_err() + ); + } + + #[test] + fn rdwf_mode_keeps_fifteen_fps_limit() { + assert!( + parse_args( + [ + "--target", + "10.0.0.2", + "--session-id", + "session-1", + "--fps", + "16", + ] + .into_iter() + .map(str::to_owned), + ) + .is_err() + ); + } } diff --git a/client/helpers/windows-agent-viewer/src/windows_keyboard_hook.rs b/client/helpers/windows-agent-viewer/src/windows_keyboard_hook.rs index 80af0c4..8fbbcca 100644 --- a/client/helpers/windows-agent-viewer/src/windows_keyboard_hook.rs +++ b/client/helpers/windows-agent-viewer/src/windows_keyboard_hook.rs @@ -1,20 +1,26 @@ #![cfg(windows)] -use std::net::{TcpStream, ToSocketAddrs}; +use crate::InputChannel; +use serde_json::json; use std::sync::OnceLock; use std::thread; -use std::time::Duration; use windows::Win32::Foundation::{LPARAM, LRESULT, WPARAM}; -use windows::Win32::UI::WindowsAndMessaging::{CallNextHookEx, DispatchMessageW, GetForegroundWindow, GetMessageW, GetWindowThreadProcessId, KBDLLHOOKSTRUCT, MSG, SetWindowsHookExW, TranslateMessage, UnhookWindowsHookEx, WH_KEYBOARD_LL, WM_KEYDOWN, WM_KEYUP, WM_SYSKEYDOWN, WM_SYSKEYUP}; +use windows::Win32::UI::WindowsAndMessaging::{ + CallNextHookEx, DispatchMessageW, GetForegroundWindow, GetMessageW, GetWindowThreadProcessId, + KBDLLHOOKSTRUCT, MSG, SetWindowsHookExW, TranslateMessage, UnhookWindowsHookEx, WH_KEYBOARD_LL, + WM_KEYDOWN, WM_KEYUP, WM_SYSKEYDOWN, WM_SYSKEYUP, +}; static PROCESS_ID: OnceLock = OnceLock::new(); -static TARGET: OnceLock = OnceLock::new(); +static INPUT: OnceLock = OnceLock::new(); -pub fn install(target: String) { +pub fn install(input: InputChannel) { let _ = PROCESS_ID.set(std::process::id()); - let _ = TARGET.set(target); + let _ = INPUT.set(input); thread::spawn(move || unsafe { - let Ok(hook) = SetWindowsHookExW(WH_KEYBOARD_LL, Some(callback), None, 0) else { return }; + let Ok(hook) = SetWindowsHookExW(WH_KEYBOARD_LL, Some(callback), None, 0) else { + return; + }; let mut message = MSG::default(); while GetMessageW(&mut message, None, 0, 0).as_bool() { let _ = TranslateMessage(&message); @@ -32,7 +38,14 @@ unsafe extern "system" fn callback(code: i32, wparam: WPARAM, lparam: LPARAM) -> let pressed = matches!(message, WM_KEYDOWN | WM_SYSKEYDOWN); let released = matches!(message, WM_KEYUP | WM_SYSKEYUP); if pressed || released { - send_remote(data.vkCode as u16, pressed); + if let Some(input) = INPUT.get() { + input.send_priority(json!({ + "kind": "input", + "input_type": "key", + "code": data.vkCode, + "down": pressed + })); + } return LRESULT(1); } } @@ -41,18 +54,14 @@ unsafe extern "system" fn callback(code: i32, wparam: WPARAM, lparam: LPARAM) -> } fn foreground_is_viewer() -> bool { - let Some(pid) = PROCESS_ID.get().copied() else { return false }; + let Some(pid) = PROCESS_ID.get().copied() else { + return false; + }; let window = unsafe { GetForegroundWindow() }; - if window.0.is_null() { return false } + if window.0.is_null() { + return false; + } let mut foreground_pid = 0; unsafe { GetWindowThreadProcessId(window, Some(&mut foreground_pid)) }; foreground_pid == pid } - -fn send_remote(vk: u16, pressed: bool) { - let Some(target) = TARGET.get() else { return }; - let Some(address) = target.to_socket_addrs().ok().and_then(|mut values| values.next()) else { return }; - let Ok(mut stream) = TcpStream::connect_timeout(&address, Duration::from_millis(250)) else { return }; - let _ = stream.set_nodelay(true); - let _ = std::io::Write::write_all(&mut stream, format!("{{\"kind\":\"input\",\"input_type\":\"key\",\"code\":{vk},\"down\":{pressed}}}\n").as_bytes()); -} diff --git a/client/native-gui/Cargo.toml b/client/native-gui/Cargo.toml new file mode 100644 index 0000000..b63c4f3 --- /dev/null +++ b/client/native-gui/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "remotedesk-native-gui" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[[bin]] +name = "remotedesk-native" +path = "src/main.rs" + +[dependencies] +egui = "0.31" +egui-wgpu = "0.31" +egui-winit = "0.31" +pollster = "0.4" +wgpu = "24" +winit = "0.30" + +[lints] +workspace = true diff --git a/client/native-gui/src/main.rs b/client/native-gui/src/main.rs new file mode 100644 index 0000000..8a3ac87 --- /dev/null +++ b/client/native-gui/src/main.rs @@ -0,0 +1,215 @@ +#![forbid(unsafe_code)] + +use std::sync::Arc; +use std::time::Instant; + +use egui_wgpu::Renderer; +use egui_winit::State; +use wgpu::SurfaceError; +use winit::application::ApplicationHandler; +use winit::dpi::PhysicalSize; +use winit::event::WindowEvent; +use winit::event_loop::{ActiveEventLoop, EventLoop}; +use winit::window::{Window, WindowId}; + +struct App { + window: Option>, + surface: Option>, + device: Option, + queue: Option, + config: Option, + renderer: Option, + egui_state: Option, + started: Instant, + endpoint: String, + status: String, +} + +impl Default for App { + fn default() -> Self { + Self { + window: None, + surface: None, + device: None, + queue: None, + config: None, + renderer: None, + egui_state: None, + started: Instant::now(), + endpoint: String::new(), + status: "Disconnected".to_owned(), + } + } +} + +impl ApplicationHandler for App { + fn resumed(&mut self, event_loop: &ActiveEventLoop) { + if self.window.is_some() { + return; + } + let window = Arc::new( + event_loop + .create_window( + Window::default_attributes() + .with_title("RemoteDesk") + .with_inner_size(PhysicalSize::new(1280, 800)), + ) + .expect("create RemoteDesk window"), + ); + let instance = wgpu::Instance::default(); + let surface = instance + .create_surface(window.clone()) + .expect("create GPU surface"); + let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions { + power_preference: wgpu::PowerPreference::HighPerformance, + compatible_surface: Some(&surface), + force_fallback_adapter: false, + })) + .expect("find a compatible GPU adapter"); + let (device, queue) = + pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())) + .expect("create GPU device"); + let size = window.inner_size(); + let capabilities = surface.get_capabilities(&adapter); + let format = capabilities.formats[0]; + let config = wgpu::SurfaceConfiguration { + usage: wgpu::TextureUsages::RENDER_ATTACHMENT, + format, + width: size.width.max(1), + height: size.height.max(1), + present_mode: capabilities.present_modes[0], + alpha_mode: capabilities.alpha_modes[0], + view_formats: vec![], + desired_maximum_frame_latency: 2, + }; + surface.configure(&device, &config); + self.egui_state = Some(State::new( + egui::ViewportId::ROOT, + event_loop, + None, + None, + None, + )); + self.renderer = Some(Renderer::new(&device, format, None, 1, false)); + self.window = Some(window); + self.surface = Some(surface); + self.device = Some(device); + self.queue = Some(queue); + self.config = Some(config); + } + + fn window_event( + &mut self, + event_loop: &ActiveEventLoop, + window_id: WindowId, + event: WindowEvent, + ) { + let Some(window) = self.window.clone() else { + return; + }; + if window.id() != window_id { + return; + } + if let WindowEvent::CloseRequested = event { + event_loop.exit(); + return; + } + let Some(state) = self.egui_state.as_mut() else { + return; + }; + let response = state.on_window_event(&window, &event); + if response.repaint { + window.request_redraw(); + } + if let WindowEvent::Resized(size) = event { + if let (Some(surface), Some(device), Some(config)) = + (&self.surface, &self.device, &mut self.config) + { + config.width = size.width.max(1); + config.height = size.height.max(1); + surface.configure(device, config); + } + } + if let WindowEvent::RedrawRequested = event { + self.redraw(&window); + } + } +} + +impl App { + fn redraw(&mut self, window: &Window) { + let (Some(surface), Some(device), Some(queue), Some(config), Some(renderer), Some(state)) = ( + &self.surface, + &self.device, + &self.queue, + &self.config, + &mut self.renderer, + &mut self.egui_state, + ) else { + return; + }; + let raw_input = state.take_egui_input(window); + let full_output = egui::Context::default().run(raw_input, |ctx| { + egui::CentralPanel::default().show(ctx, |ui| { + ui.heading("RemoteDesk"); + ui.label("Native cross-platform client"); + ui.separator(); + ui.horizontal(|ui| { + ui.label("Server"); + ui.text_edit_singleline(&mut self.endpoint); + if ui.button("Connect").clicked() { + self.status = "Connecting".to_owned(); + } + }); + ui.label(format!("Status: {}", self.status)); + ui.label(format!( + "Uptime: {:.1}s", + self.started.elapsed().as_secs_f32() + )); + }); + }); + state.handle_platform_output(window, full_output.platform_output); + let output = match surface.get_current_texture() { + Ok(output) => output, + Err(SurfaceError::Lost) => { + surface.configure(device, config); + return; + } + Err(_) => return, + }; + let view = output + .texture + .create_view(&wgpu::TextureViewDescriptor::default()); + let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("egui"), + }); + let clipped = + egui::Context::default().tessellate(full_output.shapes, full_output.pixels_per_point); + renderer.update_buffers( + device, + queue, + &mut encoder, + &clipped, + &egui_wgpu::ScreenDescriptor { + size_in_pixels: [config.width, config.height], + pixels_per_point: full_output.pixels_per_point, + }, + ); + renderer.render( + &mut encoder, + &view, + &clipped, + &egui_wgpu::ScreenDescriptor { + size_in_pixels: [config.width, config.height], + pixels_per_point: full_output.pixels_per_point, + }, + ); + queue.submit([encoder.finish()]); + output.present(); + window.request_redraw(); + } +} + +fn main() -> Result<(), winit::error::EventLoopError> { + EventLoop::new()?.run_app(&mut App::default()) +} diff --git a/docs/adr/0001-all-rust-runtime.md b/docs/adr/0001-all-rust-runtime.md new file mode 100644 index 0000000..b33ece6 --- /dev/null +++ b/docs/adr/0001-all-rust-runtime.md @@ -0,0 +1,88 @@ +# ADR 0001: All-Rust RemoteDesk Runtime + +- Status: Accepted +- Date: 2026-08-14 + +## Decision + +RemoteDesk uses Rust for every shipped runtime component. Go and Hysteria2 are +not part of the target architecture. + +The product consists of a Rust Agent server, a Rust native Client, shared Rust +protocol crates, and optional Rust signaling and relay services required by +WebRTC. + +The native Client uses winit for cross-platform windows and input events, wgpu +for GPU rendering, egui for the control interface, and str0m for WebRTC and +DataChannel support. Platform hardware codecs provide video decode and platform +audio APIs are exposed through Rust abstractions. + +The Agent uses str0m WebRTC, platform capture and input APIs, platform hardware +video encoders, Opus audio, and Rust HTTPS/WebSocket signaling where required. + +## Transport + +WebRTC is the primary real-time transport: + +- RTP/RTCP carries video and audio. +- A reliable ordered DataChannel carries authentication-bound control, + keyboard, clipboard, and session state. +- An unordered limited-retransmission DataChannel carries pointer movement and + disposable telemetry. +- ICE/STUN provides direct connectivity and TURN provides relay fallback. + +Large file transfer and software updates use a separate reliable Rust +HTTPS/QUIC service so they cannot block real-time media or input. + +## Media and rendering + +Compressed video may pass through ordinary Rust byte buffers. Decoded raw +frames must remain on platform GPU surfaces and be rendered by the native +client without JavaScript, WebView, Tauri IPC, or CPU bitmap transport. + +Target platform paths are Windows Graphics Capture or DXGI with Media +Foundation and D3D11/D3D12; Linux PipeWire, DMA-BUF, VA-API, and Vulkan; and +macOS ScreenCaptureKit, VideoToolbox, and Metal. + +wgpu is the cross-platform presentation abstraction. Platform-specific interop +remains isolated behind Rust modules where wgpu cannot directly import a codec +surface. + +## Removed architecture + +The following are not part of the target architecture: + +- Go runtime components and the Go Hysteria2 adapter. +- Hysteria2 as a default or optional transport. +- Tauri, WebView, React, or JavaScript as the shipped desktop GUI. +- Software video fallback and compatibility modes. +- A separately shipped Edge application unrelated to the Agent/Client model. + +Existing source and historical documentation may temporarily describe these +components during migration. This ADR takes precedence wherever descriptions +conflict. + +## Repository target + +```text +agent/ Rust server, capture, encode, input, audio, and WebRTC +client/ Rust native client, GUI, decode, render, input, and audio +protocol/ Shared Rust protocol and message contracts +packaging/ Windows, Linux, and macOS packages for Agent and Client +docs/ Architecture decisions and operational documentation +``` + +## Consequences + +- One language, toolchain, lockfile, and security review boundary covers the + shipped application. +- The client avoids WebView frame copies and browser-process overhead. +- WebRTC supplies NAT traversal and real-time media feedback instead of a + custom QUIC media protocol. +- str0m does not provide capture, codecs, rendering, or TURN; these remain + explicit platform and deployment responsibilities. +- Migration is complete only after Go/Hysteria2 and Tauri/Web assets are + removed from builds, packages, CI, and current documentation. + +The ordered implementation and acceptance plan is maintained in +[All-Rust Migration Plan](../all-rust-migration.md). diff --git a/docs/all-rust-migration.md b/docs/all-rust-migration.md new file mode 100644 index 0000000..6e6326a --- /dev/null +++ b/docs/all-rust-migration.md @@ -0,0 +1,160 @@ +# 全 Rust 架构迁移计划 + +本文是 [ADR 0001](adr/0001-all-rust-runtime.md) 的实施计划。它描述目标和验收门槛， +不代表对应能力已经实现。实际完成度以 [实现状态](implementation-status.md) 为准。 + +## 迁移原则 + +- Agent 是服务端，Client 是原生客户端，发布运行时全部使用 Rust。 +- 实时会话使用 str0m WebRTC；不自研替代 WebRTC 的媒体传输协议。 +- 原始视频帧不得经过 JavaScript、WebView、Tauri IPC 或跨进程 CPU bitmap。 +- 每一阶段先建立可重复测试和性能基线，再删除被替代实现。 +- 迁移期间旧代码不得继续进入正式安装包或默认 CI 发布产物。 + +## 阶段 1：删除 Go/Hysteria2 构建链 + +状态：未开始。 + +工作： + +- 从 Windows、Linux 和聚合打包脚本中删除 Hysteria2 构建步骤。 +- 从 CI、发布清单、安装器、systemd 和环境模板中删除 Go/Hysteria2。 +- 删除 transport/hysteria2-agent、go.mod、go.sum 及生成物引用。 +- 移除 Go 工具链、缓存和供应链审计要求。 + +验收：Cargo workspace、CI 和所有安装包不调用 Go；仓库不存在被引用的 Hysteria2 +二进制、服务或配置；Rust Agent 与 Client 的现有非 Hysteria2 构建仍通过。 + +## 阶段 2：新增共享 WebRTC Rust crate + +状态：未开始。 + +工作： + +- 新增共享 webrtc crate 并锁定 str0m 版本。 +- 封装 ICE、SDP、DTLS、SRTP、RTP/RTCP 和 DataChannel 事件状态机。 +- 定义 socket 驱动、时钟、超时、证书指纹和会话身份接口。 +- 对所有消息、候选地址、SDP 和队列设置上限。 + +验收：crate 在 Windows、Linux 和 macOS 目标上编译；状态机具备确定性单元测试、 +畸形输入测试和超时测试；上层代码不直接依赖 str0m 内部类型。 + +## 阶段 3：Agent/Client Loopback + +状态：未开始。 + +工作： + +- 在同机建立 Rust Agent 与 Rust Client 的 SDP offer/answer 交换。 +- 支持 trickle ICE、候选结束、DTLS 指纹校验和 ICE restart。 +- 建立可靠有序 control DataChannel。 +- 建立不可靠无序 pointer DataChannel。 +- 定义协议版本、消息大小、速率和权限边界。 + +验收：自动化测试完成 SDP、ICE、DTLS 和 DataChannel ping/pong；断开、重连、超时、 +伪造指纹、乱序和超限消息均按预期失败；输入通道不被媒体测试流阻塞。 + +## 阶段 4：接入 H.264 RTP + +状态：未开始。 + +工作： + +- 固定 H.264 SDP profile、packetization-mode 和时钟频率。 +- 实现 RFC 6184 单 NAL、STAP-A 和 FU-A 打包与重组。 +- 接入 RTP 序列号、时间戳、帧边界、NACK、PLI、RTX 和关键帧恢复。 +- 使用一到两帧有界队列并丢弃过期帧。 + +验收：录制码流可在 Agent 与 Client 间连续传输；随机和突发丢包测试可恢复； +缺片帧不呈现；关键帧丢失会请求新 IDR；媒体队列不会无界增长。 + +## 阶段 5：平台硬件编码与解码 + +状态：未开始。 + +工作： + +- Windows 接入 Windows Graphics Capture 或 DXGI、Media Foundation 和 D3D11/D3D12。 +- Linux 接入 PipeWire、DMA-BUF 和 VA-API。 +- macOS 接入 ScreenCaptureKit、VideoToolbox 和 Metal。 +- 将编码器、解码器和 GPU surface 约束到可验证的 Adapter/device。 +- 不提供软件编码、软件解码或 CPU bitmap 兼容回退。 + +验收：每个平台至少一个硬件路径完成端到端测试；运行时可证明实际硬件后端、 +surface 类型和 Adapter；硬件能力不足时明确拒绝会话，不静默降级。 + +## 阶段 6：接入 wgpu 呈现 + +状态：未开始。 + +工作： + +- 使用 winit 管理窗口、显示器、DPI、全屏和输入生命周期。 +- 使用 wgpu 创建高性能 Adapter、surface 和呈现管线。 +- 为平台解码 surface 建立零拷贝或有证据约束的 GPU interop。 +- egui 仅负责控制界面，不读取视频像素。 + +验收：Windows、Linux 和 macOS 均可呈现测试视频；resize、DPI、全屏、设备丢失和 +显示器切换可恢复；不存在通过 JavaScript、WebView 或 CPU bitmap 的帧路径。 + +## 阶段 7：接入 Opus + +状态：未开始。 + +工作： + +- Agent 采集系统输出并编码 48 kHz Opus。 +- 通过独立 RTP 音频 Track 发送并维护统一单调时钟。 +- Client 使用有界 jitter buffer 解码和播放。 +- 音频错误与视频、输入生命周期隔离。 + +验收：音频连续播放且无无界积压；丢包时使用 Opus PLC；音视频漂移受控； +音频设备切换或失败不会阻塞视频和输入。 + +## 阶段 8：接入 STUN/TURN + +状态：未开始。 + +工作： + +- 支持 host、server-reflexive 和 relay candidates。 +- 接入 STUN、TURN/UDP、TURN/TCP 和 TURN/TLS 443。 +- 使用短期、会话绑定的 TURN 凭据。 +- 支持 ICE restart、网络切换、候选优先级和路径诊断。 + +验收：局域网直连、不同 NAT、公网 TURN/UDP 和受限网络 TURN/TLS 均完成测试； +客户端显示实际路径、RTT 和 relay；凭据过期、重放和跨会话使用均失败。 + +## 阶段 9：更新安装包和 CI + +状态：未开始。 + +工作： + +- Windows、Linux 和 macOS 只打包 Rust Agent、Rust Client 和必要资源。 +- CI 覆盖格式化、Clippy、测试、跨平台编译、SBOM、签名和安装验证。 +- 增加 WebRTC loopback、RTP 丢包、DataChannel 和包内容测试。 +- 删除 npm、Tauri、WebView2 和 Go 的发布依赖。 + +验收：三平台产物可安装、升级和卸载；包内容白名单通过；CI 不下载或执行 Go、 +Node、Tauri CLI 或 WebView 构建工具；发布产物具备签名、校验和和 SBOM。 + +## 阶段 10：删除旧 Tauri/WebView 与兼容路径 + +状态：未开始。 + +工作： + +- 删除 client/web、Tauri app shell、React、Vite、npm lockfile 和 WebView 配置。 +- 删除软件视频回退、zlib framebuffer、MSTSC fallback 和 Compatibility 策略。 +- 删除旧 helper、旧协议字段、旧测试、旧打包入口和失效文档。 +- 更新安全模型、用户指南、实现状态和架构图。 + +验收：仓库搜索不再出现生产 Tauri/WebView/Hysteria2/Go/Compatibility 入口； +所有正式功能通过 Rust 原生 Client 和 Rust Agent 完成；完整 workspace、安装包和 +端到端测试通过。 + +## 完成定义 + +只有十个阶段全部达到验收条件，并且旧构建链不再产生发布产物，才能将全 Rust +迁移标记为完成。存在源码骨架、未运行的平台代码或仅通过 cargo check 均不算完成。 diff --git a/docs/architecture.md b/docs/architecture.md index 33abca7..6802aa0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,5 +1,10 @@ # RemoteDesk 总体架构 +> 当前生效的目标架构以 [ADR 0001](adr/0001-all-rust-runtime.md) 为准： +> Agent、Client、协议和实时通信运行时全部使用 Rust，客户端采用 +> winit、wgpu、egui，实时传输采用 str0m WebRTC。下文若出现 Tauri、 +> WebView、Go、Hysteria2、兼容模式或独立 Edge，均视为迁移历史。 + ## 1. 目标和边界 RemoteDesk 解决两个问题： @@ -105,6 +110,36 @@ Agent 通过出站 TLS WebSocket 连接最近的 Rendezvous Gateway，客户端 Windows RDP 不使用 ICE/TURN。其 CDN 加速需要目标网络中的出站 L4 Edge Connector，在两端 POP 间透明转发 RDP TCP/UDP；RDP TLS/NLA 仍由 IronRDP 与目标 Windows 端到端完成。 +### 3.5 Windows Headless Native Endpoint + +Windows 被控端的高性能桌面会话使用 IDD/IddCx 创建 Headless SDR 虚拟显示器。IDD 只负责虚拟显示器、显示模式和 swap-chain 生命周期，不负责编码、网络或业务授权。当前产品约束为 SDR 8-bit；IDD 不作为 HDR10/10-bit 采集源。 + +正式组件边界如下： + + Windows Service / Go Backend + - session、认证、配置和生命周期 + - 输入路由（独立高优先级控制面） + - Hysteria2 路径和媒体策略 + + Windows Capture Worker（Rust/C++ 原生 helper） + - IDD frame 或 Desktop Duplication 兼容捕获 + - BGRA8 GPU texture -> NV12 GPU conversion + - NVENC / AMF(VCE/VCN) / Quick Sync + - x264 / x265 / SVT-AV1 软件兜底 + - 独立视频和音频队列 + - 有界媒体 datagram / 控制流 + + IDD Driver + - Headless SDR 虚拟显示器和 4K/120 模式 + +Windows Headless 主路径优先直接消费 IDD GPU frame；第一阶段允许使用 Desktop Duplication 复制已存在的虚拟 output 作为兼容实现。两种路径都必须在同一 DXGI Adapter 上完成 GPU 处理，原始 BGRA8 不得进入 Go、JSON、Tauri IPC 或网络队列。 + +Windows Headless 视频默认支持 H.264/AVC、H.265/HEVC 和 AV1。编码器先尝试同 Adapter 的硬件后端：NVIDIA 使用 NVENC，AMD 使用 AMF/VCE/VCN，Intel 使用 Quick Sync；硬件能力验证失败时按协商结果切换 x264、x265 或 SVT-AV1。软件编码只能作为明确的兼容性降级，并自动降低分辨率、帧率或并发上限。 + +Windows Headless 音频独立使用 WASAPI loopback 或配置的虚拟音频 endpoint 采集，编码为 Opus，通过独立音频流发送。音频和视频使用同一会话单调时钟；输入不等待任何音频、视频帧、编码器或呈现确认。 + +无硬件 GPU 时启用 compatibility 模式。启动阶段依次探测硬件 DXGI Adapter、IDD/DDA output、WARP/Basic Display Adapter、硬件编码器和软件编码器；如果只有软件路径可用，允许 DDA/IDD 继续采集，但把像素路径标记为 software，并按 CPU 实测上限自动降低分辨率、帧率和并发数。compatibility 模式不得继续宣称 zero-copy、hardware_encode 或多路 4K/120 保证；strict 性能模式则在硬件编码器不可用时拒绝请求。 + ## 4. Linux 会话模式 Wayland 不允许普通进程静默截屏和任意注入输入，这是安全边界，不应通过默认 root 运行 Agent 来绕过。 diff --git a/docs/development.md b/docs/development.md index 5bbd6cc..4e2a0ed 100644 --- a/docs/development.md +++ b/docs/development.md @@ -7,6 +7,12 @@ - Windows RDP viewer 的安全 Named Pipe 和 Tauri 2 桌面壳已完成源码接线；Linux 远程 H.264 的 RFC 6184 接收、Media Foundation NV12 解码和 D3D11 VideoProcessor 窗口呈现也已接线，但尚未编译、实机验证或通过 ETW/GPUView 证明硬件解码与隐藏复制边界。Tauri 和媒体新依赖按本轮要求尚未解析或编译。 - Linux PipeWire、Portal、DRI3 与硬件编码器必须在对应 Linux 测试机执行 M0 Spike。 +## 1.1 Windows Headless compatibility 验收 + +无 GPU 兼容模式必须覆盖仅 WARP、Microsoft Basic Display Adapter、Intel 核显和虚拟 GPU 环境；验证 DDA 的 DuplicateOutput/AcquireNextFrame 成功与失败、硬件编码器不可用时的软件编码选择、自动降到 1080p30/60 或更低档位、输入延迟不受软件编码阻塞、视频丢帧和关键帧恢复。诊断必须明确显示 software、cpu_upload、degraded 和不支持 4K/120。 + +兼容模式只在用户显式选择时启用。没有可复制的 DXGI Output 时应返回结构化 capture unavailable；WARP 或 Basic Display Adapter 不得被报告为硬件 GPU 或硬件编码。 + ## 2. Rust workspace ```powershell diff --git a/docs/gpu-acceleration.md b/docs/gpu-acceleration.md index 177d2b7..75fad70 100644 --- a/docs/gpu-acceleration.md +++ b/docs/gpu-acceleration.md @@ -13,6 +13,22 @@ Windows 客户端的 UI、远程视频、RDP 图形和终端均支持 GPU 加速 这里的“零拷贝”特指未压缩视频像素从捕获到编码、或从解码到呈现不经过 CPU map/copy。压缩码流和 RTP 包仍可经过普通内存。跨 Adapter GPU copy 不算零拷贝。 +### Windows Headless compatibility 模式 + +没有独立 GPU 不等于一定没有 DXGI 输出：机器可能存在 Intel 核显、虚拟 GPU、Microsoft Basic Display Adapter 或 WARP。采集端必须以运行时结果区分这些情况，不得因为成功创建 ID3D11Texture2D 就报告硬件加速。 + +探测顺序： + +1. 枚举 DXGI Adapter 和 Output。 +2. 尝试创建 D3D11 device 和 Desktop Duplication。 +3. 尝试 AcquireNextFrame，并验证连续帧。 +4. 探测 NVENC、AMF/VCE/VCN、Quick Sync 的实际编码能力。 +5. 硬件路径不可用时，若用户显式选择 compatibility，则允许 WARP/CPU surface 和 x264、x265、SVT-AV1。 + +compatibility 路径必须报告 adapter_type、surface_type、cpu_map_count、cpu_upload_bytes、encoder_backend、software_encode_latency_us、encode_fps 和 degraded。WARP 是软件 D3D 实现，不能报告为硬件 GPU；Microsoft Basic Display Adapter 也不能视为硬件编码器。 + +无硬件编码器时，系统自动应用软件上限。默认先尝试 1080p30/60 或实测可承受的更低档位；请求多路 4K/120 时，strict 性能模式返回 HARDWARE_ENCODER_UNAVAILABLE，compatibility 模式降档并明确显示 software/degraded。软件视频队列有界、允许丢帧，且不阻塞输入和音频线程。 + ## 2. 渲染架构 ```text diff --git a/docs/implementation-status.md b/docs/implementation-status.md index 95fc42c..e52d2ee 100644 --- a/docs/implementation-status.md +++ b/docs/implementation-status.md @@ -39,6 +39,10 @@ IronRDP 单屏 resize 源码现已补齐单调 generation、调整后尺寸回 Linux 用户会话现已将 X11 与 Wayland 生命周期收敛到统一 Agent 后端。Wayland 默认编译 `wayland-eis`，通过用户 D-Bus 实际查询 RemoteDesktop/ScreenCast Portal，按 `CreateSession -> SelectDevices -> SelectSources -> Start` 顺序授权，持有 PipeWire remote 与 EIS fd，并以 DMABUF 严格硬件 H.264 管线发送；Portal/EIS/VA 能力不足时返回明确错误，不伪装成 X11 或 CPU 回退。控制服务的 `linux.wayland_desktop` 现在表示 Linux Agent helper 已包含该能力，具体桌面是否可用仍由 Agent status 的 Portal probe 和会话启动结果决定。 +Windows Headless 方案状态：架构已确定，实现在尚未实现表中跟踪。目标是 IDD/IddCx SDR 8-bit 虚拟显示器、Desktop Duplication 首阶段兼容捕获或后续 IDD GPU frame、NVENC/AMF/VCE/VCN/Quick Sync 硬件优先、x264/x265/SVT-AV1 软件兜底、独立 Opus 音频、输入优先控制流和 Hysteria2 视频 datagram。IDD 不提供 HDR10/10-bit 能力；本方案不承诺 HDR10。 + +Windows Headless compatibility 模式允许无硬件 GPU 主机使用可用的 Desktop Duplication/IDD 输出、WARP 或 CPU 编码，但只报告 software/degraded，并根据实测 CPU 能力降低分辨率、帧率和并发上限。该模式不保证多路 4K/120；strict 性能模式在硬件编码器不可用时拒绝请求。当前仍未完成真实无 GPU 主机验收。 + ## 尚未实现 | 路线 | 未实现的主要能力 | @@ -47,6 +51,8 @@ Linux 用户会话现已将 X11 与 Wayland 生命周期收敛到统一 Agent | M1 Windows RDP MVP | 原生客户端全部本地显示器布局、自定义显示器子集、D3D11 呈现路径已编译并通过自动化测试；仍缺真实主机验收、硬件解码/零拷贝、断线恢复实机验收、Credential Manager 远端 NLA 验收和正式签名安装包 | | M2-M4 Linux Agent | X11 `GetImage`/XTEST、Windows 软件桌面窗口、权限路由、有界帧协议、三次有界桌面重连、用户会话进程内 15 秒绑定令牌续接、GStreamer H.264 AU 编码/WebRTC sender、Linux 输出 monitor Opus 编码/有界 IPC、Windows 远端 H.264 RTP 消费/原生呈现、Opus 解码/WASAPI 播放、认证 WSS 直连 WebRTC 信令、X11/Windows 双向 UTF-8 文本剪贴板，以及 Windows Credential Manager 支持的 helper 崩溃后短时续接已完成源码接线并通过相关 Rust 测试；真实 Linux/Windows 主机验收仍未完成。Wayland Portal 已完成能力探测、认证后授权、PipeWire/EIS fd 所有权、严格 DMABUF-to-VA-H.264 会话循环、呈现 ACK、resize 重建、`reis` EIS 输入和绑定 Portal/EIS 的 15 秒会话恢复源码；仍缺 GNOME/KDE/Xorg/纯终端端到端验证和 Wayland 文本剪贴板实现 | | M5 体验和性能 | Linux X11 兼容帧链路已完成采集/编码/解码/呈现耗时源码遥测，并已接入基于呈现 ACK 的有界帧率和 zlib level 1..6 压缩策略自适应；IronRDP 已完成网络图形 PDU 本地处理与像素转换耗时源码遥测；相关 Rust/Web 自动化已通过。正式更新源发布自动化和 Authenticode 签名/验签已完成源码接线但尚未执行生产发布；仍缺 RDP 服务端编码遥测、H.264/Opus 码率与编解码策略自适应、HEVC/AV1、通用会话崩溃恢复，以及真实证书/时间戳/下载/原位升级验收；文件上传/下载断点续传已实现 | +| Windows Headless Native | 部分 | IDD/IddCx 虚拟显示器、IDD GPU frame、DDA 到 GPU 编码器 zero-copy、NVENC/AMF/VCE/VCN/Quick Sync、HEVC/AV1 和认证 session lifecycle 仍未完成；独立 Opus/WASAPI 源码链已接入 compatibility runtime，但统一 QPC 精确同步和实机验收未完成；SDR 8-bit H.264 compatibility 已接通到 RDV1 GUI；HDR10 明确不在目标范围内 | +| Windows Headless compatibility | 部分 | DDA staging/readback、CPU BGRA->NV12、Media Foundation H.264、Annex-B、encoded mmap ring、Go RDV1/RDA1/Hysteria2、Rust RDV1 重组、IDR 恢复、GUI MFT/D3D11 呈现，以及独立 WASAPI loopback/Opus/audio ring/jitter/PLC/FEC/WASAPI playback 源码已接入；无 GPU、4K/120、多路压力、统一 QPC 精确音画同步和真实 Windows 端到端验收仍未完成 | | M6 CDN 边缘中继 | 已有单节点短 TTL Presence、Linux Agent 心跳、Windows Client 签名 Session Intent、Agent 本地授权、角色票据、远程首次配对、已知设备 outbound-only rendezvous、Linux 终端/文件 TLS-over-relay transport、设备签名的有界双向 SDP/ICE mailbox、授权响应内的 session-bound TURN 短期凭据、Agent/Client 共用的真实 PeerConnection/H.264/RTX/Opus/DataChannel 传输内核、Linux X11 捕获/GStreamer H.264 与 Opus track 发送，以及 Windows H.264 RTP 解码/呈现和 Opus/WASAPI 播放源码；Edge、Agent runtime、Linux terminal 和 native video 的自动化测试已通过。仍缺实际 coturn POP 部署与 UDP/TLS 验收、真实双 POP 骨干指标/亲和/故障切换、RDP TCP/UDP L4 Edge Connector、静态发布 CDN 与跨网络端到端验证 | ## 运行时能力接口 @@ -176,3 +182,26 @@ Linux 用户会话现已将 X11 与 Wayland 生命周期收敛到统一 Agent - Windows 原生 RDP 自定义显示器子集现已完成 secure-pipe JSON、控制端请求、Web UI 选择、viewer 参数和 Win32 monitor layout 筛选：索引必须唯一、范围有效且包含主显示器，筛选后重新计算组合窗口边界；viewer 单元测试和 Web build 已通过。仍需真实多显示器 Windows RDP 服务端验收。 - 自定义显示器子集接入后的控制端 MSI 已重新生成，最新 SHA-256 为 `66b34ea4a0292e6a4ddb5b8feab56ead19d6660135785f1893405e63cec4c7f0`（未签名开发包）。 - 上述结果证明可编译和自动化测试覆盖，不等同于 Windows RDP 服务端、GNOME/KDE/Xorg、PipeWire/VA 驱动、coturn 跨网络或硬件解码的实机端到端验收；这些项目仍保持原有“部分”状态。 + +## Windows Headless 媒体实现状态 + +本轮已完成以下可运行基础： + +- protocol/src/windows_media.rs 提供 SDR 8-bit/NV12、H.264/HEVC/AV1、Opus、硬件优先和 compatibility 降级的校验模型。 +- transport/hysteria2-agent 已接入 RDV1 视频分片大小校验和 Hysteria2 UDP datagram 转发；视频重组超时丢弃，不重传过期视频。Go client 还提供可选的 encoded-ring consumer，将 Rust 的 H.264 Annex-B access unit 复制、分片并发送到同一认证 Hysteria2 UDP session。 +- agent/windows-agent/src/media_ring.rs 提供跨进程 memory-mapped SPSC ring 和 session-bound encoded-only descriptor，承载已编码码流，不承载原始 BGRA/NV12。满队列时非阻塞返回，调用方可丢弃视频包；Windows H.264 compatibility worker 已写入该 ring，Go Hysteria2 client 可通过 `--encoded-ring` 打开消费者并转成 RDV1 datagram。Windows 创建 ring 时使用受保护 DACL，只授权当前用户 SID，并允许同一会话消费者映射及关闭清理。 +- remotedesk-windows-agent pipe 已使用当前用户 SID ACL、拒绝远程 Pipe 客户端和首实例保护；默认要求 `REMOTEDESK_AGENT_PIPE_TOKEN` 提供随机 32 字节 base64url bootstrap，每个连接以随机 nonce 执行 HMAC-SHA256 challenge，认证总时限 10 秒，认证后才把 `authenticated: true` hello 和 ring descriptor 交给 Go bridge。`--allow-unauthenticated` 与 loopback TCP 自动编排只保留本地开发兼容。 +- Windows agent 的桌面请求支持 capture_mode: strict_hardware|compatibility 和 video_codec: h264|hevc|av1。Windows 兼容捕获已实现 Desktop Duplication 的 D3D11 staging/readback 路径，并在 DDA 不可用时回退 GDI；响应会报告实际后端、cpu_readback、cpu_map_count 和 degraded。H.264 compatibility worker 已完成 DDA BGRA -> CPU NV12 -> Media Foundation MFT -> Annex-B AU -> encoded ring 的短样本运行链；`hardware_encoder_verified` 只表示 MFT 短样本成功，`hardware_pipeline_verified` 仍为 false，因此 strict 请求仍返回结构化不可用错误。 +- protocol 的 RDV1 decoder/reassembler 严格校验 58 字节 header、未知 flags、分片位置、元数据一致性、重复分片和 16 MiB 上限；首帧、序号中断、分片过期或 discontinuity 后丢弃 P-frame，直到收到 IDR。viewer 以一槽待处理门控接收 loopback RDV1，积压时丢视频而不阻塞输入；H.264 Annex-B 进入 D3D11-aware Media Foundation decoder，只接受同 device NV12 DXGI surface，再由 D3D11 VideoProcessor 呈现。 +- protocol 与 Go transport 的 RDA1 decoder 严格校验 40 字节 header、version、reserved byte、已知 flags、stream/generation/sequence、10/20/40/60 ms duration 和 4 KiB Opus payload 上限；音频 datagram 不与视频帧重组，也不等待视频。Windows Agent 使用独立线程执行 WASAPI loopback、48 kHz 双声道归一化、20 ms Opus 编码并写入独立 audio mmap ring；Go bridge 独立轮询并发送 RDA1。audio ring 或 callback 积压时显式标记 discontinuity，不阻塞视频或输入。 +- windows-agent-viewer 在 --media-udp 模式严格区分 RDV1/RDA1，使用容量 64 的非阻塞接收队列、三包启动缓冲、最多 64 包的有界 jitter map、Opus in-band FEC/PLC、按实际输出设备采样率重采样和 CPAL/WASAPI playback。音频设备或解码失败只关闭音频；音频播放时钟建立后，viewer 丢弃明显落后的旧视频帧，不等待未来视频帧。播放端现在以连续按序音频 PTS 估计源时钟偏差，并结合 60 ms 目标 PCM 占用以原子 ppm（±0.5%）调节重采样步长；stream/generation/discontinuity 重置时清空旧 PCM、清零校正并重新预缓冲。 +- viewer 最多每 250 ms 合并一次 `request_keyframe`，经独立可靠输入连接按 session ID 路由到 Agent 的一槽请求队列；编码 worker 使用 `ICodecAPI/CODECAPI_AVEncVideoForceKeyFrame` 请求 IDR。Go Hysteria2 client 可通过 `--windows-agent-pipe` 完成 challenge 后自动打开编码会话并严格校验 ring descriptor，也保留手工 `--encoded-ring` 与开发用 loopback `--windows-agent-control` 模式。 + +以下仍未完成，不能作为已支持能力发布： + +- IDD/IddCx 虚拟显示器驱动、Desktop Duplication 到 GPU 编码器的 zero-copy capture、BGRA 到 NV12 的 D3D11 GPU conversion。 +- NVENC、AMF/VCE/VCN、Quick Sync 真实编码器探测与编码 worker，以及 x264/x265/SVT-AV1 真实 fallback。 +- 音视频生产端统一 session-start QPC/PTS 映射已接入 Windows Agent compatibility 链；viewer 接收端已完成连续音频 PTS 漂移估计、60 ms PCM 占用反馈和 ±0.5% 有界动态重采样。仍需 Windows 实机长时间稳定性、默认设备切换/拔出和跨设备音画端到端验收，不能据此宣称已完成生产级硬件媒体链。 +- WASAPI 默认设备切换/拔出、无输出设备、callback error、长时间稳定性、内置 libopus 产物体积/许可证和真实 Windows 音画端到端验收。 +- 认证 Named Pipe 和 ring 当前按 Windows 当前用户 SID 隔离；生产安装器/控制服务仍需负责生成每次 Agent 生命周期唯一的 bootstrap、仅注入目标 Agent 与 Go bridge、启动后清除父进程环境，并把两进程纳入统一 Job/service 生命周期。viewer 的输入事件使用独立持久连接和有界优先队列，Win32 全局键盘 hook 复用该队列。 +- Windows 实机下的多路 4K/120 性能、实际丢包/IDR 恢复和 NVENC/AMF/QSV 多路 session 压力测试。 diff --git a/docs/network-adaptation.md b/docs/network-adaptation.md index de5d88b..704f9da 100644 --- a/docs/network-adaptation.md +++ b/docs/network-adaptation.md @@ -24,6 +24,26 @@ RDP 的网络恢复能力由 IronRDP 协议实现决定。M0 必须验证： - RDP UDP multitransport。 - 图形管线在丢包和重连后的恢复行为。 + +### 2.3 Windows Headless Hysteria2 + +Windows Headless 使用 Hysteria2 作为网络承载，但应用层必须保持输入、音频和视频分流： + +control stream：输入、心跳、配置和 KeyframeRequest，可靠且最高优先级。 + +audio path：Opus，独立 jitter buffer，允许 PLC/FEC。 + +video datagram：H.264/HEVC/AV1，允许丢包和丢帧。 + +file stream：文件和剪贴板，可靠、最低优先级、限速。 + +视频不能放入会因丢包而阻塞后续数据的可靠流。Hysteria2 的 datagram 能力必须由具体 adapter 验证；不能仅因为底层使用 UDP 就假设应用已经具备不可靠语义。若某部署只能提供可靠 QUIC stream，Windows Headless 视频通道不得宣称实时丢包模式。 + +视频分片带有 stream_id、generation、frame_id、fragment_id、fragment_count、PTS 和关键帧标记。缺少任意分片时丢弃整个视频帧；重组超时或帧已过期时立即丢弃；不重传过期视频包。关键帧或参考链损坏后通过控制流请求 IDR。 + +无硬件 GPU 时 compatibility 模式会进一步受 CPU 编码预算限制。编码器报告 software/degraded 后，质量控制器必须优先降低视频帧率和分辨率，再降低码率；视频队列不得因 CPU 编码变慢而无限增长。输入控制和 Opus 音频保持独立调度，软件编码过载时可以丢弃过期视频帧，但不能阻塞输入或音频。 + +Hysteria2 的带宽和队列不能让视频占满控制面。输入预留最高优先级，音频次之，视频使用有界队列，文件只使用剩余带宽。必须监控路径 RTT、抖动、队列延迟、包丢失、输入延迟和音画偏差，而不是只观察总吞吐。 - 动态调整视觉效果、色深或编码模式的能力。 - UDP 不可用时 TCP 模式的延迟和队头阻塞表现。 diff --git a/docs/protocol.md b/docs/protocol.md index 67e40fd..8a884e5 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -217,6 +217,24 @@ DisplaySelection: Windows RDP helper 把验证后的本地布局映射为 Display Control Dynamic Monitor Layout。Linux Agent 把同一选择模型映射到 Portal/PipeWire 或 XRandR 捕获源，但不会修改 Linux 实体显示模式。 +### 8.5 Windows Headless Compatibility Contract + +Windows Headless compatibility 模式允许无硬件 GPU 主机使用可用的 Desktop Duplication 或 IDD 输出、WARP 或 CPU 编码，但必须显式报告 software 和 degraded。能力结果至少包含 capture_backend、adapter_type、surface_type、encoder_backend、hardware_encoder_verified、hardware_pipeline_verified、hardware_path_verified、cpu_map_count、max_width、max_height、max_fps 和 concurrent_sessions。`hardware_encoder_verified` 只代表真实短样本编码成功；DDA staging/readback 仍有 CPU map，因此该路径的 `hardware_pipeline_verified` 和 `hardware_path_verified` 必须为 false。若请求硬性要求 4K/120 或硬件编码，而本地只有 software 路径，Agent 返回 HARDWARE_ENCODER_UNAVAILABLE，不得静默降级。 + +Windows Headless 媒体契约固定为 SDR 8-bit：video codec 为 H.264/AVC、H.265/HEVC 或 AV1，输入格式为 NV12，audio codec 为 Opus。IDD 不提供本项目的 HDR10/10-bit 能力，不得出现 P010、Main10 或 HDR metadata。 + +Windows Headless 视频使用 Hysteria2 不可靠 datagram，允许丢失。缺少一个分片时丢弃整个帧；过期帧不重传；首帧非 IDR、sequence 中断、discontinuity、解码失败或关键帧损坏时，经独立可靠控制流发送带 session ID 的 `request_keyframe`。Client 以 250 ms 最小间隔合并请求，Agent 以容量 1 的队列合并请求并通过 `CODECAPI_AVEncVideoForceKeyFrame` 请求编码器输出 IDR。收到新 codec config 和 IDR 后才能恢复普通帧。 + +本地 encoded-ring descriptor 只能经 `\\.\pipe\RemoteDesk\...` 命名空间交付。Pipe DACL 只授权当前用户 SID，拒绝远程客户端并启用首实例保护；双方从进程环境取得同一个随机 32 字节 base64url bootstrap，Agent 为每个连接发送 32 字节随机 nonce，Go bridge 返回 `HMAC-SHA256(key, "RemoteDesk Windows Agent pipe auth v1\\0" || nonce)`。bootstrap 不进入命令行或 JSON，双方读取后从各自环境删除；认证和 hello 共用有界握手时限。仅 `authenticated: true` 的连接可作为生产 descriptor 来源，ring 路径、session ID、generation、owner PID 和 geometry 必须全部校验后才能映射。 + +RDA1 音频 datagram 使用固定 40 字节 little-endian header：magic/version/reserved/header_bytes、stream ID、generation、sequence、PTS、duration_ms 和 flags，后接一个不超过 4 KiB 的独立 Opus packet。reserved 必须为 0，当前仅定义 discontinuity flag，duration 仅允许 10/20/40/60 ms；未知 flags、零 identity、空 payload 和超限 packet 必须在进入 jitter buffer 前拒绝。音频 packet 不依赖 RDV1 frame ID，不因视频分片缺失而等待或丢弃。 + +输入控制使用独立可靠高优先级流，带 input_seq 和 layout_generation。输入发送不等待视频帧、编码完成、媒体 ACK 或音频播放。文件和剪贴板使用独立可靠低优先级流，不能阻塞输入、音频或视频。 + +音视频共享 session clock/QPC 映射和 generation，但独立采集、编码、队列和发送。客户端以音频播放时钟同步视频，视频落后时丢弃旧帧，不为追求完整帧率而增加播放延迟。 + +Hysteria2 音频包使用独立的 RDA1 datagram，不进入视频分片重组器。固定 40 字节头包含 stream、generation、sequence、100ns media PTS、duration（10/20/40/60 ms）和 flags，Opus payload 上限为 4 KiB。音频包独立丢弃或播放，视频缺片不能阻塞音频，音频丢包也不能阻塞视频。Windows compatibility runtime 已使用独立 worker 完成 WASAPI loopback、48 kHz 双声道 20 ms Opus、独立 mmap ring、Go RDA1 发送，以及 viewer 有界 jitter/FEC/PLC/WASAPI playback 的源码接线；统一 session-start QPC、长时间漂移校正和真实设备验收仍是发布前条件。 + ## 9. WebRTC 信令 - Offer/Answer 和 ICE candidate 放入 Protobuf 信令消息。 diff --git a/docs/roadmap.md b/docs/roadmap.md index fee4794..bba5193 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,5 +1,10 @@ # RemoteDesk 实施路线 +> 全 Rust 架构迁移的当前执行顺序和验收标准见 +> [全 Rust 架构迁移计划](all-rust-migration.md)。该计划依据 +> [ADR 0001](adr/0001-all-rust-runtime.md)，优先于下文旧版里程碑中的 +> Tauri、WebView、Go、Hysteria2 和兼容模式描述。 + 路线按风险优先，而不是按界面完成度排序。最先验证 Agent 与客户端原生 helper 的双端零拷贝、IronRDP、Wayland Portal、Xorg DRI3、PTY 和差网络自适应等核心技术。 ## M0：技术验证 diff --git a/docs/technology-stack.md b/docs/technology-stack.md index 1fdf9ca..8eb615a 100644 --- a/docs/technology-stack.md +++ b/docs/technology-stack.md @@ -1,5 +1,11 @@ # RemoteDesk 技术栈规划 +> 当前决策：所有发布运行时使用 Rust；Agent 是服务端；原生客户端使用 +> winit、wgpu 和 egui；实时传输使用 str0m WebRTC。Go、Hysteria2、 +> Tauri 和 WebView 不属于目标组件。详见 +> [ADR 0001](adr/0001-all-rust-runtime.md)。下文保留的旧技术栈内容仅用于 +> 迁移对照，后续随代码迁移删除。 + ## 1. 混合架构 RemoteDesk Windows 客户端采用 Web UI 与 Rust 原生会话进程混合方案。JavaScript 负责管理界面和终端；Linux 桌面 WebRTC、Windows RDP 和 GPU 呈现均由 Rust 原生 helper 承载，原始视频像素不进入 WebView。 diff --git a/protocol/src/lib.rs b/protocol/src/lib.rs index 894eafc..1922c34 100644 --- a/protocol/src/lib.rs +++ b/protocol/src/lib.rs @@ -8,6 +8,7 @@ mod edge_signaling; mod features; mod path_validation; mod transcript; +mod windows_media; pub use displays::{ DISPLAY_ID_LENGTH, DisplayAxis, DisplayDescriptor, DisplayId, DisplayLayout, @@ -32,3 +33,13 @@ pub use path_validation::{ pub use transcript::{ NegotiationTranscriptV1, TRANSCRIPT_DOMAIN_V1, TranscriptBuildError, TranscriptInputV1, }; +pub use windows_media::{ + AUDIO_DATAGRAM_HEADER_BYTES, AUDIO_DATAGRAM_MAX_BYTES, AudioPacket, AudioPacketFlags, + AudioStreamConfig, EncoderBackend, EncoderCapability, MAX_AUDIO_PACKET_BYTES, + MAX_VIDEO_FRAGMENT_PAYLOAD, MAX_VIDEO_FRAME_BYTES, MAX_VIDEO_FRAME_FRAGMENTS, MediaPacketFlags, + SelectedEncoder, VIDEO_DATAGRAM_HEADER_BYTES, VIDEO_DATAGRAM_MAX_BYTES, VideoAccessUnit, + VideoCodec, VideoModePolicy, VideoPacket, VideoPixelFormat, VideoProfile, VideoReassembler, + VideoRequest, VideoStreamConfig, WindowsCaptureBackend, WindowsMediaConfigError, + WindowsMediaDatagramError, WindowsMediaPacketError, WindowsMediaSelectionError, + decode_audio_datagram, decode_video_datagram, select_encoder, +}; diff --git a/protocol/src/path_validation.rs b/protocol/src/path_validation.rs index 50e2d1a..43c322c 100644 --- a/protocol/src/path_validation.rs +++ b/protocol/src/path_validation.rs @@ -11,7 +11,6 @@ pub type VerifierChallenge = [u8; VERIFIER_CHALLENGE_LENGTH]; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ZeroCopyPolicy { RequiredEndToEnd, - Compatibility, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -316,9 +315,6 @@ impl StrictPathCoordinator { cross_adapter_copy_count: u64, verification_source: VerificationSource, ) -> Result<(), PathReportError> { - if self.policy == ZeroCopyPolicy::Compatibility { - return Ok(()); - } if !status.satisfies_strict_zero_copy() { return Err(PathReportError::PathDoesNotSatisfyStrictPolicy(status)); } @@ -530,10 +526,10 @@ mod tests { } #[test] - fn compatibility_mode_accepts_software_paths_but_still_needs_both_reports() { + fn strict_mode_rejects_software_paths() { let expected = context(1, 2, 3); let mut coordinator = - StrictPathCoordinator::new(ZeroCopyPolicy::Compatibility, expected.clone()); + StrictPathCoordinator::new(ZeroCopyPolicy::RequiredEndToEnd, expected.clone()); let mut agent = agent_report(&expected); agent.status = MemoryPathStatus::Software; agent.surface_type = MemorySurfaceType::Cpu; @@ -541,15 +537,9 @@ mod tests { agent.verification_source = VerificationSource::Unverified; assert_eq!( coordinator.accept_agent_report(agent), - Ok(StrictPathState::Negotiating) - ); - - let mut client = client_report(&expected); - client.status = MemoryPathStatus::CpuUpload; - client.surface_type = MemorySurfaceType::Cpu; - assert_eq!( - coordinator.accept_client_report(client), - Ok(StrictPathState::Connected) + Err(PathReportError::PathDoesNotSatisfyStrictPolicy( + MemoryPathStatus::Software + )) ); } diff --git a/protocol/src/transcript.rs b/protocol/src/transcript.rs index 0ce66f9..f60e05f 100644 --- a/protocol/src/transcript.rs +++ b/protocol/src/transcript.rs @@ -78,7 +78,6 @@ impl NegotiationTranscriptV1 { 7, &[match input.zero_copy_policy { ZeroCopyPolicy::RequiredEndToEnd => 1, - ZeroCopyPolicy::Compatibility => 2, }], ); append_field(&mut output, 8, &input.client_capabilities_digest); @@ -145,15 +144,6 @@ mod tests { .unwrap() .stable_hash_input(); - let mut changed = transcript_input(); - changed.zero_copy_policy = ZeroCopyPolicy::Compatibility; - assert_ne!( - original, - NegotiationTranscriptV1::new(changed, RequiredFeatures::ALL_KNOWN) - .unwrap() - .stable_hash_input() - ); - let mut changed = transcript_input(); changed.selected_config_hash = [99; 32]; assert_ne!( diff --git a/protocol/src/windows_media.rs b/protocol/src/windows_media.rs new file mode 100644 index 0000000..52ceaa1 --- /dev/null +++ b/protocol/src/windows_media.rs @@ -0,0 +1,1113 @@ +use core::fmt; +use std::collections::{HashMap, HashSet}; +use std::time::Duration; + +pub const MAX_VIDEO_FRAGMENT_PAYLOAD: usize = 1_200; +pub const MAX_VIDEO_FRAME_FRAGMENTS: u16 = 4_096; +pub const MAX_VIDEO_FRAME_BYTES: usize = 16 * 1024 * 1024; +pub const VIDEO_DATAGRAM_HEADER_BYTES: usize = 58; +pub const VIDEO_DATAGRAM_MAX_BYTES: usize = + VIDEO_DATAGRAM_HEADER_BYTES + MAX_VIDEO_FRAGMENT_PAYLOAD; +pub const MAX_AUDIO_PACKET_BYTES: usize = 4 * 1024; +pub const AUDIO_DATAGRAM_HEADER_BYTES: usize = 40; +pub const AUDIO_DATAGRAM_MAX_BYTES: usize = AUDIO_DATAGRAM_HEADER_BYTES + MAX_AUDIO_PACKET_BYTES; +const VIDEO_DATAGRAM_MAGIC: &[u8; 4] = b"RDV1"; +const VIDEO_DATAGRAM_VERSION: u8 = 1; +const AUDIO_DATAGRAM_MAGIC: &[u8; 4] = b"RDA1"; +const AUDIO_DATAGRAM_VERSION: u8 = 1; +const VIDEO_REASSEMBLY_MAX_FRAMES: usize = 8; +const VIDEO_REASSEMBLY_TIMEOUT: Duration = Duration::from_millis(100); + +/// Windows Headless currently exposes SDR 8-bit video only. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum WindowsCaptureBackend { + IddGpuFrame, + DesktopDuplication, + Warp, + Cpu, +} + +impl WindowsCaptureBackend { + #[must_use] + pub const fn is_software(self) -> bool { + matches!(self, Self::Warp | Self::Cpu) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum VideoCodec { + Avc, + Hevc, + Av1, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum VideoProfile { + High, + Main, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum EncoderBackend { + Nvenc, + Amf, + QuickSync, + X264, + X265, + SvtAv1, +} + +impl EncoderBackend { + #[must_use] + pub const fn is_hardware(self) -> bool { + matches!(self, Self::Nvenc | Self::Amf | Self::QuickSync) + } + + #[must_use] + pub const fn supports(self, codec: VideoCodec) -> bool { + matches!( + (self, codec), + (Self::Nvenc | Self::Amf | Self::QuickSync, _) + | (Self::X264, VideoCodec::Avc) + | (Self::X265, VideoCodec::Hevc) + | (Self::SvtAv1, VideoCodec::Av1) + ) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum VideoPixelFormat { + Nv12, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum VideoModePolicy { + StrictHardware, + Compatibility, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct VideoStreamConfig { + pub stream_id: u32, + pub generation: u64, + pub codec: VideoCodec, + pub profile: VideoProfile, + pub pixel_format: VideoPixelFormat, + pub bit_depth: u8, + pub width: u32, + pub height: u32, + pub fps_num: u32, + pub fps_den: u32, + pub encoder_backend: EncoderBackend, + pub capture_backend: WindowsCaptureBackend, + pub adapter_luid: Option, + pub hardware_path_verified: bool, + pub degraded: bool, +} + +impl VideoStreamConfig { + /// Validates the current Windows Headless SDR contract. + /// + /// # Errors + /// + /// Returns [`WindowsMediaConfigError`] when any stream field violates the + /// supported SDR format, codec/backend pairing, or hardware-path contract. + pub fn validate(&self) -> Result<(), WindowsMediaConfigError> { + if self.stream_id == 0 || self.generation == 0 { + return Err(WindowsMediaConfigError::InvalidIdentity); + } + if self.width == 0 || self.height == 0 || self.width > 16_384 || self.height > 16_384 { + return Err(WindowsMediaConfigError::InvalidDimensions); + } + if self.fps_num == 0 || self.fps_den == 0 { + return Err(WindowsMediaConfigError::InvalidFrameRate); + } + if self.bit_depth != 8 || self.pixel_format != VideoPixelFormat::Nv12 { + return Err(WindowsMediaConfigError::HdrOrUnsupportedPixelFormat); + } + if !self.encoder_backend.supports(self.codec) { + return Err(WindowsMediaConfigError::CodecBackendMismatch); + } + if matches!(self.codec, VideoCodec::Avc) && self.profile != VideoProfile::High { + return Err(WindowsMediaConfigError::InvalidProfile); + } + if matches!(self.codec, VideoCodec::Hevc | VideoCodec::Av1) + && self.profile != VideoProfile::Main + { + return Err(WindowsMediaConfigError::InvalidProfile); + } + if self.hardware_path_verified != self.encoder_backend.is_hardware() { + return Err(WindowsMediaConfigError::HardwareVerificationMismatch); + } + if self.degraded == self.hardware_path_verified { + return Err(WindowsMediaConfigError::DegradedFlagMismatch); + } + if self.hardware_path_verified && self.adapter_luid.is_none() { + return Err(WindowsMediaConfigError::MissingAdapterIdentity); + } + if self.hardware_path_verified && self.capture_backend.is_software() { + return Err(WindowsMediaConfigError::HardwareCaptureMismatch); + } + Ok(()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AudioStreamConfig { + pub stream_id: u32, + pub generation: u64, + pub sample_rate: u32, + pub channels: u8, + pub frame_duration_ms: u8, +} + +impl AudioStreamConfig { + /// Validates the current Opus-compatible audio stream contract. + /// + /// # Errors + /// + /// Returns [`WindowsMediaConfigError`] when the stream identity, sample + /// rate, channel count, or packet duration is unsupported. + pub fn validate(&self) -> Result<(), WindowsMediaConfigError> { + if self.stream_id == 0 || self.generation == 0 { + return Err(WindowsMediaConfigError::InvalidIdentity); + } + if self.sample_rate != 48_000 || !(1..=8).contains(&self.channels) { + return Err(WindowsMediaConfigError::InvalidAudioFormat); + } + if !matches!(self.frame_duration_ms, 10 | 20 | 40 | 60) { + return Err(WindowsMediaConfigError::InvalidAudioFormat); + } + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct AudioPacketFlags(u16); + +impl AudioPacketFlags { + pub const DISCONTINUITY: Self = Self(1 << 0); + + #[must_use] + pub const fn bits(self) -> u16 { + self.0 + } + + #[must_use] + pub const fn contains(self, other: Self) -> bool { + self.0 & other.0 == other.0 + } + + #[must_use] + pub const fn from_bits(bits: u16) -> Option { + if bits & !Self::DISCONTINUITY.0 == 0 { + Some(Self(bits)) + } else { + None + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AudioPacket { + pub stream_id: u32, + pub generation: u64, + pub sequence: u64, + pub pts: u64, + pub duration_ms: u16, + pub flags: AudioPacketFlags, + pub payload: Vec, +} + +impl AudioPacket { + /// Validates one independently decodable Opus packet. + /// + /// # Errors + /// + /// Returns [`WindowsMediaPacketError`] when identity, duration, or payload + /// bounds violate the Windows audio transport contract. + pub fn validate(&self) -> Result<(), WindowsMediaPacketError> { + if self.stream_id == 0 || self.generation == 0 || self.sequence == 0 { + return Err(WindowsMediaPacketError::InvalidIdentity); + } + if !matches!(self.duration_ms, 10 | 20 | 40 | 60) { + return Err(WindowsMediaPacketError::InvalidAudioDuration); + } + if self.payload.is_empty() || self.payload.len() > MAX_AUDIO_PACKET_BYTES { + return Err(WindowsMediaPacketError::InvalidPayloadSize); + } + Ok(()) + } +} + +/// Decodes and validates one RDA1 Opus datagram. +/// +/// # Errors +/// +/// Returns [`WindowsMediaDatagramError`] for malformed headers, unknown flags, +/// unsupported durations, invalid identities, or payloads outside wire bounds. +pub fn decode_audio_datagram(input: &[u8]) -> Result { + if input.len() < AUDIO_DATAGRAM_HEADER_BYTES || input.len() > AUDIO_DATAGRAM_MAX_BYTES { + return Err(WindowsMediaDatagramError::InvalidSize); + } + if &input[..4] != AUDIO_DATAGRAM_MAGIC + || input[4] != AUDIO_DATAGRAM_VERSION + || input[5] != 0 + || read_u16(input, 6) != Some(40) + { + return Err(WindowsMediaDatagramError::InvalidHeader); + } + let flags = AudioPacketFlags::from_bits( + read_u16(input, 38).ok_or(WindowsMediaDatagramError::InvalidHeader)?, + ) + .ok_or(WindowsMediaDatagramError::InvalidFlags)?; + let packet = AudioPacket { + stream_id: read_u32(input, 8).ok_or(WindowsMediaDatagramError::InvalidHeader)?, + generation: read_u64(input, 12).ok_or(WindowsMediaDatagramError::InvalidHeader)?, + sequence: read_u64(input, 20).ok_or(WindowsMediaDatagramError::InvalidHeader)?, + pts: read_u64(input, 28).ok_or(WindowsMediaDatagramError::InvalidHeader)?, + duration_ms: read_u16(input, 36).ok_or(WindowsMediaDatagramError::InvalidHeader)?, + flags, + payload: input[AUDIO_DATAGRAM_HEADER_BYTES..].to_vec(), + }; + packet + .validate() + .map_err(WindowsMediaDatagramError::InvalidPacket)?; + Ok(packet) +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct EncoderCapability { + pub backend: EncoderBackend, + pub codec: VideoCodec, + pub profile: VideoProfile, + pub adapter_luid: Option, + pub max_width: u32, + pub max_height: u32, + pub max_fps: u32, + pub concurrent_sessions: u16, + pub hardware_path_verified: bool, +} + +impl EncoderCapability { + #[must_use] + pub const fn supports_request(&self, width: u32, height: u32, fps: u32) -> bool { + self.max_width >= width + && self.max_height >= height + && self.max_fps >= fps + && self.concurrent_sessions > 0 + && self.hardware_path_verified == self.backend.is_hardware() + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct VideoRequest { + pub codec: VideoCodec, + pub width: u32, + pub height: u32, + pub fps: u32, + pub policy: VideoModePolicy, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SelectedEncoder { + pub capability: EncoderCapability, + pub degraded: bool, +} + +/// Selects a verified hardware encoder first, then a software encoder only in +/// explicit compatibility mode. The caller must still run a real short-sample +/// probe before publishing the capability. +/// +/// # Errors +/// +/// Returns [`WindowsMediaSelectionError`] when no verified hardware encoder is +/// available under strict policy, or when compatibility policy also has no +/// suitable software encoder. +pub fn select_encoder( + request: &VideoRequest, + candidates: &[EncoderCapability], +) -> Result { + let hardware = candidates.iter().find(|candidate| { + candidate.backend.is_hardware() + && candidate.codec == request.codec + && candidate.supports_request(request.width, request.height, request.fps) + }); + if let Some(capability) = hardware { + return Ok(SelectedEncoder { + capability: capability.clone(), + degraded: false, + }); + } + + if request.policy == VideoModePolicy::StrictHardware { + return Err(WindowsMediaSelectionError::HardwareEncoderUnavailable { + codec: request.codec, + }); + } + + let software = candidates.iter().find(|candidate| { + !candidate.backend.is_hardware() + && candidate.codec == request.codec + && candidate.supports_request(request.width, request.height, request.fps) + }); + software + .cloned() + .map(|capability| SelectedEncoder { + capability, + degraded: true, + }) + .ok_or(WindowsMediaSelectionError::SoftwareEncoderUnavailable { + codec: request.codec, + }) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct MediaPacketFlags(u16); + +impl MediaPacketFlags { + pub const CONFIG: Self = Self(1 << 0); + pub const KEY_FRAME: Self = Self(1 << 1); + pub const FIRST_FRAGMENT: Self = Self(1 << 2); + pub const LAST_FRAGMENT: Self = Self(1 << 3); + pub const DISCONTINUITY: Self = Self(1 << 4); + + #[must_use] + pub const fn bits(self) -> u16 { + self.0 + } + + #[must_use] + pub const fn contains(self, other: Self) -> bool { + self.0 & other.0 == other.0 + } + + #[must_use] + pub const fn union(self, other: Self) -> Self { + Self(self.0 | other.0) + } + + #[must_use] + pub const fn from_bits(bits: u16) -> Option { + const KNOWN: u16 = MediaPacketFlags::CONFIG.0 + | MediaPacketFlags::KEY_FRAME.0 + | MediaPacketFlags::FIRST_FRAGMENT.0 + | MediaPacketFlags::LAST_FRAGMENT.0 + | MediaPacketFlags::DISCONTINUITY.0; + if bits & !KNOWN == 0 { + Some(Self(bits)) + } else { + None + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct VideoPacket { + pub stream_id: u32, + pub generation: u64, + pub sequence: u64, + pub frame_id: u64, + pub pts: u64, + pub dts: u64, + pub fragment_id: u16, + pub fragment_count: u16, + pub flags: MediaPacketFlags, + pub payload: Vec, +} + +impl VideoPacket { + /// Validates packet identity, bounds, and fragment-position flags. + /// + /// # Errors + /// + /// Returns [`WindowsMediaPacketError`] when the packet cannot be accepted + /// by the bounded video reassembler. + pub fn validate(&self) -> Result<(), WindowsMediaPacketError> { + if self.stream_id == 0 || self.generation == 0 { + return Err(WindowsMediaPacketError::InvalidIdentity); + } + if self.fragment_count == 0 || self.fragment_count > MAX_VIDEO_FRAME_FRAGMENTS { + return Err(WindowsMediaPacketError::InvalidFragmentCount); + } + if self.fragment_id >= self.fragment_count { + return Err(WindowsMediaPacketError::InvalidFragmentId); + } + if self.payload.is_empty() || self.payload.len() > MAX_VIDEO_FRAGMENT_PAYLOAD { + return Err(WindowsMediaPacketError::InvalidPayloadSize); + } + if self.flags.contains(MediaPacketFlags::FIRST_FRAGMENT) != (self.fragment_id == 0) { + return Err(WindowsMediaPacketError::FirstFlagMismatch); + } + if self.flags.contains(MediaPacketFlags::LAST_FRAGMENT) + != (self.fragment_id + 1 == self.fragment_count) + { + return Err(WindowsMediaPacketError::LastFlagMismatch); + } + Ok(()) + } +} + +/// Decodes and validates one RDV1 video datagram. +/// +/// # Errors +/// +/// Returns [`WindowsMediaDatagramError`] for malformed headers, unknown flags, +/// invalid packet fields, or datagrams outside the wire-size bounds. +pub fn decode_video_datagram(input: &[u8]) -> Result { + if input.len() < VIDEO_DATAGRAM_HEADER_BYTES || input.len() > VIDEO_DATAGRAM_MAX_BYTES { + return Err(WindowsMediaDatagramError::InvalidSize); + } + if &input[..4] != VIDEO_DATAGRAM_MAGIC + || input[4] != VIDEO_DATAGRAM_VERSION + || input[5] != 0 + || read_u16(input, 6) != Some(58) + { + return Err(WindowsMediaDatagramError::InvalidHeader); + } + let flags = MediaPacketFlags::from_bits( + read_u16(input, 56).ok_or(WindowsMediaDatagramError::InvalidHeader)?, + ) + .ok_or(WindowsMediaDatagramError::InvalidFlags)?; + let packet = VideoPacket { + stream_id: read_u32(input, 8).ok_or(WindowsMediaDatagramError::InvalidHeader)?, + generation: read_u64(input, 12).ok_or(WindowsMediaDatagramError::InvalidHeader)?, + sequence: read_u64(input, 20).ok_or(WindowsMediaDatagramError::InvalidHeader)?, + frame_id: read_u64(input, 28).ok_or(WindowsMediaDatagramError::InvalidHeader)?, + pts: read_u64(input, 36).ok_or(WindowsMediaDatagramError::InvalidHeader)?, + dts: read_u64(input, 44).ok_or(WindowsMediaDatagramError::InvalidHeader)?, + fragment_id: read_u16(input, 52).ok_or(WindowsMediaDatagramError::InvalidHeader)?, + fragment_count: read_u16(input, 54).ok_or(WindowsMediaDatagramError::InvalidHeader)?, + flags, + payload: input[VIDEO_DATAGRAM_HEADER_BYTES..].to_vec(), + }; + packet + .validate() + .map_err(WindowsMediaDatagramError::InvalidPacket)?; + if packet.sequence == 0 || packet.frame_id == 0 { + return Err(WindowsMediaDatagramError::InvalidIdentity); + } + Ok(packet) +} + +fn read_u16(input: &[u8], offset: usize) -> Option { + let bytes: [u8; 2] = input.get(offset..offset.checked_add(2)?)?.try_into().ok()?; + Some(u16::from_le_bytes(bytes)) +} + +fn read_u32(input: &[u8], offset: usize) -> Option { + let bytes: [u8; 4] = input.get(offset..offset.checked_add(4)?)?.try_into().ok()?; + Some(u32::from_le_bytes(bytes)) +} + +fn read_u64(input: &[u8], offset: usize) -> Option { + let bytes: [u8; 8] = input.get(offset..offset.checked_add(8)?)?.try_into().ok()?; + Some(u64::from_le_bytes(bytes)) +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct VideoAccessUnit { + pub stream_id: u32, + pub generation: u64, + pub sequence: u64, + pub frame_id: u64, + pub pts: u64, + pub dts: u64, + pub flags: MediaPacketFlags, + pub payload: Vec, +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +struct StreamKey { + stream_id: u32, + generation: u64, +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +struct FrameKey { + stream: StreamKey, + frame_id: u64, +} + +struct FrameAssembly { + created_at: Duration, + sequence: u64, + pts: u64, + dts: u64, + flags: MediaPacketFlags, + fragment_count: u16, + fragments: Vec>>, + bytes: usize, +} + +pub struct VideoReassembler { + frames: HashMap, + waiting_for_keyframe: HashSet, + last_delivered_sequence: HashMap, +} + +impl Default for VideoReassembler { + fn default() -> Self { + Self::new() + } +} + +impl VideoReassembler { + #[must_use] + pub fn new() -> Self { + Self { + frames: HashMap::new(), + waiting_for_keyframe: HashSet::new(), + last_delivered_sequence: HashMap::new(), + } + } + + /// Decodes and submits one RDV1 datagram to the bounded reassembler. + /// + /// # Errors + /// + /// Returns [`WindowsMediaDatagramError`] when decoding fails or the frame + /// violates reassembly bounds and consistency requirements. + pub fn push_datagram( + &mut self, + input: &[u8], + now: Duration, + ) -> Result, WindowsMediaDatagramError> { + let packet = decode_video_datagram(input)?; + self.push(packet, now) + } + + /// Submits one validated video fragment to the bounded reassembler. + /// + /// # Errors + /// + /// Returns [`WindowsMediaDatagramError`] when packet validation fails, + /// fragments disagree, or the assembled frame exceeds configured bounds. + pub fn push( + &mut self, + packet: VideoPacket, + now: Duration, + ) -> Result, WindowsMediaDatagramError> { + packet + .validate() + .map_err(WindowsMediaDatagramError::InvalidPacket)?; + self.expire(now); + let stream = StreamKey { + stream_id: packet.stream_id, + generation: packet.generation, + }; + let key = FrameKey { + stream, + frame_id: packet.frame_id, + }; + if !self.frames.contains_key(&key) && self.frames.len() >= VIDEO_REASSEMBLY_MAX_FRAMES { + self.drop_oldest(); + } + let stable_flags = MediaPacketFlags( + packet.flags.bits() + & !(MediaPacketFlags::FIRST_FRAGMENT.bits() + | MediaPacketFlags::LAST_FRAGMENT.bits()), + ); + let assembly = self.frames.entry(key).or_insert_with(|| FrameAssembly { + created_at: now, + sequence: packet.sequence, + pts: packet.pts, + dts: packet.dts, + flags: stable_flags, + fragment_count: packet.fragment_count, + fragments: vec![None; usize::from(packet.fragment_count)], + bytes: 0, + }); + if assembly.sequence != packet.sequence + || assembly.pts != packet.pts + || assembly.dts != packet.dts + || assembly.flags != stable_flags + || assembly.fragment_count != packet.fragment_count + { + self.frames.remove(&key); + self.waiting_for_keyframe.insert(stream); + return Err(WindowsMediaDatagramError::MetadataChanged); + } + let fragment = &mut assembly.fragments[usize::from(packet.fragment_id)]; + if let Some(existing) = fragment { + if *existing != packet.payload { + self.frames.remove(&key); + self.waiting_for_keyframe.insert(stream); + return Err(WindowsMediaDatagramError::DuplicateChanged); + } + return Ok(None); + } + assembly.bytes = assembly + .bytes + .checked_add(packet.payload.len()) + .ok_or(WindowsMediaDatagramError::FrameTooLarge)?; + if assembly.bytes > MAX_VIDEO_FRAME_BYTES { + self.frames.remove(&key); + self.waiting_for_keyframe.insert(stream); + return Err(WindowsMediaDatagramError::FrameTooLarge); + } + *fragment = Some(packet.payload); + if assembly.fragments.iter().any(Option::is_none) { + return Ok(None); + } + let Some(assembly) = self.frames.remove(&key) else { + self.waiting_for_keyframe.insert(stream); + return Err(WindowsMediaDatagramError::IncompleteFrame); + }; + let mut payload = Vec::with_capacity(assembly.bytes); + for fragment in assembly.fragments { + let Some(fragment) = fragment else { + self.waiting_for_keyframe.insert(stream); + return Err(WindowsMediaDatagramError::IncompleteFrame); + }; + payload.extend(fragment); + } + let previous = self.last_delivered_sequence.get(&stream).copied(); + let sequence_gap = previous.is_some_and(|last| packet.sequence != last.saturating_add(1)); + if previous.is_none() + || sequence_gap + || assembly.flags.contains(MediaPacketFlags::DISCONTINUITY) + { + self.waiting_for_keyframe.insert(stream); + } + let keyframe = assembly.flags.contains(MediaPacketFlags::KEY_FRAME); + if self.waiting_for_keyframe.contains(&stream) && !keyframe { + return Ok(None); + } + if keyframe { + self.waiting_for_keyframe.remove(&stream); + } + self.last_delivered_sequence.insert(stream, packet.sequence); + Ok(Some(VideoAccessUnit { + stream_id: stream.stream_id, + generation: stream.generation, + sequence: packet.sequence, + frame_id: packet.frame_id, + pts: packet.pts, + dts: packet.dts, + flags: assembly.flags, + payload, + })) + } + + #[must_use] + pub fn keyframe_required(&self) -> bool { + !self.waiting_for_keyframe.is_empty() + } + + fn expire(&mut self, now: Duration) { + let expired = self + .frames + .iter() + .filter_map(|(key, assembly)| { + (now.saturating_sub(assembly.created_at) >= VIDEO_REASSEMBLY_TIMEOUT) + .then_some(*key) + }) + .collect::>(); + for key in expired { + self.frames.remove(&key); + self.waiting_for_keyframe.insert(key.stream); + } + } + + fn drop_oldest(&mut self) { + let oldest = self + .frames + .iter() + .min_by_key(|(_, assembly)| assembly.created_at) + .map(|(key, _)| *key); + if let Some(key) = oldest { + self.frames.remove(&key); + self.waiting_for_keyframe.insert(key.stream); + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum WindowsMediaConfigError { + InvalidIdentity, + InvalidDimensions, + InvalidFrameRate, + HdrOrUnsupportedPixelFormat, + CodecBackendMismatch, + InvalidProfile, + HardwareVerificationMismatch, + DegradedFlagMismatch, + MissingAdapterIdentity, + HardwareCaptureMismatch, + InvalidAudioFormat, +} + +impl fmt::Display for WindowsMediaConfigError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::InvalidIdentity => "media identity is invalid", + Self::InvalidDimensions => "media dimensions are invalid", + Self::InvalidFrameRate => "media frame rate is invalid", + Self::HdrOrUnsupportedPixelFormat => "only 8-bit NV12 SDR is supported", + Self::CodecBackendMismatch => "codec and encoder backend do not match", + Self::InvalidProfile => "codec profile is invalid", + Self::HardwareVerificationMismatch => "hardware verification does not match backend", + Self::DegradedFlagMismatch => "degraded flag does not match hardware path", + Self::MissingAdapterIdentity => "hardware path is missing adapter identity", + Self::HardwareCaptureMismatch => "hardware encoder is paired with software capture", + Self::InvalidAudioFormat => "audio format is invalid", + }) + } +} + +impl std::error::Error for WindowsMediaConfigError {} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum WindowsMediaSelectionError { + HardwareEncoderUnavailable { codec: VideoCodec }, + SoftwareEncoderUnavailable { codec: VideoCodec }, +} + +impl fmt::Display for WindowsMediaSelectionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::HardwareEncoderUnavailable { codec } + | Self::SoftwareEncoderUnavailable { codec } => { + write!(formatter, "no verified encoder available for {codec:?}") + } + } + } +} + +impl std::error::Error for WindowsMediaSelectionError {} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum WindowsMediaPacketError { + InvalidIdentity, + InvalidFragmentCount, + InvalidFragmentId, + InvalidPayloadSize, + FirstFlagMismatch, + LastFlagMismatch, + InvalidAudioDuration, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum WindowsMediaDatagramError { + InvalidSize, + InvalidHeader, + InvalidFlags, + InvalidIdentity, + InvalidPacket(WindowsMediaPacketError), + MetadataChanged, + DuplicateChanged, + FrameTooLarge, + IncompleteFrame, +} + +impl fmt::Display for WindowsMediaDatagramError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::InvalidSize => "media datagram size is invalid", + Self::InvalidHeader => "media datagram header is invalid", + Self::InvalidFlags => "media datagram flags are invalid", + Self::InvalidIdentity => "media datagram identity is invalid", + Self::InvalidPacket(_) => "media datagram packet is invalid", + Self::MetadataChanged => "video frame metadata changed during reassembly", + Self::DuplicateChanged => "video frame duplicate fragment changed", + Self::FrameTooLarge => "video frame exceeds its bounded size", + Self::IncompleteFrame => "video frame became incomplete during reassembly", + }) + } +} + +impl std::error::Error for WindowsMediaDatagramError {} + +impl fmt::Display for WindowsMediaPacketError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::InvalidIdentity => "media packet identity is invalid", + Self::InvalidFragmentCount => "media packet fragment count is invalid", + Self::InvalidFragmentId => "media packet fragment id is invalid", + Self::InvalidPayloadSize => "media packet payload size is invalid", + Self::FirstFlagMismatch => "media packet first flag is inconsistent", + Self::LastFlagMismatch => "media packet last flag is inconsistent", + Self::InvalidAudioDuration => "audio packet duration is invalid", + }) + } +} + +impl std::error::Error for WindowsMediaPacketError {} + +#[cfg(test)] +mod tests { + use super::*; + + fn datagram( + sequence: u64, + frame_id: u64, + fragment_id: u16, + fragment_count: u16, + flags: MediaPacketFlags, + payload: &[u8], + ) -> Vec { + let mut output = vec![0_u8; VIDEO_DATAGRAM_HEADER_BYTES]; + output[..4].copy_from_slice(VIDEO_DATAGRAM_MAGIC); + output[4] = VIDEO_DATAGRAM_VERSION; + output[6..8].copy_from_slice(&(VIDEO_DATAGRAM_HEADER_BYTES as u16).to_le_bytes()); + output[8..12].copy_from_slice(&1_u32.to_le_bytes()); + output[12..20].copy_from_slice(&2_u64.to_le_bytes()); + output[20..28].copy_from_slice(&sequence.to_le_bytes()); + output[28..36].copy_from_slice(&frame_id.to_le_bytes()); + output[36..44].copy_from_slice(&5_u64.to_le_bytes()); + output[44..52].copy_from_slice(&5_u64.to_le_bytes()); + output[52..54].copy_from_slice(&fragment_id.to_le_bytes()); + output[54..56].copy_from_slice(&fragment_count.to_le_bytes()); + output[56..58].copy_from_slice(&flags.bits().to_le_bytes()); + output.extend_from_slice(payload); + output + } + + fn audio_datagram(flags: AudioPacketFlags, duration_ms: u16, payload: &[u8]) -> Vec { + let mut output = vec![0_u8; AUDIO_DATAGRAM_HEADER_BYTES]; + output[..4].copy_from_slice(AUDIO_DATAGRAM_MAGIC); + output[4] = AUDIO_DATAGRAM_VERSION; + output[6..8].copy_from_slice(&40_u16.to_le_bytes()); + output[8..12].copy_from_slice(&3_u32.to_le_bytes()); + output[12..20].copy_from_slice(&4_u64.to_le_bytes()); + output[20..28].copy_from_slice(&5_u64.to_le_bytes()); + output[28..36].copy_from_slice(&6_u64.to_le_bytes()); + output[36..38].copy_from_slice(&duration_ms.to_le_bytes()); + output[38..40].copy_from_slice(&flags.bits().to_le_bytes()); + output.extend_from_slice(payload); + output + } + + #[test] + fn rda1_wire_decoder_matches_go_layout() { + let input = audio_datagram(AudioPacketFlags::DISCONTINUITY, 20, b"opus"); + let packet = decode_audio_datagram(&input).unwrap(); + assert_eq!(packet.stream_id, 3); + assert_eq!(packet.generation, 4); + assert_eq!(packet.sequence, 5); + assert_eq!(packet.pts, 6); + assert_eq!(packet.duration_ms, 20); + assert!(packet.flags.contains(AudioPacketFlags::DISCONTINUITY)); + assert_eq!(packet.payload, b"opus"); + } + + #[test] + fn rda1_rejects_reserved_unknown_flags_and_duration() { + let mut reserved = audio_datagram(AudioPacketFlags(0), 20, b"opus"); + reserved[5] = 1; + assert_eq!( + decode_audio_datagram(&reserved), + Err(WindowsMediaDatagramError::InvalidHeader) + ); + let mut unknown_flags = audio_datagram(AudioPacketFlags(0), 20, b"opus"); + unknown_flags[38..40].copy_from_slice(&2_u16.to_le_bytes()); + assert_eq!( + decode_audio_datagram(&unknown_flags), + Err(WindowsMediaDatagramError::InvalidFlags) + ); + assert!(decode_audio_datagram(&audio_datagram(AudioPacketFlags(0), 30, b"opus")).is_err()); + } + + fn hardware(codec: VideoCodec, backend: EncoderBackend) -> EncoderCapability { + EncoderCapability { + backend, + codec, + profile: if codec == VideoCodec::Avc { + VideoProfile::High + } else { + VideoProfile::Main + }, + adapter_luid: Some(7), + max_width: 3_840, + max_height: 2_160, + max_fps: 120, + concurrent_sessions: 2, + hardware_path_verified: true, + } + } + + fn software(codec: VideoCodec, backend: EncoderBackend) -> EncoderCapability { + EncoderCapability { + adapter_luid: None, + hardware_path_verified: false, + ..hardware(codec, backend) + } + } + + #[test] + fn strict_selection_rejects_missing_hardware() { + let request = VideoRequest { + codec: VideoCodec::Av1, + width: 3_840, + height: 2_160, + fps: 120, + policy: VideoModePolicy::StrictHardware, + }; + assert_eq!( + select_encoder( + &request, + &[software(VideoCodec::Av1, EncoderBackend::SvtAv1)] + ), + Err(WindowsMediaSelectionError::HardwareEncoderUnavailable { + codec: VideoCodec::Av1 + }) + ); + } + + #[test] + fn compatibility_selects_software_and_marks_degraded() { + let request = VideoRequest { + codec: VideoCodec::Avc, + width: 1_920, + height: 1_080, + fps: 60, + policy: VideoModePolicy::Compatibility, + }; + let selected = + select_encoder(&request, &[software(VideoCodec::Avc, EncoderBackend::X264)]).unwrap(); + assert!(selected.degraded); + assert_eq!(selected.capability.backend, EncoderBackend::X264); + } + + #[test] + fn packet_flags_must_match_fragment_position() { + let packet = VideoPacket { + stream_id: 1, + generation: 1, + sequence: 1, + frame_id: 1, + pts: 0, + dts: 0, + fragment_id: 1, + fragment_count: 2, + flags: MediaPacketFlags::FIRST_FRAGMENT, + payload: vec![1], + }; + assert_eq!( + packet.validate(), + Err(WindowsMediaPacketError::FirstFlagMismatch) + ); + } + + #[test] + fn stream_config_rejects_main10_like_configuration() { + let config = VideoStreamConfig { + stream_id: 1, + generation: 1, + codec: VideoCodec::Hevc, + profile: VideoProfile::Main, + pixel_format: VideoPixelFormat::Nv12, + bit_depth: 10, + width: 1_920, + height: 1_080, + fps_num: 60, + fps_den: 1, + encoder_backend: EncoderBackend::Nvenc, + capture_backend: WindowsCaptureBackend::IddGpuFrame, + adapter_luid: Some(1), + hardware_path_verified: true, + degraded: false, + }; + assert_eq!( + config.validate(), + Err(WindowsMediaConfigError::HdrOrUnsupportedPixelFormat) + ); + } + + #[test] + fn rdv1_wire_decoder_matches_go_layout() { + let packet = decode_video_datagram(&datagram( + 3, + 4, + 0, + 1, + MediaPacketFlags::KEY_FRAME + .union(MediaPacketFlags::FIRST_FRAGMENT) + .union(MediaPacketFlags::LAST_FRAGMENT), + &[0, 0, 0, 1, 5], + )) + .unwrap(); + assert_eq!(packet.stream_id, 1); + assert_eq!(packet.generation, 2); + assert_eq!(packet.sequence, 3); + assert_eq!(packet.frame_id, 4); + assert!(packet.flags.contains(MediaPacketFlags::KEY_FRAME)); + assert_eq!(packet.payload, [0, 0, 0, 1, 5]); + } + + #[test] + fn reassembler_requires_keyframe_then_preserves_fragment_order() { + let mut reassembler = VideoReassembler::new(); + let first = MediaPacketFlags::KEY_FRAME.union(MediaPacketFlags::FIRST_FRAGMENT); + let last = MediaPacketFlags::KEY_FRAME.union(MediaPacketFlags::LAST_FRAGMENT); + assert!( + reassembler + .push_datagram(&datagram(1, 1, 1, 2, last, b"second"), Duration::ZERO) + .unwrap() + .is_none() + ); + let unit = reassembler + .push_datagram(&datagram(1, 1, 0, 2, first, b"first-"), Duration::ZERO) + .unwrap() + .unwrap(); + assert_eq!(unit.payload, b"first-second"); + assert!(unit.flags.contains(MediaPacketFlags::KEY_FRAME)); + } + + #[test] + fn sequence_gap_drops_inter_frames_until_next_keyframe() { + let mut reassembler = VideoReassembler::new(); + let boundaries = MediaPacketFlags::FIRST_FRAGMENT.union(MediaPacketFlags::LAST_FRAGMENT); + let key = boundaries.union(MediaPacketFlags::KEY_FRAME); + assert!( + reassembler + .push_datagram(&datagram(1, 1, 0, 1, key, b"idr"), Duration::ZERO) + .unwrap() + .is_some() + ); + assert!( + reassembler + .push_datagram( + &datagram(3, 3, 0, 1, boundaries, b"p"), + Duration::from_millis(1) + ) + .unwrap() + .is_none() + ); + assert!( + reassembler + .push_datagram( + &datagram(4, 4, 0, 1, key, b"idr2"), + Duration::from_millis(2) + ) + .unwrap() + .is_some() + ); + } + + #[test] + fn invalid_reserved_byte_and_unknown_flag_are_rejected() { + let mut reserved = datagram( + 1, + 1, + 0, + 1, + MediaPacketFlags::KEY_FRAME + .union(MediaPacketFlags::FIRST_FRAGMENT) + .union(MediaPacketFlags::LAST_FRAGMENT), + &[1], + ); + reserved[5] = 1; + assert_eq!( + decode_video_datagram(&reserved), + Err(WindowsMediaDatagramError::InvalidHeader) + ); + reserved[5] = 0; + reserved[56..58].copy_from_slice(&0x8000_u16.to_le_bytes()); + assert_eq!( + decode_video_datagram(&reserved), + Err(WindowsMediaDatagramError::InvalidFlags) + ); + } +} diff --git a/transport/hysteria2-agent/README.md b/transport/hysteria2-agent/README.md index df2a24a..a98e9f5 100644 --- a/transport/hysteria2-agent/README.md +++ b/transport/hysteria2-agent/README.md @@ -30,3 +30,93 @@ UDP session: `--insecure` is intentionally opt-in and should only be used for a local test certificate. No TCP listener, TCP proxy, or SOCKS5 endpoint is created. + +## Rust encoded ring bridge + +The Windows Agent compatibility encoder can publish H.264 Annex-B access units +to its session-bound memory-mapped ring. The Hysteria2 client can consume that +ring directly and send RDV1 video datagrams on the same authenticated UDP +session: + +```text +remotedesk-hysteria2-agent --mode client \ + --listen 127.0.0.1:39503 \ + --server relay.example:39502 --password 'change-me' \ + --server-name relay.example \ + --agent-udp 127.0.0.1:39502 \ + --encoded-ring C:\Users\user\AppData\Local\Temp\remotedesk-session-...ring \ + --ring-stream-id 1 --ring-generation 7 +``` + +`--encoded-ring` is optional and is intended for the authenticated session +orchestrator, which obtains the path, stream ID, and generation from the Rust +control response. The Go consumer validates the ring header and geometry, +copies each encoded access unit before releasing the slot, fragments it into +RDV1, and drops video on cancellation or transport failure. It never maps or +transmits raw BGRA/NV12 surfaces. + +For production local orchestration, start the Agent and bridge with the same +random 32-byte base64url bootstrap. The Agent removes its inherited copy from +the environment after startup, restricts the pipe and ring DACL to the current +Windows user, rejects remote pipe clients, and authenticates each connection +with a nonce-bound HMAC-SHA256 challenge: + +```powershell +$token = New-Object byte[] 32 +[Security.Cryptography.RandomNumberGenerator]::Fill($token) +$env:REMOTEDESK_AGENT_PIPE_TOKEN = [Convert]::ToBase64String($token).TrimEnd('=').Replace('+','-').Replace('/','_') +Start-Process -WindowStyle Hidden remotedesk-windows-agent -ArgumentList 'pipe' + +remotedesk-hysteria2-agent --mode client ` + --server relay.example:39502 --password 'change-me' ` + --server-name relay.example --agent-udp 127.0.0.1:39502 ` + --windows-agent-pipe '\\.\pipe\RemoteDesk\windows-agent' ` + --session-id session-1 --ring-fps 120 --ring-stream-id 1 +Remove-Item Env:REMOTEDESK_AGENT_PIPE_TOKEN +``` + +`--windows-agent-control` remains available only for a same-machine development +bridge. Its address must be a literal loopback IP: + +```text +remotedesk-hysteria2-agent --mode client \ + --server relay.example:39502 --password 'change-me' \ + --server-name relay.example --agent-udp 127.0.0.1:39502 \ + --windows-agent-control 127.0.0.1:39501 \ + --session-id session-1 --ring-fps 120 --ring-stream-id 1 +``` + +The receiver-side Windows viewer binds only loopback UDP, pins the first valid +RDV1 source, uses bounded reassembly and a one-frame UI mailbox, and presents +H.264 through Media Foundation/D3D11: + +```text +remotedesk-windows-agent-viewer --target 10.0.0.20:39501 \ + --session-id session-1 --fps 120 --media-udp 127.0.0.1:39502 +``` + +RDWF remains the default when `--media-udp` is absent. The authenticated Named +Pipe is the production descriptor path; loopback TCP and manual `--encoded-ring` +are diagnostic/development paths and do not prove descriptor authenticity. + +## RemoteDesk media datagrams + +Packets beginning with RDV1 are RemoteDesk H.264/HEVC/AV1 video fragments. +The relay enforces the complete datagram limit (58-byte header plus at most +1200 bytes of payload) and otherwise forwards the bytes without decoding them. +Audio and control packets use their own framing and remain opaque to this +adapter. A missing video fragment is not retransmitted by this helper; the +receiver drops the incomplete access unit after its bounded reassembly timeout. + +The transport is an unreliable media path. Input and control traffic must use a +separate reliable path in the Agent session protocol; this UDP adapter does not +provide ordering or delivery guarantees for those messages. + +RemoteDesk Opus packets use RDA1: a 40-byte header followed by one bounded +Opus packet (maximum 4 KiB). Valid durations are 10/20/40/60 ms. The adapter +validates the framing and forwards the packet without reordering or waiting for +video. With an authenticated Windows Agent descriptor it independently drains +the Opus mmap ring and sends RDA1 datagrams. The Windows viewer uses a bounded +jitter queue plus Opus FEC/PLC and WASAPI playback; audio failure does not stop +video or input. Real-device validation and precise shared-QPC drift correction +remain release requirements. diff --git a/transport/hysteria2-agent/go.mod b/transport/hysteria2-agent/go.mod index 4f74981..62af2ef 100644 --- a/transport/hysteria2-agent/go.mod +++ b/transport/hysteria2-agent/go.mod @@ -2,7 +2,11 @@ module github.com/remotedesk/remotedesk/transport/hysteria2-agent go 1.25.0 -require github.com/apernet/hysteria/core/v2 v2.12.1 +require ( + github.com/Microsoft/go-winio v0.6.2 + github.com/apernet/hysteria/core/v2 v2.12.1 + golang.org/x/sys v0.47.0 +) require ( github.com/andybalholm/brotli v1.1.0 // indirect @@ -17,7 +21,6 @@ require ( golang.org/x/crypto v0.54.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/net v0.57.0 // indirect - golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.40.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/transport/hysteria2-agent/go.sum b/transport/hysteria2-agent/go.sum index 9075540..0107c83 100644 --- a/transport/hysteria2-agent/go.sum +++ b/transport/hysteria2-agent/go.sum @@ -1,3 +1,5 @@ +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/apernet/hysteria/core/v2 v2.12.1 h1:7jA/jSC+1tFP8Z9COzI0AdT1p9c0SY7D8vngRd1YX7s= diff --git a/transport/hysteria2-agent/main.go b/transport/hysteria2-agent/main.go index ec523c9..206bfb9 100644 --- a/transport/hysteria2-agent/main.go +++ b/transport/hysteria2-agent/main.go @@ -5,7 +5,9 @@ package main import ( + "context" "crypto/tls" + "errors" "flag" "fmt" "log" @@ -46,6 +48,46 @@ func (o localDatagramOutbound) UDP(_ string) (server.UDPConn, error) { type udpConn struct{ conn *net.UDPConn } +type datagramKind uint8 + +const ( + datagramOpaque datagramKind = iota + datagramVideo +) + +// classifyDatagram keeps the relay payload opaque. Only the RemoteDesk video +// framing has a local size contract; control and audio packets are forwarded +// without parsing so they can evolve independently. +func classifyDatagram(data []byte) (datagramKind, error) { + if len(data) >= len(videoDatagramMagic) && string(data[:len(videoDatagramMagic)]) == videoDatagramMagic { + if err := validateVideoDatagramSize(data); err != nil { + return datagramVideo, err + } + return datagramVideo, nil + } + if len(data) >= len(audioDatagramMagic) && string(data[:len(audioDatagramMagic)]) == audioDatagramMagic { + if err := validateAudioDatagramSize(data); err != nil { + return datagramOpaque, err + } + if _, err := decodeAudioDatagram(data); err != nil { + return datagramOpaque, err + } + } + return datagramOpaque, nil +} + +func prepareDatagram(data []byte) ([]byte, datagramKind, error) { + if len(data) == 0 { + return nil, datagramOpaque, fmt.Errorf("empty RemoteDesk datagram") + } + kind, err := classifyDatagram(data) + if err != nil { + return nil, kind, err + } + // Always copy a UDP read buffer before handing it to Hysteria2. + return append([]byte(nil), data...), kind, nil +} + func (c *udpConn) ReadFrom(b []byte) (int, string, error) { n, addr, err := c.conn.ReadFromUDP(b) if addr == nil { @@ -66,9 +108,30 @@ func main() { agent := flag.String("agent-udp", env("REMOTEDESK_AGENT_UDP", "127.0.0.1:39502"), "local Agent UDP endpoint") serverName := flag.String("server-name", env("REMOTEDESK_HYSTERIA2_SERVER_NAME", "localhost"), "TLS server name (client mode)") insecure := flag.Bool("insecure", false, "disable TLS certificate verification (client mode only)") + ringPath := flag.String("encoded-ring", os.Getenv("REMOTEDESK_ENCODED_RING"), "local Rust encoded-access-unit ring path (optional)") + ringStreamID := flag.Uint("ring-stream-id", 0, "RemoteDesk video stream ID for the encoded ring") + ringGeneration := flag.Uint64("ring-generation", 0, "RemoteDesk encoded ring generation") + agentControl := flag.String("windows-agent-control", os.Getenv("REMOTEDESK_WINDOWS_AGENT_CONTROL"), "loopback Windows Agent TCP control endpoint (optional)") + agentPipe := flag.String("windows-agent-pipe", os.Getenv("REMOTEDESK_WINDOWS_AGENT_PIPE"), "authenticated local Windows Agent Named Pipe (recommended)") + sessionID := flag.String("session-id", os.Getenv("REMOTEDESK_SESSION_ID"), "encoded desktop session ID") + ringFPS := flag.Uint("ring-fps", 30, "encoded desktop frame rate (1-120)") + allowSoftwareFallback := flag.Bool("allow-software-fallback", true, "allow Media Foundation software H.264 fallback") flag.Parse() if *mode == "client" { - runClient(*listen, *serverAddr, *password, *agent, *serverName, *insecure) + if *agentControl != "" && *agentPipe != "" { + log.Fatal("--windows-agent-control and --windows-agent-pipe are mutually exclusive") + } + controlEndpoint := *agentPipe + if controlEndpoint == "" { + controlEndpoint = *agentControl + } + if uint64(*ringStreamID) > uint64(^uint32(0)) { + log.Fatal("--ring-stream-id exceeds uint32") + } + if *ringFPS == 0 || *ringFPS > 120 { + log.Fatal("--ring-fps must be between 1 and 120") + } + runClient(*listen, *serverAddr, *password, *agent, *serverName, *insecure, *ringPath, uint32(*ringStreamID), *ringGeneration, controlEndpoint, *sessionID, uint8(*ringFPS), *allowSoftwareFallback) return } if *mode != "server" { @@ -118,7 +181,7 @@ func main() { time.Sleep(50 * time.Millisecond) } -func runClient(localListen, serverAddr, password, remoteAgent, serverName string, insecure bool) { +func runClient(localListen, serverAddr, password, remoteAgent, serverName string, insecure bool, ringPath string, ringStreamID uint32, ringGeneration uint64, agentControl, sessionID string, ringFPS uint8, allowSoftwareFallback bool) { if password == "" { log.Fatal("--password is required in client mode") } @@ -152,20 +215,92 @@ func runClient(localListen, serverAddr, password, remoteAgent, serverName string log.Fatalf("open Hysteria2 UDP session: %v", err) } defer stream.Close() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var sendMu sync.Mutex log.Printf("RemoteDesk Hysteria2 client listening on %s (UDP only)", localConn.LocalAddr()) log.Printf("authenticated Hysteria2 server %s; remote Agent %s", serverAddr, remoteAgent) + var agentControlConnection net.Conn + var audioDescriptor *audioRingDescriptor + if ringPath == "" && agentControl != "" { + connection, descriptor, err := openWindowsEncodedSession(agentControl, sessionID, ringFPS, allowSoftwareFallback) + if err != nil { + log.Fatalf("start Windows encoded desktop: %v", err) + } + agentControlConnection = connection + ringPath = descriptor.RingPath + ringGeneration = descriptor.Generation + if ringStreamID == 0 { + ringStreamID = 1 + } + audioDescriptor = descriptor.AudioRing + log.Printf("Windows encoded desktop session %s generation %d opened", sessionID, ringGeneration) + } + if agentControlConnection != nil { + defer agentControlConnection.Close() + } + if ringPath != "" { + if ringStreamID == 0 || ringGeneration == 0 { + log.Fatal("--ring-stream-id and --ring-generation are required with --encoded-ring") + } + ring, err := openEncodedRing(ringPath) + if err != nil { + log.Fatalf("open encoded media ring: %v", err) + } + go func() { + defer ring.Close() + err := pumpEncodedRing(ctx, ring, ringStreamID, ringGeneration, func(data []byte) error { + sendMu.Lock() + defer sendMu.Unlock() + return stream.Send(data, remoteAgent) + }) + if err != nil && !errors.Is(err, context.Canceled) { + log.Printf("encoded media ring stopped: %v", err) + } + cancel() + if agentControlConnection != nil { + _ = agentControlConnection.Close() + } + _ = localConn.Close() + }() + } + if audioDescriptor != nil { + audioRing, err := openAudioRing(audioDescriptor.RingPath) + if err != nil { + log.Printf("audio ring unavailable, continuing without audio: %v", err) + } else { + go func() { + defer audioRing.Close() + err := pumpAudioRing(ctx, audioRing, ringStreamID, audioDescriptor.Generation, func(data []byte) error { + sendMu.Lock() + defer sendMu.Unlock() + return stream.Send(data, remoteAgent) + }) + if err != nil && !errors.Is(err, context.Canceled) { + log.Printf("audio media ring stopped: %v", err) + } + }() + } + } var peerMu sync.RWMutex var peer *net.UDPAddr go func() { for { data, _, err := stream.Receive() if err != nil { + cancel() + _ = localConn.Close() return } + prepared, _, err := prepareDatagram(data) + if err != nil { + log.Printf("drop invalid datagram from Hysteria2: %v", err) + continue + } peerMu.RLock() currentPeer := peer if currentPeer != nil { - _, _ = localConn.WriteToUDP(data, currentPeer) + _, _ = localConn.WriteToUDP(prepared, currentPeer) } peerMu.RUnlock() } @@ -179,9 +314,17 @@ func runClient(localListen, serverAddr, password, remoteAgent, serverName string peerMu.Lock() peer = sourcePeer peerMu.Unlock() + prepared, _, err := prepareDatagram(buf[:n]) + if err != nil { + log.Printf("drop invalid local datagram from %s: %v", sourcePeer, err) + continue + } // The Hysteria2 destination is metadata only. The server's outbound // implementation pins delivery to its configured local Agent UDP port. - if err := stream.Send(append([]byte(nil), buf[:n]...), remoteAgent); err != nil { + sendMu.Lock() + err = stream.Send(prepared, remoteAgent) + sendMu.Unlock() + if err != nil { log.Printf("send Hysteria2 datagram from %s: %v", sourcePeer, err) return } diff --git a/transport/hysteria2-agent/media.go b/transport/hysteria2-agent/media.go new file mode 100644 index 0000000..7d1df8b --- /dev/null +++ b/transport/hysteria2-agent/media.go @@ -0,0 +1,352 @@ +package main + +import ( + "encoding/binary" + "errors" + "fmt" + "time" +) + +const ( + videoDatagramMagic = "RDV1" + videoDatagramVersion = 1 + videoDatagramHeaderBytes = 58 + videoDatagramPayloadMax = 1200 + videoDatagramMaxBytes = videoDatagramHeaderBytes + videoDatagramPayloadMax + videoFrameMaxBytes = 16 * 1024 * 1024 + videoFrameMaxFragments = 4096 + videoReassemblyMaxFrames = 8 + videoReassemblyTimeout = 100 * time.Millisecond +) + +const ( + audioDatagramMagic = "RDA1" + audioDatagramVersion = 1 + audioDatagramHeaderSize = 40 + audioPacketMaxBytes = 4 * 1024 + audioDatagramMaxBytes = audioDatagramHeaderSize + audioPacketMaxBytes +) + +const ( + videoFlagConfig uint16 = 1 << 0 + videoFlagKeyFrame uint16 = 1 << 1 + videoFlagFirstFragment uint16 = 1 << 2 + videoFlagLastFragment uint16 = 1 << 3 + videoFlagDiscontinuity uint16 = 1 << 4 +) + +const audioFlagDiscontinuity uint16 = 1 << 0 + +// VideoAccessUnit is a complete encoded frame before transport fragmentation. +// The Hysteria2 relay must treat the resulting datagrams as opaque bytes. +type VideoAccessUnit struct { + StreamID uint32 + Generation uint64 + Sequence uint64 + FrameID uint64 + PTS uint64 + DTS uint64 + Flags uint16 + Payload []byte +} + +// AudioPacket is one independently decodable Opus packet. Audio is never +// reassembled with video and is forwarded on its own priority path. +type AudioPacket struct { + StreamID uint32 + Generation uint64 + Sequence uint64 + PTS uint64 + DurationMS uint16 + Flags uint16 + Payload []byte +} + +func encodeAudioDatagram(packet AudioPacket) ([]byte, error) { + if err := validateAudioPacket(packet); err != nil { + return nil, err + } + output := make([]byte, audioDatagramHeaderSize+len(packet.Payload)) + copy(output, audioDatagramMagic) + output[4] = audioDatagramVersion + binary.LittleEndian.PutUint16(output[6:8], audioDatagramHeaderSize) + binary.LittleEndian.PutUint32(output[8:12], packet.StreamID) + binary.LittleEndian.PutUint64(output[12:20], packet.Generation) + binary.LittleEndian.PutUint64(output[20:28], packet.Sequence) + binary.LittleEndian.PutUint64(output[28:36], packet.PTS) + binary.LittleEndian.PutUint16(output[36:38], packet.DurationMS) + binary.LittleEndian.PutUint16(output[38:40], packet.Flags) + copy(output[audioDatagramHeaderSize:], packet.Payload) + return output, nil +} + +func decodeAudioDatagram(input []byte) (AudioPacket, error) { + if len(input) < audioDatagramHeaderSize || len(input) > audioDatagramMaxBytes { + return AudioPacket{}, errors.New("audio datagram size is invalid") + } + if string(input[:4]) != audioDatagramMagic || input[4] != audioDatagramVersion || input[5] != 0 { + return AudioPacket{}, errors.New("audio datagram version is invalid") + } + headerBytes := int(binary.LittleEndian.Uint16(input[6:8])) + if headerBytes != audioDatagramHeaderSize || headerBytes > len(input) { + return AudioPacket{}, errors.New("audio datagram header is invalid") + } + packet := AudioPacket{ + StreamID: binary.LittleEndian.Uint32(input[8:12]), + Generation: binary.LittleEndian.Uint64(input[12:20]), + Sequence: binary.LittleEndian.Uint64(input[20:28]), + PTS: binary.LittleEndian.Uint64(input[28:36]), + DurationMS: binary.LittleEndian.Uint16(input[36:38]), + Flags: binary.LittleEndian.Uint16(input[38:40]), + Payload: input[headerBytes:], + } + if err := validateAudioPacket(packet); err != nil { + return AudioPacket{}, err + } + return packet, nil +} + +func validateAudioPacket(packet AudioPacket) error { + if packet.StreamID == 0 || packet.Generation == 0 || packet.Sequence == 0 { + return errors.New("audio packet identity is invalid") + } + if packet.DurationMS != 10 && packet.DurationMS != 20 && packet.DurationMS != 40 && packet.DurationMS != 60 { + return errors.New("audio packet duration is invalid") + } + if len(packet.Payload) == 0 || len(packet.Payload) > audioPacketMaxBytes { + return errors.New("audio packet payload size is invalid") + } + if packet.Flags & ^audioFlagDiscontinuity != 0 { + return errors.New("audio packet flags are invalid") + } + return nil +} + +type videoDatagram struct { + StreamID uint32 + Generation uint64 + Sequence uint64 + FrameID uint64 + PTS uint64 + DTS uint64 + FragmentID uint16 + FragmentCount uint16 + Flags uint16 + Payload []byte +} + +func fragmentVideoAccessUnit(unit VideoAccessUnit) ([][]byte, error) { + if unit.StreamID == 0 || unit.Generation == 0 { + return nil, errors.New("video stream identity is invalid") + } + if len(unit.Payload) == 0 || len(unit.Payload) > videoFrameMaxBytes { + return nil, errors.New("video access unit size is invalid") + } + fragmentCount := (len(unit.Payload) + videoDatagramPayloadMax - 1) / videoDatagramPayloadMax + if fragmentCount > videoFrameMaxFragments { + return nil, errors.New("video access unit has too many fragments") + } + + datagrams := make([][]byte, 0, fragmentCount) + for fragmentID, offset := 0, 0; offset < len(unit.Payload); fragmentID++ { + end := min(offset+videoDatagramPayloadMax, len(unit.Payload)) + flags := unit.Flags + if fragmentID == 0 { + flags |= videoFlagFirstFragment + } + if end == len(unit.Payload) { + flags |= videoFlagLastFragment + } + packet := videoDatagram{ + StreamID: unit.StreamID, + Generation: unit.Generation, + Sequence: unit.Sequence, + FrameID: unit.FrameID, + PTS: unit.PTS, + DTS: unit.DTS, + FragmentID: uint16(fragmentID), + FragmentCount: uint16(fragmentCount), + Flags: flags, + Payload: unit.Payload[offset:end], + } + datagrams = append(datagrams, encodeVideoDatagram(packet)) + offset = end + } + return datagrams, nil +} + +func encodeVideoDatagram(packet videoDatagram) []byte { + output := make([]byte, videoDatagramHeaderBytes+len(packet.Payload)) + copy(output, videoDatagramMagic) + output[4] = videoDatagramVersion + binary.LittleEndian.PutUint16(output[6:8], videoDatagramHeaderBytes) + binary.LittleEndian.PutUint32(output[8:12], packet.StreamID) + binary.LittleEndian.PutUint64(output[12:20], packet.Generation) + binary.LittleEndian.PutUint64(output[20:28], packet.Sequence) + binary.LittleEndian.PutUint64(output[28:36], packet.FrameID) + binary.LittleEndian.PutUint64(output[36:44], packet.PTS) + binary.LittleEndian.PutUint64(output[44:52], packet.DTS) + binary.LittleEndian.PutUint16(output[52:54], packet.FragmentID) + binary.LittleEndian.PutUint16(output[54:56], packet.FragmentCount) + binary.LittleEndian.PutUint16(output[56:58], packet.Flags) + copy(output[videoDatagramHeaderBytes:], packet.Payload) + return output +} + +func decodeVideoDatagram(input []byte) (videoDatagram, error) { + if len(input) < videoDatagramHeaderBytes || len(input) > videoDatagramHeaderBytes+videoDatagramPayloadMax { + return videoDatagram{}, errors.New("video datagram size is invalid") + } + if string(input[:4]) != videoDatagramMagic || input[4] != videoDatagramVersion { + return videoDatagram{}, errors.New("video datagram version is invalid") + } + headerBytes := int(binary.LittleEndian.Uint16(input[6:8])) + if headerBytes != videoDatagramHeaderBytes || headerBytes > len(input) { + return videoDatagram{}, errors.New("video datagram header is invalid") + } + packet := videoDatagram{ + StreamID: binary.LittleEndian.Uint32(input[8:12]), + Generation: binary.LittleEndian.Uint64(input[12:20]), + Sequence: binary.LittleEndian.Uint64(input[20:28]), + FrameID: binary.LittleEndian.Uint64(input[28:36]), + PTS: binary.LittleEndian.Uint64(input[36:44]), + DTS: binary.LittleEndian.Uint64(input[44:52]), + FragmentID: binary.LittleEndian.Uint16(input[52:54]), + FragmentCount: binary.LittleEndian.Uint16(input[54:56]), + Flags: binary.LittleEndian.Uint16(input[56:58]), + Payload: input[headerBytes:], + } + if packet.StreamID == 0 || packet.Generation == 0 || packet.FragmentCount == 0 || + packet.FragmentCount > videoFrameMaxFragments || packet.FragmentID >= packet.FragmentCount || + len(packet.Payload) == 0 || len(packet.Payload) > videoDatagramPayloadMax { + return videoDatagram{}, errors.New("video datagram fields are invalid") + } + if packet.Flags&videoFlagFirstFragment != 0 != (packet.FragmentID == 0) || + packet.Flags&videoFlagLastFragment != 0 != (packet.FragmentID+1 == packet.FragmentCount) { + return videoDatagram{}, errors.New("video datagram fragment flags are invalid") + } + return packet, nil +} + +type videoFrameKey struct { + streamID uint32 + generation uint64 + frameID uint64 +} + +type videoFrameAssembly struct { + created time.Time + sequence uint64 + pts uint64 + dts uint64 + flags uint16 + fragmentCount uint16 + fragments map[uint16][]byte + bytes int +} + +// videoReassembler is intentionally bounded. It never waits for a missing +// datagram past the deadline and never retransmits an expired video frame. +type videoReassembler struct { + frames map[videoFrameKey]*videoFrameAssembly +} + +func newVideoReassembler() *videoReassembler { + return &videoReassembler{frames: make(map[videoFrameKey]*videoFrameAssembly)} +} + +func (r *videoReassembler) push(input []byte, now time.Time) (*VideoAccessUnit, error) { + packet, err := decodeVideoDatagram(input) + if err != nil { + return nil, err + } + r.expire(now) + key := videoFrameKey{streamID: packet.StreamID, generation: packet.Generation, frameID: packet.FrameID} + assembly := r.frames[key] + if assembly == nil { + if len(r.frames) >= videoReassemblyMaxFrames { + r.dropOldest() + } + assembly = &videoFrameAssembly{ + created: now, + sequence: packet.Sequence, + pts: packet.PTS, + dts: packet.DTS, + flags: packet.Flags &^ (videoFlagFirstFragment | videoFlagLastFragment), + fragmentCount: packet.FragmentCount, + fragments: make(map[uint16][]byte, packet.FragmentCount), + } + r.frames[key] = assembly + } else if assembly.fragmentCount != packet.FragmentCount || assembly.sequence != packet.Sequence { + delete(r.frames, key) + return nil, errors.New("video frame metadata changed during reassembly") + } + if _, exists := assembly.fragments[packet.FragmentID]; !exists { + assembly.fragments[packet.FragmentID] = append([]byte(nil), packet.Payload...) + assembly.bytes += len(packet.Payload) + } + if assembly.bytes > videoFrameMaxBytes || len(assembly.fragments) != int(assembly.fragmentCount) { + return nil, nil + } + payload := make([]byte, 0, assembly.bytes) + for index := uint16(0); index < assembly.fragmentCount; index++ { + fragment, ok := assembly.fragments[index] + if !ok { + return nil, nil + } + payload = append(payload, fragment...) + } + delete(r.frames, key) + return &VideoAccessUnit{ + StreamID: packet.StreamID, + Generation: packet.Generation, + Sequence: assembly.sequence, + FrameID: packet.FrameID, + PTS: assembly.pts, + DTS: assembly.dts, + Flags: assembly.flags | (packet.Flags & (videoFlagConfig | videoFlagKeyFrame | videoFlagDiscontinuity)), + Payload: payload, + }, nil +} + +func (r *videoReassembler) expire(now time.Time) { + for key, assembly := range r.frames { + if now.Sub(assembly.created) >= videoReassemblyTimeout { + delete(r.frames, key) + } + } +} + +func (r *videoReassembler) dropOldest() { + var oldest videoFrameKey + var oldestTime time.Time + for key, assembly := range r.frames { + if oldestTime.IsZero() || assembly.created.Before(oldestTime) { + oldest, oldestTime = key, assembly.created + } + } + if !oldestTime.IsZero() { + delete(r.frames, oldest) + } +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func validateVideoDatagramSize(input []byte) error { + if len(input) > videoDatagramMaxBytes { + return fmt.Errorf("video datagram exceeds %d bytes", videoDatagramMaxBytes) + } + return nil +} + +func validateAudioDatagramSize(input []byte) error { + if len(input) > audioDatagramMaxBytes { + return fmt.Errorf("audio datagram exceeds %d bytes", audioDatagramMaxBytes) + } + return nil +} diff --git a/transport/hysteria2-agent/media_ring.go b/transport/hysteria2-agent/media_ring.go new file mode 100644 index 0000000..a770323 --- /dev/null +++ b/transport/hysteria2-agent/media_ring.go @@ -0,0 +1,156 @@ +package main + +import ( + "context" + "errors" + "fmt" +) + +// EncodedRingRecord is the cross-process contract produced by the Rust +// Windows agent. PTS is the Media Foundation 100-ns media timeline. +type EncodedRingRecord struct { + Sequence uint64 + PTS uint64 + Kind uint32 + KeyFrame bool + Payload []byte +} + +// EncodedRing is intentionally a single-consumer interface. The producer may +// drop a video record when this queue is full; the consumer must never block +// input or audio while polling it. +type EncodedRing interface { + Pop() (*EncodedRingRecord, error) + Close() error +} + +// AudioRing is a separate consumer contract even though its mmap slot layout +// matches the video ring. Keeping the interfaces distinct prevents audio from +// accidentally sharing video backpressure or drop policy. +type AudioRing interface { + Pop() (*EncodedRingRecord, error) + Close() error +} + +func validateRingRecord(record *EncodedRingRecord) error { + if record == nil || record.Sequence == 0 || len(record.Payload) == 0 || len(record.Payload) > videoFrameMaxBytes { + return errors.New("encoded ring record is invalid") + } + return nil +} + +// pumpEncodedRing drains the ring into RDV1 datagrams. The callback is the +// already-authenticated Hysteria2 UDP stream; this function never retries a +// failed video packet. +func pumpEncodedRing(ctx context.Context, ring EncodedRing, streamID uint32, generation uint64, send func([]byte) error) error { + if streamID == 0 || generation == 0 { + return errors.New("encoded ring stream identity is invalid") + } + if ring == nil || send == nil { + return errors.New("encoded ring pump arguments are invalid") + } + ticker := newMediaPollTicker() + defer ticker.Stop() + for { + drained := false + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + record, err := ring.Pop() + if err != nil { + return fmt.Errorf("read encoded media ring: %w", err) + } + if record == nil { + break + } + drained = true + if err := validateRingRecord(record); err != nil { + return err + } + flags := uint16(0) + if record.KeyFrame { + flags |= videoFlagKeyFrame + } + datagrams, err := fragmentVideoAccessUnit(VideoAccessUnit{ + StreamID: streamID, Generation: generation, Sequence: record.Sequence, + FrameID: record.Sequence, PTS: record.PTS, DTS: record.PTS, + Flags: flags, Payload: record.Payload, + }) + if err != nil { + return fmt.Errorf("fragment encoded media: %w", err) + } + for _, datagram := range datagrams { + if err := send(datagram); err != nil { + return fmt.Errorf("send encoded video datagram: %w", err) + } + } + } + if drained { + continue + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} + +// pumpAudioRing sends independent Opus packets as RDA1 datagrams. Audio does +// not wait for video and does not use video frame reassembly. +func pumpAudioRing(ctx context.Context, ring AudioRing, streamID uint32, generation uint64, send func([]byte) error) error { + if streamID == 0 || generation == 0 { + return errors.New("audio ring stream identity is invalid") + } + if ring == nil || send == nil { + return errors.New("audio ring pump arguments are invalid") + } + ticker := newMediaPollTicker() + defer ticker.Stop() + for { + drained := false + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + record, err := ring.Pop() + if err != nil { + return fmt.Errorf("read audio media ring: %w", err) + } + if record == nil { + break + } + drained = true + if record.Sequence == 0 || len(record.Payload) == 0 || len(record.Payload) > audioPacketMaxBytes { + return errors.New("audio ring record is invalid") + } + flags := uint16(0) + if record.Kind == 2 { + flags |= audioFlagDiscontinuity + } + datagram, err := encodeAudioDatagram(AudioPacket{ + StreamID: streamID, Generation: generation, Sequence: record.Sequence, + PTS: record.PTS, DurationMS: 20, Flags: flags, Payload: record.Payload, + }) + if err != nil { + return fmt.Errorf("encode audio datagram: %w", err) + } + if err := send(datagram); err != nil { + return fmt.Errorf("send audio datagram: %w", err) + } + } + if drained { + continue + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} diff --git a/transport/hysteria2-agent/media_ring_poll.go b/transport/hysteria2-agent/media_ring_poll.go new file mode 100644 index 0000000..f43f167 --- /dev/null +++ b/transport/hysteria2-agent/media_ring_poll.go @@ -0,0 +1,11 @@ +package main + +import "time" + +type mediaPollTicker struct { + *time.Ticker +} + +func newMediaPollTicker() *mediaPollTicker { + return &mediaPollTicker{Ticker: time.NewTicker(2 * time.Millisecond)} +} diff --git a/transport/hysteria2-agent/media_ring_unix.go b/transport/hysteria2-agent/media_ring_unix.go new file mode 100644 index 0000000..553333d --- /dev/null +++ b/transport/hysteria2-agent/media_ring_unix.go @@ -0,0 +1,105 @@ +//go:build !windows + +package main + +import ( + "encoding/binary" + "errors" + "fmt" + "os" + "sync/atomic" + "syscall" + "unsafe" +) + +const ( + ringMagic = "RDMRING1" + ringVersion = uint32(1) + ringHeaderBytes = 40 + slotHeaderBytes = 24 + maxRingSlots = uint32(256) + maxRingSlotBytes = uint32(16 * 1024 * 1024) +) + +type mappedEncodedRing struct { + file *os.File + path string + data []byte + slotCount uint32 + slotBytes uint32 +} + +func openEncodedRing(path string) (EncodedRing, error) { + file, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return nil, fmt.Errorf("open encoded ring: %w", err) + } + stat, err := file.Stat() + if err != nil { + _ = file.Close() + return nil, err + } + if stat.Size() < ringHeaderBytes { + _ = file.Close() + return nil, errors.New("encoded ring is shorter than its header") + } + data, err := syscall.Mmap(int(file.Fd()), 0, int(stat.Size()), syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED) + if err != nil { + _ = file.Close() + return nil, fmt.Errorf("map encoded ring: %w", err) + } + if string(data[:8]) != ringMagic || binary.LittleEndian.Uint32(data[8:12]) != ringVersion { + _ = syscall.Munmap(data) + _ = file.Close() + return nil, errors.New("encoded ring header is invalid") + } + slots, slotBytes := binary.LittleEndian.Uint32(data[12:16]), binary.LittleEndian.Uint32(data[16:20]) + expected := uint64(ringHeaderBytes) + uint64(slots)*uint64(slotHeaderBytes+slotBytes) + if slots < 2 || slots > maxRingSlots || slots&(slots-1) != 0 || slotBytes == 0 || slotBytes > maxRingSlotBytes || expected != uint64(len(data)) { + _ = syscall.Munmap(data) + _ = file.Close() + return nil, errors.New("encoded ring geometry is invalid") + } + return &mappedEncodedRing{file: file, path: path, data: data, slotCount: slots, slotBytes: slotBytes}, nil +} + +func openAudioRing(path string) (AudioRing, error) { + ring, err := openEncodedRing(path) + if err != nil { + return nil, err + } + return ring, nil +} + +func (r *mappedEncodedRing) Pop() (*EncodedRingRecord, error) { + read := atomic.LoadUint64((*uint64)(unsafe.Pointer(&r.data[32]))) + write := atomic.LoadUint64((*uint64)(unsafe.Pointer(&r.data[24]))) + if read == write { + return nil, nil + } + start := ringHeaderBytes + int(read%uint64(r.slotCount))*int(slotHeaderBytes+r.slotBytes) + length := binary.LittleEndian.Uint32(r.data[start+20 : start+24]) + if length == 0 || length > r.slotBytes { + return nil, errors.New("encoded ring record length is invalid") + } + record := &EncodedRingRecord{Sequence: binary.LittleEndian.Uint64(r.data[start : start+8]), PTS: binary.LittleEndian.Uint64(r.data[start+8 : start+16]), Kind: binary.LittleEndian.Uint32(r.data[start+16 : start+20]), Payload: append([]byte(nil), r.data[start+slotHeaderBytes:start+slotHeaderBytes+int(length)]...)} + record.KeyFrame = record.Kind == 2 + if record.Sequence == 0 { + return nil, errors.New("encoded ring record sequence is invalid") + } + atomic.StoreUint64((*uint64)(unsafe.Pointer(&r.data[32])), read+1) + return record, nil +} + +func (r *mappedEncodedRing) Close() error { + err := syscall.Munmap(r.data) + if r.file != nil { + if closeErr := r.file.Close(); err == nil { + err = closeErr + } + } + if removeErr := os.Remove(r.path); err == nil && removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + err = removeErr + } + return err +} diff --git a/transport/hysteria2-agent/media_ring_windows.go b/transport/hysteria2-agent/media_ring_windows.go new file mode 100644 index 0000000..bf47e29 --- /dev/null +++ b/transport/hysteria2-agent/media_ring_windows.go @@ -0,0 +1,146 @@ +//go:build windows + +package main + +import ( + "encoding/binary" + "errors" + "fmt" + "os" + "sync/atomic" + "unsafe" + + "golang.org/x/sys/windows" +) + +const ( + ringMagic = "RDMRING1" + ringVersion = uint32(1) + ringHeaderBytes = 40 + slotHeaderBytes = 24 + maxRingSlots = uint32(256) + maxRingSlotBytes = uint32(16 * 1024 * 1024) +) + +type mappedEncodedRing struct { + file *os.File + path string + mapping windows.Handle + address uintptr + data []byte + slotCount uint32 + slotBytes uint32 +} + +func openEncodedRing(path string) (EncodedRing, error) { + file, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return nil, fmt.Errorf("open encoded ring: %w", err) + } + closeOnError := true + defer func() { + if closeOnError { + _ = file.Close() + } + }() + stat, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("stat encoded ring: %w", err) + } + length := stat.Size() + if length < ringHeaderBytes { + return nil, errors.New("encoded ring is shorter than its header") + } + mapping, err := windows.CreateFileMapping(windows.Handle(file.Fd()), nil, windows.PAGE_READWRITE, 0, 0, nil) + if err != nil { + return nil, fmt.Errorf("map encoded ring file: %w", err) + } + address, err := windows.MapViewOfFile(mapping, windows.FILE_MAP_READ|windows.FILE_MAP_WRITE, 0, 0, uintptr(length)) + if err != nil { + _ = windows.CloseHandle(mapping) + return nil, fmt.Errorf("map encoded ring view: %w", err) + } + if uintptr(length) > uintptr(^uint(0)>>1) { + _ = windows.UnmapViewOfFile(address) + _ = windows.CloseHandle(mapping) + return nil, errors.New("encoded ring is too large for this process") + } + data := unsafe.Slice((*byte)(unsafe.Pointer(address)), int(length)) + if string(data[:8]) != ringMagic || binary.LittleEndian.Uint32(data[8:12]) != ringVersion { + _ = windows.UnmapViewOfFile(address) + _ = windows.CloseHandle(mapping) + return nil, errors.New("encoded ring header magic or version is invalid") + } + slots := binary.LittleEndian.Uint32(data[12:16]) + slotBytes := binary.LittleEndian.Uint32(data[16:20]) + if slots < 2 || slots > maxRingSlots || slots&(slots-1) != 0 || slotBytes == 0 || slotBytes > maxRingSlotBytes { + _ = windows.UnmapViewOfFile(address) + _ = windows.CloseHandle(mapping) + return nil, errors.New("encoded ring geometry is invalid") + } + expected := uint64(ringHeaderBytes) + uint64(slots)*uint64(slotHeaderBytes+slotBytes) + if expected != uint64(length) { + _ = windows.UnmapViewOfFile(address) + _ = windows.CloseHandle(mapping) + return nil, errors.New("encoded ring file size does not match its header") + } + closeOnError = false + return &mappedEncodedRing{file: file, path: path, mapping: mapping, address: address, data: data, slotCount: slots, slotBytes: slotBytes}, nil +} + +func openAudioRing(path string) (AudioRing, error) { + ring, err := openEncodedRing(path) + if err != nil { + return nil, err + } + return ring, nil +} + +func (r *mappedEncodedRing) Pop() (*EncodedRingRecord, error) { + read := atomic.LoadUint64((*uint64)(unsafe.Pointer(&r.data[32]))) + write := atomic.LoadUint64((*uint64)(unsafe.Pointer(&r.data[24]))) + if read == write { + return nil, nil + } + offset := ringHeaderBytes + (read%uint64(r.slotCount))*uint64(slotHeaderBytes+r.slotBytes) + start := int(offset) + if start+slotHeaderBytes > len(r.data) { + return nil, errors.New("encoded ring slot header is out of bounds") + } + length := binary.LittleEndian.Uint32(r.data[start+20 : start+24]) + if length == 0 || length > r.slotBytes || uint64(start+slotHeaderBytes)+uint64(length) > uint64(len(r.data)) { + return nil, errors.New("encoded ring record length is invalid") + } + payload := append([]byte(nil), r.data[start+slotHeaderBytes:start+slotHeaderBytes+int(length)]...) + record := &EncodedRingRecord{ + Sequence: binary.LittleEndian.Uint64(r.data[start : start+8]), + PTS: binary.LittleEndian.Uint64(r.data[start+8 : start+16]), + Kind: binary.LittleEndian.Uint32(r.data[start+16 : start+20]), + Payload: payload, + } + record.KeyFrame = record.Kind == 2 + if record.Sequence == 0 { + return nil, errors.New("encoded ring record sequence is invalid") + } + atomic.StoreUint64((*uint64)(unsafe.Pointer(&r.data[32])), read+1) + return record, nil +} + +func (r *mappedEncodedRing) Close() error { + if r.address != 0 { + _ = windows.UnmapViewOfFile(r.address) + r.address = 0 + } + if r.mapping != 0 { + _ = windows.CloseHandle(r.mapping) + r.mapping = 0 + } + if r.file != nil { + err := r.file.Close() + if removeErr := os.Remove(r.path); err == nil && removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + err = removeErr + } + return err + } + return nil +} diff --git a/transport/hysteria2-agent/media_test.go b/transport/hysteria2-agent/media_test.go new file mode 100644 index 0000000..a2c1ffb --- /dev/null +++ b/transport/hysteria2-agent/media_test.go @@ -0,0 +1,302 @@ +package main + +import ( + "bytes" + "context" + "encoding/binary" + "testing" + "time" +) + +type testEncodedRing struct { + records []*EncodedRingRecord +} + +func (r *testEncodedRing) Pop() (*EncodedRingRecord, error) { + if len(r.records) == 0 { + return nil, nil + } + record := r.records[0] + r.records = r.records[1:] + return record, nil +} + +func (*testEncodedRing) Close() error { return nil } + +func TestVideoAccessUnitRoundTripThroughFragments(t *testing.T) { + payload := bytes.Repeat([]byte{0x5a}, videoDatagramPayloadMax*2+17) + unit := VideoAccessUnit{ + StreamID: 1, + Generation: 2, + Sequence: 3, + FrameID: 4, + PTS: 5, + DTS: 6, + Flags: videoFlagKeyFrame, + Payload: payload, + } + datagrams, err := fragmentVideoAccessUnit(unit) + if err != nil { + t.Fatal(err) + } + if len(datagrams) != 3 { + t.Fatalf("got %d datagrams, want 3", len(datagrams)) + } + + reassembler := newVideoReassembler() + var recovered *VideoAccessUnit + now := time.Unix(100, 0) + for _, datagram := range datagrams { + recovered, err = reassembler.push(datagram, now) + if err != nil { + t.Fatal(err) + } + } + if recovered == nil { + t.Fatal("frame was not reassembled") + } + if !bytes.Equal(recovered.Payload, payload) || recovered.Flags&videoFlagKeyFrame == 0 { + t.Fatal("reassembled frame does not match source") + } +} + +func TestVideoReassemblerExpiresIncompleteFrame(t *testing.T) { + datagrams, err := fragmentVideoAccessUnit(VideoAccessUnit{ + StreamID: 1, + Generation: 1, + Sequence: 1, + FrameID: 1, + PTS: 1, + DTS: 1, + Flags: videoFlagKeyFrame, + Payload: bytes.Repeat([]byte{1}, videoDatagramPayloadMax+1), + }) + if err != nil { + t.Fatal(err) + } + reassembler := newVideoReassembler() + now := time.Unix(200, 0) + if _, err := reassembler.push(datagrams[0], now); err != nil { + t.Fatal(err) + } + if _, err := reassembler.push(datagrams[0], now.Add(videoReassemblyTimeout+time.Nanosecond)); err != nil { + t.Fatal(err) + } + if len(reassembler.frames) != 1 { + t.Fatalf("expected only the new incomplete assembly, got %d", len(reassembler.frames)) + } +} + +func TestVideoDatagramRejectsInvalidFragmentFlags(t *testing.T) { + datagrams, err := fragmentVideoAccessUnit(VideoAccessUnit{ + StreamID: 1, + Generation: 1, + Sequence: 1, + FrameID: 1, + Payload: []byte{1}, + }) + if err != nil { + t.Fatal(err) + } + datagrams[0][56] = 0 + datagrams[0][57] = 0 + if _, err := decodeVideoDatagram(datagrams[0]); err == nil { + t.Fatal("invalid first/last flags were accepted") + } +} + +func TestVideoDatagramSizeUsesCompleteDatagramLimit(t *testing.T) { + if err := validateVideoDatagramSize(make([]byte, videoDatagramMaxBytes)); err != nil { + t.Fatal(err) + } + if err := validateVideoDatagramSize(make([]byte, videoDatagramMaxBytes+1)); err == nil { + t.Fatal("oversized datagram was accepted") + } +} + +func TestPrepareDatagramKeepsOpaquePacketsUnparsed(t *testing.T) { + input := []byte("audio-or-control") + prepared, kind, err := prepareDatagram(input) + if err != nil { + t.Fatal(err) + } + if kind != datagramOpaque || !bytes.Equal(prepared, input) { + t.Fatalf("unexpected opaque packet result: kind=%d payload=%q", kind, prepared) + } + prepared[0] = 'X' + if input[0] == 'X' { + t.Fatal("prepared datagram aliases the UDP input buffer") + } +} + +func TestPrepareDatagramRecognizesVideoPackets(t *testing.T) { + datagrams, err := fragmentVideoAccessUnit(VideoAccessUnit{ + StreamID: 1, Generation: 1, Sequence: 1, FrameID: 1, Payload: []byte{1}, + }) + if err != nil { + t.Fatal(err) + } + _, kind, err := prepareDatagram(datagrams[0]) + if err != nil { + t.Fatal(err) + } + if kind != datagramVideo { + t.Fatalf("got datagram kind %d, want video", kind) + } +} + +func TestAudioPacketRoundTripIsIndependent(t *testing.T) { + input, err := encodeAudioDatagram(AudioPacket{ + StreamID: 1, Generation: 2, Sequence: 3, PTS: 4, DurationMS: 20, + Payload: []byte{0x01, 0x02, 0x03}, + }) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeAudioDatagram(input) + if err != nil { + t.Fatal(err) + } + if decoded.StreamID != 1 || decoded.Generation != 2 || decoded.Sequence != 3 || + decoded.PTS != 4 || decoded.DurationMS != 20 || !bytes.Equal(decoded.Payload, []byte{1, 2, 3}) { + t.Fatalf("decoded audio packet does not match source: %+v", decoded) + } + prepared, kind, err := prepareDatagram(input) + if err != nil { + t.Fatal(err) + } + if kind != datagramOpaque || !bytes.Equal(prepared, input) { + t.Fatalf("audio packet was not preserved as opaque transport data") + } +} + +func TestAudioDatagramRejectsInvalidDurationAndSize(t *testing.T) { + if _, err := encodeAudioDatagram(AudioPacket{ + StreamID: 1, Generation: 1, Sequence: 1, PTS: 1, DurationMS: 15, Payload: []byte{1}, + }); err == nil { + t.Fatal("invalid audio duration was accepted") + } + if err := validateAudioDatagramSize(make([]byte, audioDatagramMaxBytes+1)); err == nil { + t.Fatal("oversized audio datagram was accepted") + } +} + +func TestAudioDatagramRejectsReservedByteAndUnknownFlags(t *testing.T) { + encoded, err := encodeAudioDatagram(AudioPacket{ + StreamID: 1, Generation: 2, Sequence: 3, PTS: 4, + DurationMS: 20, Payload: []byte("opus"), + }) + if err != nil { + t.Fatal(err) + } + encoded[5] = 1 + if _, err := decodeAudioDatagram(encoded); err == nil { + t.Fatal("audio reserved byte was accepted") + } + encoded[5] = 0 + binary.LittleEndian.PutUint16(encoded[38:40], 2) + if _, err := decodeAudioDatagram(encoded); err == nil { + t.Fatal("unknown audio flags were accepted") + } +} + +func TestPumpEncodedRingFragmentsAndMarksKeyFrame(t *testing.T) { + payload := bytes.Repeat([]byte{0x42}, videoDatagramPayloadMax+3) + ring := &testEncodedRing{records: []*EncodedRingRecord{{ + Sequence: 9, PTS: 123, KeyFrame: true, Payload: payload, + }}} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var datagrams [][]byte + send := func(data []byte) error { + datagrams = append(datagrams, append([]byte(nil), data...)) + if len(datagrams) == 2 { + cancel() + } + return nil + } + err := pumpEncodedRing(ctx, ring, 7, 8, send) + if err != context.Canceled { + t.Fatalf("pump returned %v, want context cancellation", err) + } + if len(datagrams) != 2 { + t.Fatalf("got %d datagrams, want 2", len(datagrams)) + } + first, err := decodeVideoDatagram(datagrams[0]) + if err != nil { + t.Fatal(err) + } + if first.Flags&videoFlagKeyFrame == 0 || first.PTS != 123 || first.FrameID != 9 { + t.Fatalf("ring metadata was not preserved: %+v", first) + } +} + +func TestPumpEncodedRingRejectsInvalidRecord(t *testing.T) { + ring := &testEncodedRing{records: []*EncodedRingRecord{{Sequence: 0, Payload: []byte{1}}}} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + err := pumpEncodedRing(ctx, ring, 1, 1, func([]byte) error { return nil }) + if err == nil { + t.Fatal("invalid encoded ring record was accepted") + } +} + +func TestPumpAudioRingEncodesRDA1AndDiscontinuity(t *testing.T) { + ring := &testEncodedRing{records: []*EncodedRingRecord{{ + Sequence: 7, + PTS: 400_000, + Kind: 2, + Payload: []byte("opus"), + }}} + ctx, cancel := context.WithCancel(context.Background()) + var datagram []byte + err := pumpAudioRing(ctx, ring, 11, 12, func(data []byte) error { + datagram = append([]byte(nil), data...) + cancel() + return nil + }) + if err != context.Canceled { + t.Fatalf("pump returned %v, want context cancellation", err) + } + packet, err := decodeAudioDatagram(datagram) + if err != nil { + t.Fatal(err) + } + if packet.StreamID != 11 || packet.Generation != 12 || packet.Sequence != 7 || + packet.PTS != 400_000 || packet.DurationMS != 20 || + packet.Flags&audioFlagDiscontinuity == 0 || !bytes.Equal(packet.Payload, []byte("opus")) { + t.Fatalf("audio ring metadata was not preserved: %+v", packet) + } +} + +func TestPumpAudioRingRejectsInvalidRecords(t *testing.T) { + tests := []struct { + name string + record *EncodedRingRecord + }{ + {name: "zero sequence", record: &EncodedRingRecord{Payload: []byte{1}}}, + {name: "empty payload", record: &EncodedRingRecord{Sequence: 1}}, + { + name: "oversized payload", + record: &EncodedRingRecord{ + Sequence: 1, + Payload: make([]byte, audioPacketMaxBytes+1), + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ring := &testEncodedRing{records: []*EncodedRingRecord{test.record}} + err := pumpAudioRing( + context.Background(), + ring, + 1, + 1, + func([]byte) error { return nil }, + ) + if err == nil { + t.Fatal("invalid audio ring record was accepted") + } + }) + } +} diff --git a/transport/hysteria2-agent/windows_agent_control.go b/transport/hysteria2-agent/windows_agent_control.go new file mode 100644 index 0000000..498ae29 --- /dev/null +++ b/transport/hysteria2-agent/windows_agent_control.go @@ -0,0 +1,254 @@ +package main + +import ( + "bufio" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "strings" + "time" +) + +const ( + agentControlLineMax = 16 * 1024 + agentPipeTokenEnv = "REMOTEDESK_AGENT_PIPE_TOKEN" + agentPipeAuthDomain = "RemoteDesk Windows Agent pipe auth v1\x00" +) + +type mediaRingDescriptor struct { + SchemaVersion uint16 `json:"schema_version"` + SessionID string `json:"session_id"` + Generation uint64 `json:"generation"` + OwnerPID uint32 `json:"owner_pid"` + RingPath string `json:"ring_path"` + SlotCount uint32 `json:"slot_count"` + SlotBytes uint32 `json:"slot_bytes"` + VideoTransport string `json:"video_transport"` + AudioTransport string `json:"audio_transport"` + AudioRing *audioRingDescriptor `json:"audio_ring,omitempty"` +} + +type audioRingDescriptor struct { + SchemaVersion uint16 `json:"schema_version"` + SessionID string `json:"session_id"` + Generation uint64 `json:"generation"` + OwnerPID uint32 `json:"owner_pid"` + RingPath string `json:"ring_path"` + SlotCount uint32 `json:"slot_count"` + SlotBytes uint32 `json:"slot_bytes"` + AudioTransport string `json:"audio_transport"` +} + +type encodedDesktopOpened struct { + Kind string `json:"kind"` + Error string `json:"error"` + VideoCodec string `json:"video_codec"` + MediaTransport string `json:"media_transport"` + Ring mediaRingDescriptor `json:"ring"` +} + +type agentHello struct { + Kind string `json:"kind"` + Authenticated bool `json:"authenticated"` +} + +type agentAuthChallenge struct { + Kind string `json:"kind"` + ProtocolVersion uint8 `json:"protocol_version"` + Nonce string `json:"nonce"` +} + +func openWindowsEncodedSession(address, sessionID string, fps uint8, allowSoftwareFallback bool) (net.Conn, mediaRingDescriptor, error) { + if err := validateAgentControlAddress(address); err != nil { + return nil, mediaRingDescriptor{}, err + } + if !validSessionID(sessionID) || fps == 0 || fps > 120 { + return nil, mediaRingDescriptor{}, errors.New("Windows Agent encoded session parameters are invalid") + } + isPipe := isWindowsAgentPipe(address) + connection, err := dialWindowsAgentControl(address, 2*time.Second) + if err != nil { + return nil, mediaRingDescriptor{}, fmt.Errorf("connect Windows Agent control: %w", err) + } + closeOnError := true + defer func() { + if closeOnError { + _ = connection.Close() + } + }() + _ = connection.SetDeadline(time.Now().Add(10 * time.Second)) + reader := bufio.NewReaderSize(connection, agentControlLineMax+1) + if isPipe { + if err := authenticateWindowsAgentPipe(reader, connection); err != nil { + return nil, mediaRingDescriptor{}, err + } + } + var hello agentHello + if err := readBoundedAgentJSON(reader, &hello); err != nil || hello.Kind != "windows_agent_hello" || (isPipe && !hello.Authenticated) { + return nil, mediaRingDescriptor{}, errors.New("Windows Agent control hello is invalid") + } + request := struct { + Kind string `json:"kind"` + SessionID string `json:"session_id"` + CaptureMode string `json:"capture_mode"` + VideoCodec string `json:"video_codec"` + FramesPerSecond uint8 `json:"frames_per_second"` + AllowSoftwareFallback bool `json:"allow_software_fallback"` + }{ + Kind: "open_encoded_desktop", SessionID: sessionID, + CaptureMode: "compatibility", VideoCodec: "h264", FramesPerSecond: fps, + AllowSoftwareFallback: allowSoftwareFallback, + } + encoded, err := json.Marshal(request) + if err != nil { + return nil, mediaRingDescriptor{}, err + } + encoded = append(encoded, '\n') + if _, err := connection.Write(encoded); err != nil { + return nil, mediaRingDescriptor{}, fmt.Errorf("open Windows encoded desktop: %w", err) + } + var opened encodedDesktopOpened + if err := readBoundedAgentJSON(reader, &opened); err != nil { + return nil, mediaRingDescriptor{}, err + } + if opened.Kind != "encoded_desktop_opened" || opened.VideoCodec != "h264" || + opened.MediaTransport != "shared_memory_encoded_access_units" { + return nil, mediaRingDescriptor{}, fmt.Errorf("Windows encoded desktop unavailable: %s", opened.Error) + } + if err := validateMediaRingDescriptor(opened.Ring, sessionID); err != nil { + return nil, mediaRingDescriptor{}, err + } + _ = connection.SetDeadline(time.Time{}) + closeOnError = false + return connection, opened.Ring, nil +} + +func dialWindowsAgentControl(address string, timeout time.Duration) (net.Conn, error) { + if isWindowsAgentPipe(address) { + return dialWindowsAgentPipe(address, timeout) + } + return net.DialTimeout("tcp", address, timeout) +} + +func authenticateWindowsAgentPipe(reader *bufio.Reader, connection net.Conn) error { + var challenge agentAuthChallenge + if err := readBoundedAgentJSON(reader, &challenge); err != nil { + return err + } + if challenge.Kind != "agent_auth_challenge" || challenge.ProtocolVersion != 1 { + return errors.New("Windows Agent Pipe challenge is invalid") + } + nonce, err := base64.RawURLEncoding.DecodeString(challenge.Nonce) + if err != nil || len(nonce) != 32 { + return errors.New("Windows Agent Pipe nonce is invalid") + } + encodedToken, ok := os.LookupEnv(agentPipeTokenEnv) + if !ok { + return fmt.Errorf("%s is required for authenticated Windows Agent Pipe", agentPipeTokenEnv) + } + _ = os.Unsetenv(agentPipeTokenEnv) + key, err := base64.RawURLEncoding.DecodeString(encodedToken) + if err != nil || len(key) != 32 { + return errors.New("Windows Agent Pipe token must be 32-byte base64url") + } + mac := windowsAgentPipeMAC(key, nonce) + request := struct { + Kind string `json:"kind"` + ProtocolVersion uint8 `json:"protocol_version"` + MAC string `json:"mac"` + }{ + Kind: "agent_auth", + ProtocolVersion: 1, + MAC: base64.RawURLEncoding.EncodeToString(mac), + } + encoded, err := json.Marshal(request) + if err != nil { + return err + } + encoded = append(encoded, '\n') + if _, err := connection.Write(encoded); err != nil { + return fmt.Errorf("authenticate Windows Agent Pipe: %w", err) + } + return nil +} + +func windowsAgentPipeMAC(key, nonce []byte) []byte { + mac := hmac.New(sha256.New, key) + _, _ = mac.Write([]byte(agentPipeAuthDomain)) + _, _ = mac.Write(nonce) + return mac.Sum(nil) +} + +func readBoundedAgentJSON(reader *bufio.Reader, output any) error { + line, err := reader.ReadSlice('\n') + if err != nil { + return fmt.Errorf("read Windows Agent control: %w", err) + } + if len(line) == 0 || len(line) > agentControlLineMax || line[len(line)-1] != '\n' { + return errors.New("Windows Agent control response exceeds its bound") + } + if err := json.Unmarshal(line, output); err != nil { + return errors.New("Windows Agent control response is invalid JSON") + } + return nil +} + +func validateAgentControlAddress(address string) error { + if isWindowsAgentPipe(address) { + if len(address) > 256 || !strings.HasPrefix(strings.ToLower(address), `\\.\pipe\remotedesk\`) || strings.Contains(address, "..") { + return errors.New("Windows Agent Pipe name is outside the RemoteDesk namespace") + } + return nil + } + host, port, err := net.SplitHostPort(address) + if err != nil || port == "" { + return errors.New("Windows Agent control must be an IP socket address") + } + ip := net.ParseIP(host) + if ip == nil || !ip.IsLoopback() { + return errors.New("Windows Agent control must use a loopback IP") + } + return nil +} + +func isWindowsAgentPipe(address string) bool { + return strings.HasPrefix(strings.ToLower(address), `\\.\pipe\`) +} + +func validateMediaRingDescriptor(descriptor mediaRingDescriptor, sessionID string) error { + if descriptor.SchemaVersion != 1 || descriptor.SessionID != sessionID || descriptor.Generation == 0 || descriptor.OwnerPID == 0 || + descriptor.VideoTransport != "shared_memory_encoded_access_units" || descriptor.AudioTransport != "separate_transport" || + !filepath.IsAbs(descriptor.RingPath) || descriptor.SlotCount < 2 || descriptor.SlotCount > maxRingSlots || + descriptor.SlotCount&(descriptor.SlotCount-1) != 0 || descriptor.SlotBytes == 0 || descriptor.SlotBytes > maxRingSlotBytes { + return errors.New("Windows Agent media ring descriptor is invalid") + } + if descriptor.AudioRing != nil { + audio := descriptor.AudioRing + if audio.SchemaVersion != 1 || audio.SessionID != sessionID || audio.Generation != descriptor.Generation || + audio.OwnerPID != descriptor.OwnerPID || audio.AudioTransport != "shared_memory_opus_packets" || + !filepath.IsAbs(audio.RingPath) || audio.SlotCount < 2 || audio.SlotCount > maxRingSlots || + audio.SlotCount&(audio.SlotCount-1) != 0 || audio.SlotBytes == 0 || audio.SlotBytes > maxRingSlotBytes { + return errors.New("Windows Agent audio ring descriptor is invalid") + } + } + return nil +} + +func validSessionID(value string) bool { + if len(value) == 0 || len(value) > 128 { + return false + } + for _, character := range []byte(value) { + if !((character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z') || + (character >= '0' && character <= '9') || character == '-' || character == '_') { + return false + } + } + return true +} diff --git a/transport/hysteria2-agent/windows_agent_control_test.go b/transport/hysteria2-agent/windows_agent_control_test.go new file mode 100644 index 0000000..e2366e0 --- /dev/null +++ b/transport/hysteria2-agent/windows_agent_control_test.go @@ -0,0 +1,111 @@ +package main + +import ( + "bufio" + "encoding/base64" + "encoding/json" + "net" + "path/filepath" + "testing" +) + +func TestOpenWindowsEncodedSessionBindsDescriptorToRequest(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + serverError := make(chan error, 1) + go func() { + connection, err := listener.Accept() + if err != nil { + serverError <- err + return + } + defer connection.Close() + if _, err := connection.Write([]byte("{\"kind\":\"windows_agent_hello\"}\n")); err != nil { + serverError <- err + return + } + var request struct { + Kind string `json:"kind"` + SessionID string `json:"session_id"` + FramesPerSecond uint8 `json:"frames_per_second"` + AllowSoftwareFallback bool `json:"allow_software_fallback"` + } + if err := readBoundedAgentJSON(bufio.NewReader(connection), &request); err != nil { + serverError <- err + return + } + if request.Kind != "open_encoded_desktop" || request.SessionID != "session-1" || request.FramesPerSecond != 60 || !request.AllowSoftwareFallback { + serverError <- &testControlError{"encoded desktop request mismatch"} + return + } + response := encodedDesktopOpened{ + Kind: "encoded_desktop_opened", VideoCodec: "h264", + MediaTransport: "shared_memory_encoded_access_units", + Ring: mediaRingDescriptor{ + SchemaVersion: 1, SessionID: "session-1", Generation: 7, OwnerPID: 42, + RingPath: filepath.Join(t.TempDir(), "media.ring"), SlotCount: 16, + SlotBytes: 8 * 1024 * 1024, VideoTransport: "shared_memory_encoded_access_units", + AudioTransport: "separate_transport", + }, + } + encoded, err := json.Marshal(response) + if err == nil { + encoded = append(encoded, '\n') + _, err = connection.Write(encoded) + } + serverError <- err + }() + connection, descriptor, err := openWindowsEncodedSession(listener.Addr().String(), "session-1", 60, true) + if err != nil { + t.Fatal(err) + } + defer connection.Close() + if descriptor.Generation != 7 || descriptor.SessionID != "session-1" { + t.Fatalf("unexpected descriptor: %+v", descriptor) + } + if err := <-serverError; err != nil { + t.Fatal(err) + } +} + +func TestWindowsAgentControlRejectsRemoteAndMismatchedDescriptor(t *testing.T) { + if validateAgentControlAddress("192.0.2.10:39501") == nil { + t.Fatal("remote Windows Agent control address was accepted") + } + descriptor := mediaRingDescriptor{ + SchemaVersion: 1, SessionID: "other", Generation: 1, OwnerPID: 1, + RingPath: filepath.Join(t.TempDir(), "media.ring"), SlotCount: 16, + SlotBytes: 1024, VideoTransport: "shared_memory_encoded_access_units", + AudioTransport: "separate_transport", + } + if validateMediaRingDescriptor(descriptor, "session-1") == nil { + t.Fatal("descriptor for another session was accepted") + } +} + +func TestWindowsAgentPipeAuthenticationMatchesRustWireVector(t *testing.T) { + key := make([]byte, 32) + nonce := make([]byte, 32) + for index := range key { + key[index] = byte(index) + nonce[index] = byte(index + 32) + } + actual := base64.RawURLEncoding.EncodeToString(windowsAgentPipeMAC(key, nonce)) + const expected = "KinBDOBvVhz0eRv37iahq8BO0z8QdiI0Qnx7HZmcxIk" + if actual != expected { + t.Fatalf("unexpected Agent Pipe MAC: %s", actual) + } + if validateAgentControlAddress(`\\.\pipe\RemoteDesk\windows-agent`) != nil { + t.Fatal("RemoteDesk Agent Pipe was rejected") + } + if validateAgentControlAddress(`\\.\pipe\Other\windows-agent`) == nil { + t.Fatal("Pipe outside the RemoteDesk namespace was accepted") + } +} + +type testControlError struct{ message string } + +func (e *testControlError) Error() string { return e.message } diff --git a/transport/hysteria2-agent/windows_agent_pipe_other.go b/transport/hysteria2-agent/windows_agent_pipe_other.go new file mode 100644 index 0000000..d0794aa --- /dev/null +++ b/transport/hysteria2-agent/windows_agent_pipe_other.go @@ -0,0 +1,13 @@ +//go:build !windows + +package main + +import ( + "errors" + "net" + "time" +) + +func dialWindowsAgentPipe(_ string, _ time.Duration) (net.Conn, error) { + return nil, errors.New("Windows Agent Named Pipe requires Windows") +} diff --git a/transport/hysteria2-agent/windows_agent_pipe_windows.go b/transport/hysteria2-agent/windows_agent_pipe_windows.go new file mode 100644 index 0000000..f16564f --- /dev/null +++ b/transport/hysteria2-agent/windows_agent_pipe_windows.go @@ -0,0 +1,17 @@ +//go:build windows + +package main + +import ( + "context" + "net" + "time" + + "github.com/Microsoft/go-winio" +) + +func dialWindowsAgentPipe(name string, timeout time.Duration) (net.Conn, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return winio.DialPipeContext(ctx, name) +}