Document all-Rust migration and extend native media stack
ci / rust (push) Canceled after 0s
ci / web (push) Canceled after 0s
ci / package-preview (push) Canceled after 0s
ci / package-installer (push) Canceled after 0s
ci / linux-agent (push) Canceled after 0s
ci / edge-service (push) Canceled after 0s
ci / coturn-pop (push) Canceled after 0s
ci / package-windows-host (push) Canceled after 0s
ci / rust (push) Canceled after 0s
ci / web (push) Canceled after 0s
ci / package-preview (push) Canceled after 0s
ci / package-installer (push) Canceled after 0s
ci / linux-agent (push) Canceled after 0s
ci / edge-service (push) Canceled after 0s
ci / coturn-pop (push) Canceled after 0s
ci / package-windows-host (push) Canceled after 0s
This commit is contained in:
Generated
+893
-52
File diff suppressed because it is too large
Load Diff
+2
-3
@@ -5,15 +5,14 @@ members = [
|
|||||||
"agent/agent-runtime",
|
"agent/agent-runtime",
|
||||||
"agent/windows-agent",
|
"agent/windows-agent",
|
||||||
"client/app-shell",
|
"client/app-shell",
|
||||||
|
"client/native-gui",
|
||||||
"client/crates/client-core",
|
"client/crates/client-core",
|
||||||
"client/helpers/control-service",
|
"client/helpers/control-service",
|
||||||
"client/helpers/credential-store",
|
"client/helpers/credential-store",
|
||||||
"client/helpers/linux-terminal",
|
|
||||||
"client/helpers/native-video",
|
"client/helpers/native-video",
|
||||||
|
"client/helpers/windows-agent-viewer",
|
||||||
"client/helpers/rdp-session",
|
"client/helpers/rdp-session",
|
||||||
"client/helpers/rdp-viewer",
|
"client/helpers/rdp-viewer",
|
||||||
"client/helpers/windows-agent-viewer",
|
|
||||||
"edge/edge-service",
|
|
||||||
"protocol",
|
"protocol",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# RemoteDesk
|
# RemoteDesk
|
||||||
|
|
||||||
|
> 架构决策(2026-08-14):所有发布运行时统一使用 Rust。Agent 是服务端;
|
||||||
|
> 原生 Client 使用 winit、wgpu 和 egui;两端使用 str0m WebRTC。Go、
|
||||||
|
> Hysteria2、Tauri、WebView、React 和软件兼容路径仅属于迁移期旧代码,
|
||||||
|
> 不属于目标架构。详见 [ADR 0001](docs/adr/0001-all-rust-runtime.md)。
|
||||||
|
|
||||||
RemoteDesk 是一套面向个人工作站的 Windows 远程桌面客户端。项目不包含 VDI、桌面池或多租户调度,重点是从 Windows 控制端连接 Windows 和现代 Linux 被控端。
|
RemoteDesk 是一套面向个人工作站的 Windows 远程桌面客户端。项目不包含 VDI、桌面池或多租户调度,重点是从 Windows 控制端连接 Windows 和现代 Linux 被控端。
|
||||||
|
|
||||||
## 产品定位
|
## 产品定位
|
||||||
|
|||||||
@@ -703,7 +703,7 @@ mod tests {
|
|||||||
|
|
||||||
let selected = select_encoder(
|
let selected = select_encoder(
|
||||||
DesktopBackend::XorgShmCompatibility,
|
DesktopBackend::XorgShmCompatibility,
|
||||||
ZeroCopyPolicy::Compatibility,
|
ZeroCopyPolicy::RequiredEndToEnd,
|
||||||
&capture,
|
&capture,
|
||||||
&candidates,
|
&candidates,
|
||||||
)
|
)
|
||||||
@@ -788,7 +788,7 @@ mod tests {
|
|||||||
|
|
||||||
let selected = select_encoder(
|
let selected = select_encoder(
|
||||||
DesktopBackend::WaylandPipeWire,
|
DesktopBackend::WaylandPipeWire,
|
||||||
ZeroCopyPolicy::Compatibility,
|
ZeroCopyPolicy::RequiredEndToEnd,
|
||||||
&capture,
|
&capture,
|
||||||
&candidates,
|
&candidates,
|
||||||
)
|
)
|
||||||
@@ -828,7 +828,7 @@ mod tests {
|
|||||||
|
|
||||||
let selected = select_encoder(
|
let selected = select_encoder(
|
||||||
DesktopBackend::WaylandPipeWire,
|
DesktopBackend::WaylandPipeWire,
|
||||||
ZeroCopyPolicy::Compatibility,
|
ZeroCopyPolicy::RequiredEndToEnd,
|
||||||
&capture,
|
&capture,
|
||||||
&candidates,
|
&candidates,
|
||||||
)
|
)
|
||||||
@@ -936,7 +936,7 @@ mod tests {
|
|||||||
transfer_mode: TransferMode::SameAdapterZeroCopy,
|
transfer_mode: TransferMode::SameAdapterZeroCopy,
|
||||||
};
|
};
|
||||||
let mut pipeline = VideoPipeline::active(
|
let mut pipeline = VideoPipeline::active(
|
||||||
ZeroCopyPolicy::Compatibility,
|
ZeroCopyPolicy::RequiredEndToEnd,
|
||||||
Resolution::new(1920, 1080).unwrap(),
|
Resolution::new(1920, 1080).unwrap(),
|
||||||
zero_copy,
|
zero_copy,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -989,7 +989,7 @@ mod tests {
|
|||||||
|
|
||||||
let plan = MultiDisplayCapturePlan::create(
|
let plan = MultiDisplayCapturePlan::create(
|
||||||
DesktopBackend::WaylandPipeWire,
|
DesktopBackend::WaylandPipeWire,
|
||||||
ZeroCopyPolicy::Compatibility,
|
ZeroCopyPolicy::RequiredEndToEnd,
|
||||||
&topology,
|
&topology,
|
||||||
&sources,
|
&sources,
|
||||||
&DisplaySelection::all(7),
|
&DisplaySelection::all(7),
|
||||||
@@ -1071,7 +1071,7 @@ mod tests {
|
|||||||
|
|
||||||
let plan = MultiDisplayCapturePlan::create(
|
let plan = MultiDisplayCapturePlan::create(
|
||||||
DesktopBackend::WaylandPipeWire,
|
DesktopBackend::WaylandPipeWire,
|
||||||
ZeroCopyPolicy::Compatibility,
|
ZeroCopyPolicy::RequiredEndToEnd,
|
||||||
&topology,
|
&topology,
|
||||||
&sources,
|
&sources,
|
||||||
&DisplaySelection::all(7),
|
&DisplaySelection::all(7),
|
||||||
@@ -1098,7 +1098,7 @@ mod tests {
|
|||||||
assert_eq!(
|
assert_eq!(
|
||||||
MultiDisplayCapturePlan::create(
|
MultiDisplayCapturePlan::create(
|
||||||
DesktopBackend::WaylandPipeWire,
|
DesktopBackend::WaylandPipeWire,
|
||||||
ZeroCopyPolicy::Compatibility,
|
ZeroCopyPolicy::RequiredEndToEnd,
|
||||||
&topology,
|
&topology,
|
||||||
&sources,
|
&sources,
|
||||||
&DisplaySelection::all(7),
|
&DisplaySelection::all(7),
|
||||||
|
|||||||
@@ -12,21 +12,43 @@ path = "src/main.rs"
|
|||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
anyhow = "1.0"
|
anyhow = "1.0"
|
||||||
|
base64 = "0.22"
|
||||||
clap = { version = "4.5", features = ["derive"] }
|
clap = { version = "4.5", features = ["derive"] }
|
||||||
flate2 = "1.1"
|
flate2 = "1.1"
|
||||||
|
getrandom = { version = "0.3", features = ["std"] }
|
||||||
|
hmac = "=0.13.0"
|
||||||
|
memmap2 = "0.9"
|
||||||
serde = { version = "1.0", features = ["derive"] }
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
|
sha2 = "=0.11.0"
|
||||||
tokio = { version = "1.47", features = ["macros", "net", "rt-multi-thread", "io-util", "sync", "time"] }
|
tokio = { version = "1.47", features = ["macros", "net", "rt-multi-thread", "io-util", "sync", "time"] }
|
||||||
|
zeroize = "1.8"
|
||||||
windows = { version = "0.62.2", features = [
|
windows = { version = "0.62.2", features = [
|
||||||
"Win32_Foundation",
|
"Win32_Foundation",
|
||||||
|
"Win32_Graphics_Direct3D",
|
||||||
"Win32_Graphics_Direct3D11",
|
"Win32_Graphics_Direct3D11",
|
||||||
"Win32_Graphics_Dxgi",
|
"Win32_Graphics_Dxgi",
|
||||||
"Win32_Graphics_Dxgi_Common",
|
"Win32_Graphics_Dxgi_Common",
|
||||||
"Win32_Graphics_Gdi",
|
"Win32_Graphics_Gdi",
|
||||||
|
"Win32_Media_MediaFoundation",
|
||||||
|
"Win32_System_Com",
|
||||||
|
"Win32_System_Ole",
|
||||||
|
"Win32_System_Variant",
|
||||||
"Win32_UI_Input_KeyboardAndMouse",
|
"Win32_UI_Input_KeyboardAndMouse",
|
||||||
"Win32_UI_WindowsAndMessaging",
|
"Win32_UI_WindowsAndMessaging",
|
||||||
] }
|
] }
|
||||||
|
|
||||||
|
[target.'cfg(windows)'.dependencies]
|
||||||
|
cpal = "0.17.3"
|
||||||
|
opus2 = { version = "0.4.0", features = ["bundled"] }
|
||||||
|
windows-sys = { version = "0.61.2", features = [
|
||||||
|
"Win32_Foundation",
|
||||||
|
"Win32_Security",
|
||||||
|
"Win32_Security_Authorization",
|
||||||
|
"Win32_Storage_FileSystem",
|
||||||
|
"Win32_System_Threading",
|
||||||
|
] }
|
||||||
|
|
||||||
[lints.rust]
|
[lints.rust]
|
||||||
unsafe_code = "allow"
|
unsafe_code = "allow"
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,471 @@
|
|||||||
|
//! Windows WASAPI loopback capture and fixed-duration Opus packetization.
|
||||||
|
//!
|
||||||
|
//! The capture callback is deliberately tiny: it only copies normalized f32
|
||||||
|
//! samples into a bounded queue. Resampling, Opus encoding, timestamps, and
|
||||||
|
//! discontinuity marking stay on the media worker thread.
|
||||||
|
|
||||||
|
#[cfg(windows)]
|
||||||
|
mod windows_audio {
|
||||||
|
use cpal::traits::{DeviceTrait as _, HostTrait as _, StreamTrait as _};
|
||||||
|
use opus2::{Application, Bitrate, Channels, Encoder};
|
||||||
|
use std::io;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
use std::sync::mpsc::{Receiver, SyncSender, TryRecvError, TrySendError, sync_channel};
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
const SAMPLE_RATE: u32 = 48_000;
|
||||||
|
const CHANNELS: usize = 2;
|
||||||
|
const FRAME_SAMPLES: usize = 960;
|
||||||
|
const MAX_OPUS_BYTES: usize = 4 * 1024;
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct CapturedAudioChunk {
|
||||||
|
pub(crate) samples: Vec<f32>,
|
||||||
|
pub(crate) capture_end_pts_100ns: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) struct AudioCapture {
|
||||||
|
receiver: Receiver<CapturedAudioChunk>,
|
||||||
|
pub(crate) source_rate: u32,
|
||||||
|
pub(crate) source_channels: usize,
|
||||||
|
pub(crate) discontinuity: Arc<AtomicBool>,
|
||||||
|
_stream: cpal::Stream,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AudioCapture {
|
||||||
|
pub(crate) fn new(session_started: Instant) -> io::Result<Self> {
|
||||||
|
let host = cpal::default_host();
|
||||||
|
let device = host.default_output_device().ok_or_else(|| {
|
||||||
|
io::Error::new(
|
||||||
|
io::ErrorKind::NotFound,
|
||||||
|
"WASAPI default output device unavailable",
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let supported = device
|
||||||
|
.default_output_config()
|
||||||
|
.map_err(|error| io::Error::other(error.to_string()))?;
|
||||||
|
let source_rate = supported.sample_rate();
|
||||||
|
let source_channels = usize::from(supported.channels());
|
||||||
|
if source_rate == 0 || source_channels == 0 || source_channels > 8 {
|
||||||
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::InvalidData,
|
||||||
|
"WASAPI loopback format is outside supported bounds",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let config = supported.config();
|
||||||
|
let (sender, receiver) = sync_channel(32);
|
||||||
|
let discontinuity = Arc::new(AtomicBool::new(false));
|
||||||
|
let callback_discontinuity = Arc::clone(&discontinuity);
|
||||||
|
let error_discontinuity = Arc::clone(&discontinuity);
|
||||||
|
let error_callback = move |_error: cpal::StreamError| {
|
||||||
|
error_discontinuity.store(true, Ordering::Release);
|
||||||
|
};
|
||||||
|
let stream = match supported.sample_format() {
|
||||||
|
cpal::SampleFormat::F32 => build_stream::<f32, _>(
|
||||||
|
&device,
|
||||||
|
&config,
|
||||||
|
sender,
|
||||||
|
callback_discontinuity,
|
||||||
|
session_started,
|
||||||
|
error_callback,
|
||||||
|
)?,
|
||||||
|
cpal::SampleFormat::I16 => build_stream::<i16, _>(
|
||||||
|
&device,
|
||||||
|
&config,
|
||||||
|
sender,
|
||||||
|
callback_discontinuity,
|
||||||
|
session_started,
|
||||||
|
error_callback,
|
||||||
|
)?,
|
||||||
|
cpal::SampleFormat::I32 => build_stream::<i32, _>(
|
||||||
|
&device,
|
||||||
|
&config,
|
||||||
|
sender,
|
||||||
|
callback_discontinuity,
|
||||||
|
session_started,
|
||||||
|
error_callback,
|
||||||
|
)?,
|
||||||
|
cpal::SampleFormat::U16 => build_stream::<u16, _>(
|
||||||
|
&device,
|
||||||
|
&config,
|
||||||
|
sender,
|
||||||
|
callback_discontinuity,
|
||||||
|
session_started,
|
||||||
|
error_callback,
|
||||||
|
)?,
|
||||||
|
cpal::SampleFormat::F64 => build_stream::<f64, _>(
|
||||||
|
&device,
|
||||||
|
&config,
|
||||||
|
sender,
|
||||||
|
callback_discontinuity,
|
||||||
|
session_started,
|
||||||
|
error_callback,
|
||||||
|
)?,
|
||||||
|
_ => {
|
||||||
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::Unsupported,
|
||||||
|
"WASAPI loopback sample format is unsupported",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
stream
|
||||||
|
.play()
|
||||||
|
.map_err(|error| io::Error::other(error.to_string()))?;
|
||||||
|
Ok(Self {
|
||||||
|
receiver,
|
||||||
|
source_rate,
|
||||||
|
source_channels,
|
||||||
|
discontinuity,
|
||||||
|
_stream: stream,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn try_receive(&self) -> Result<Option<CapturedAudioChunk>, TryRecvError> {
|
||||||
|
self.receiver.try_recv().map(Some)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn try_receive_latest(
|
||||||
|
&self,
|
||||||
|
) -> Result<Option<CapturedAudioChunk>, TryRecvError> {
|
||||||
|
let mut latest = self.receiver.try_recv()?;
|
||||||
|
loop {
|
||||||
|
match self.receiver.try_recv() {
|
||||||
|
Ok(chunk) => latest = chunk,
|
||||||
|
Err(TryRecvError::Empty) => return Ok(Some(latest)),
|
||||||
|
Err(TryRecvError::Disconnected) => return Err(TryRecvError::Disconnected),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_stream<T, E>(
|
||||||
|
device: &cpal::Device,
|
||||||
|
config: &cpal::StreamConfig,
|
||||||
|
sender: SyncSender<CapturedAudioChunk>,
|
||||||
|
discontinuity: Arc<AtomicBool>,
|
||||||
|
session_started: Instant,
|
||||||
|
error_callback: E,
|
||||||
|
) -> io::Result<cpal::Stream>
|
||||||
|
where
|
||||||
|
T: cpal::SizedSample + cpal::Sample + Send + 'static,
|
||||||
|
f32: cpal::FromSample<T>,
|
||||||
|
E: FnMut(cpal::StreamError) + Send + 'static,
|
||||||
|
{
|
||||||
|
let callback = move |data: &[T], _info: &cpal::InputCallbackInfo| {
|
||||||
|
let samples = data
|
||||||
|
.iter()
|
||||||
|
.map(|sample| sample.to_sample::<f32>())
|
||||||
|
.collect();
|
||||||
|
let capture_end_pts_100ns =
|
||||||
|
duration_to_100ns(Instant::now().saturating_duration_since(session_started));
|
||||||
|
match sender.try_send(CapturedAudioChunk {
|
||||||
|
samples,
|
||||||
|
capture_end_pts_100ns,
|
||||||
|
}) {
|
||||||
|
Ok(()) => {}
|
||||||
|
Err(TrySendError::Full(_) | TrySendError::Disconnected(_)) => {
|
||||||
|
discontinuity.store(true, Ordering::Release);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
device
|
||||||
|
.build_input_stream(config, callback, error_callback, None)
|
||||||
|
.map_err(|error| io::Error::other(error.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) struct AudioPipeline {
|
||||||
|
resampler: Resampler,
|
||||||
|
encoder: Encoder,
|
||||||
|
pending: Vec<f32>,
|
||||||
|
sample_count: u64,
|
||||||
|
sequence: u64,
|
||||||
|
discontinuity: bool,
|
||||||
|
timeline_base_pts_100ns: Option<u64>,
|
||||||
|
needs_reanchor: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) struct EncodedAudioPacket {
|
||||||
|
pub(crate) sequence: u64,
|
||||||
|
pub(crate) pts_100ns: u64,
|
||||||
|
pub(crate) discontinuity: bool,
|
||||||
|
pub(crate) data: Vec<u8>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AudioPipeline {
|
||||||
|
pub(crate) fn new(source_rate: u32, source_channels: usize) -> io::Result<Self> {
|
||||||
|
let mut encoder = Encoder::new(SAMPLE_RATE, Channels::Stereo, Application::Audio)
|
||||||
|
.map_err(|error| io::Error::other(error.to_string()))?;
|
||||||
|
encoder
|
||||||
|
.set_bitrate(Bitrate::Bits(96_000))
|
||||||
|
.map_err(|error| io::Error::other(error.to_string()))?;
|
||||||
|
encoder
|
||||||
|
.set_inband_fec(true)
|
||||||
|
.map_err(|error| io::Error::other(error.to_string()))?;
|
||||||
|
encoder
|
||||||
|
.set_packet_loss_perc(5)
|
||||||
|
.map_err(|error| io::Error::other(error.to_string()))?;
|
||||||
|
encoder
|
||||||
|
.set_dtx(true)
|
||||||
|
.map_err(|error| io::Error::other(error.to_string()))?;
|
||||||
|
Ok(Self {
|
||||||
|
resampler: Resampler::new(source_rate, source_channels)?,
|
||||||
|
encoder,
|
||||||
|
pending: Vec::with_capacity(FRAME_SAMPLES * CHANNELS * 2),
|
||||||
|
sample_count: 0,
|
||||||
|
sequence: 1,
|
||||||
|
discontinuity: false,
|
||||||
|
timeline_base_pts_100ns: None,
|
||||||
|
needs_reanchor: true,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn mark_discontinuity(&mut self) {
|
||||||
|
self.discontinuity = true;
|
||||||
|
self.needs_reanchor = true;
|
||||||
|
self.pending.clear();
|
||||||
|
self.resampler.reset();
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn push_source(
|
||||||
|
&mut self,
|
||||||
|
samples: &[f32],
|
||||||
|
capture_end_pts_100ns: u64,
|
||||||
|
) -> io::Result<Vec<EncodedAudioPacket>> {
|
||||||
|
if !samples.len().is_multiple_of(self.resampler.source_channels) {
|
||||||
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::InvalidData,
|
||||||
|
"audio callback frame is incomplete",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if self.needs_reanchor {
|
||||||
|
let source_frames = samples.len() / self.resampler.source_channels;
|
||||||
|
let source_duration_100ns = u64::try_from(source_frames)
|
||||||
|
.unwrap_or(u64::MAX)
|
||||||
|
.saturating_mul(10_000_000)
|
||||||
|
/ u64::from(self.resampler.source_rate);
|
||||||
|
self.timeline_base_pts_100ns =
|
||||||
|
Some(capture_end_pts_100ns.saturating_sub(source_duration_100ns));
|
||||||
|
self.sample_count = 0;
|
||||||
|
self.needs_reanchor = false;
|
||||||
|
}
|
||||||
|
self.pending.extend(self.resampler.push(samples)?);
|
||||||
|
let frame_width = FRAME_SAMPLES * CHANNELS;
|
||||||
|
let mut packets = Vec::new();
|
||||||
|
while self.pending.len() >= frame_width {
|
||||||
|
let frame = self.pending.drain(..frame_width).collect::<Vec<_>>();
|
||||||
|
let data = self
|
||||||
|
.encoder
|
||||||
|
.encode_vec_float(&frame, MAX_OPUS_BYTES)
|
||||||
|
.map_err(|error| io::Error::other(error.to_string()))?;
|
||||||
|
let packet = EncodedAudioPacket {
|
||||||
|
sequence: self.sequence,
|
||||||
|
pts_100ns: self
|
||||||
|
.timeline_base_pts_100ns
|
||||||
|
.unwrap_or(capture_end_pts_100ns)
|
||||||
|
.saturating_add(
|
||||||
|
self.sample_count.saturating_mul(10_000_000) / u64::from(SAMPLE_RATE),
|
||||||
|
),
|
||||||
|
discontinuity: self.discontinuity,
|
||||||
|
data,
|
||||||
|
};
|
||||||
|
self.sequence = self.sequence.saturating_add(1);
|
||||||
|
self.sample_count = self.sample_count.saturating_add(FRAME_SAMPLES as u64);
|
||||||
|
self.discontinuity = false;
|
||||||
|
packets.push(packet);
|
||||||
|
}
|
||||||
|
Ok(packets)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Resampler {
|
||||||
|
source_rate: u32,
|
||||||
|
source_channels: usize,
|
||||||
|
buffer: Vec<f32>,
|
||||||
|
position_numerator: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Resampler {
|
||||||
|
fn new(source_rate: u32, source_channels: usize) -> io::Result<Self> {
|
||||||
|
if source_rate == 0 || source_channels == 0 || source_channels > 8 {
|
||||||
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::InvalidInput,
|
||||||
|
"invalid source audio format",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(Self {
|
||||||
|
source_rate,
|
||||||
|
source_channels,
|
||||||
|
buffer: Vec::new(),
|
||||||
|
position_numerator: 0,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push(&mut self, samples: &[f32]) -> io::Result<Vec<f32>> {
|
||||||
|
if !samples.len().is_multiple_of(self.source_channels) {
|
||||||
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::InvalidData,
|
||||||
|
"audio callback frame is incomplete",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
self.buffer.extend_from_slice(samples);
|
||||||
|
let source_frames = self.buffer.len() / self.source_channels;
|
||||||
|
let source_frames_u64 = u64::try_from(source_frames).map_err(|_| {
|
||||||
|
io::Error::new(io::ErrorKind::InvalidData, "audio callback is too large")
|
||||||
|
})?;
|
||||||
|
let mut output = Vec::new();
|
||||||
|
while self.position_numerator / u64::from(SAMPLE_RATE) < source_frames_u64 {
|
||||||
|
let index_u64 = self.position_numerator / u64::from(SAMPLE_RATE);
|
||||||
|
let index = usize::try_from(index_u64).map_err(|_| {
|
||||||
|
io::Error::new(io::ErrorKind::InvalidData, "audio position overflow")
|
||||||
|
})?;
|
||||||
|
let remainder = self.position_numerator % u64::from(SAMPLE_RATE);
|
||||||
|
if remainder != 0 && index + 1 >= source_frames {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let fraction = f32::from(u16::try_from(remainder).map_err(|_| {
|
||||||
|
io::Error::new(io::ErrorKind::InvalidData, "audio fraction overflow")
|
||||||
|
})?) / 48_000.0;
|
||||||
|
for channel in 0..CHANNELS {
|
||||||
|
let source_channel = if self.source_channels == 1 {
|
||||||
|
0
|
||||||
|
} else {
|
||||||
|
channel.min(self.source_channels - 1)
|
||||||
|
};
|
||||||
|
let first = self.buffer[index * self.source_channels + source_channel];
|
||||||
|
let second = if index + 1 < source_frames {
|
||||||
|
self.buffer[(index + 1) * self.source_channels + source_channel]
|
||||||
|
} else {
|
||||||
|
first
|
||||||
|
};
|
||||||
|
output.push(first + (second - first) * fraction);
|
||||||
|
}
|
||||||
|
self.position_numerator = self
|
||||||
|
.position_numerator
|
||||||
|
.saturating_add(u64::from(self.source_rate));
|
||||||
|
}
|
||||||
|
let consumed = usize::try_from(self.position_numerator / u64::from(SAMPLE_RATE))
|
||||||
|
.unwrap_or(source_frames);
|
||||||
|
if consumed > 0 {
|
||||||
|
let remove = consumed.min(source_frames);
|
||||||
|
self.buffer.drain(..remove * self.source_channels);
|
||||||
|
self.position_numerator = self.position_numerator.saturating_sub(
|
||||||
|
u64::try_from(remove)
|
||||||
|
.unwrap_or(u64::MAX)
|
||||||
|
.saturating_mul(u64::from(SAMPLE_RATE)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(output)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reset(&mut self) {
|
||||||
|
self.buffer.clear();
|
||||||
|
self.position_numerator = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn duration_to_100ns(duration: std::time::Duration) -> u64 {
|
||||||
|
u64::try_from(duration.as_nanos() / 100).unwrap_or(u64::MAX)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resampler_preserves_48khz_stereo_frames() {
|
||||||
|
let input = vec![0.125_f32; FRAME_SAMPLES * 2];
|
||||||
|
let output = Resampler::new(48_000, 2).unwrap().push(&input).unwrap();
|
||||||
|
assert_eq!(output, input);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resampler_duplicates_mono_and_downsamples_96khz() {
|
||||||
|
let mono = vec![0.25_f32; 960];
|
||||||
|
let mono_output = Resampler::new(48_000, 1).unwrap().push(&mono).unwrap();
|
||||||
|
assert_eq!(mono_output.len(), 960 * 2);
|
||||||
|
assert!(
|
||||||
|
mono_output
|
||||||
|
.chunks_exact(2)
|
||||||
|
.all(|frame| frame == [0.25, 0.25])
|
||||||
|
);
|
||||||
|
|
||||||
|
let stereo_96khz = vec![0.5_f32; 960 * 2];
|
||||||
|
let downsampled = Resampler::new(96_000, 2)
|
||||||
|
.unwrap()
|
||||||
|
.push(&stereo_96khz)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(downsampled.len(), 480 * 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resampler_keeps_44100hz_ratio_across_callback_boundaries() {
|
||||||
|
let mut resampler = Resampler::new(44_100, 2).unwrap();
|
||||||
|
let mut output_frames = 0;
|
||||||
|
for _ in 0..10 {
|
||||||
|
output_frames += resampler.push(&vec![0.0; 441 * 2]).unwrap().len() / 2;
|
||||||
|
}
|
||||||
|
assert!((4_799..=4_800).contains(&output_frames));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn opus_pipeline_emits_bounded_monotonic_twenty_ms_packets() {
|
||||||
|
let mut pipeline = AudioPipeline::new(48_000, 2).unwrap();
|
||||||
|
let first = pipeline
|
||||||
|
.push_source(&vec![0.0; FRAME_SAMPLES * 2], 1_000_000)
|
||||||
|
.unwrap();
|
||||||
|
let second = pipeline
|
||||||
|
.push_source(&vec![0.0; FRAME_SAMPLES * 2], 1_200_000)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(first.len(), 1);
|
||||||
|
assert_eq!(second.len(), 1);
|
||||||
|
assert_eq!(first[0].sequence, 1);
|
||||||
|
assert_eq!(first[0].pts_100ns, 800_000);
|
||||||
|
assert_eq!(second[0].sequence, 2);
|
||||||
|
assert_eq!(second[0].pts_100ns, 1_000_000);
|
||||||
|
assert!(!first[0].data.is_empty() && first[0].data.len() <= MAX_OPUS_BYTES);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn discontinuity_is_applied_to_exactly_one_packet() {
|
||||||
|
let mut pipeline = AudioPipeline::new(48_000, 2).unwrap();
|
||||||
|
pipeline.mark_discontinuity();
|
||||||
|
let first = pipeline
|
||||||
|
.push_source(&vec![0.0; FRAME_SAMPLES * 2], 2_000_000)
|
||||||
|
.unwrap();
|
||||||
|
let second = pipeline
|
||||||
|
.push_source(&vec![0.0; FRAME_SAMPLES * 2], 2_200_000)
|
||||||
|
.unwrap();
|
||||||
|
assert!(first[0].discontinuity);
|
||||||
|
assert!(!second[0].discontinuity);
|
||||||
|
assert_eq!(first[0].pts_100ns, 1_800_000);
|
||||||
|
assert_eq!(second[0].pts_100ns, 2_000_000);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn discontinuity_reanchors_pts_without_resetting_sequence() {
|
||||||
|
let mut pipeline = AudioPipeline::new(48_000, 2).unwrap();
|
||||||
|
let first = pipeline
|
||||||
|
.push_source(&vec![0.0; FRAME_SAMPLES * 2], 200_000)
|
||||||
|
.unwrap();
|
||||||
|
pipeline.mark_discontinuity();
|
||||||
|
let resumed = pipeline
|
||||||
|
.push_source(&vec![0.0; FRAME_SAMPLES * 2], 5_200_000)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(first[0].sequence, 1);
|
||||||
|
assert_eq!(first[0].pts_100ns, 0);
|
||||||
|
assert_eq!(resumed[0].sequence, 2);
|
||||||
|
assert_eq!(resumed[0].pts_100ns, 5_000_000);
|
||||||
|
assert!(resumed[0].discontinuity);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(windows)]
|
||||||
|
pub(crate) use windows_audio::{AudioCapture, AudioPipeline, CapturedAudioChunk};
|
||||||
|
|
||||||
|
#[cfg(not(windows))]
|
||||||
|
pub(crate) struct AudioCapture;
|
||||||
|
|
||||||
|
#[cfg(not(windows))]
|
||||||
|
pub(crate) struct AudioPipeline;
|
||||||
@@ -0,0 +1,166 @@
|
|||||||
|
//! Runtime capability probes. A registry/API presence check is not treated as
|
||||||
|
//! an active zero-copy encoder; it only reports whether Windows exposes an
|
||||||
|
//! approved hardware MFT that can be probed by the session worker.
|
||||||
|
|
||||||
|
use std::sync::OnceLock;
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Default)]
|
||||||
|
pub(crate) struct CapabilitySnapshot {
|
||||||
|
pub(crate) desktop_duplication: bool,
|
||||||
|
pub(crate) hardware_h264_mft: bool,
|
||||||
|
pub(crate) hardware_h264_encoder_verified: bool,
|
||||||
|
pub(crate) hardware_hevc_mft: bool,
|
||||||
|
pub(crate) hardware_av1_mft: bool,
|
||||||
|
pub(crate) hardware_pipeline_verified: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CapabilitySnapshot {
|
||||||
|
#[must_use]
|
||||||
|
pub(crate) const fn capture_backend(&self) -> &'static str {
|
||||||
|
if self.desktop_duplication {
|
||||||
|
"desktop-duplication-compatibility"
|
||||||
|
} else {
|
||||||
|
"gdi-bgra-zlib"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
pub(crate) const fn pipeline_state(&self) -> &'static str {
|
||||||
|
if self.hardware_pipeline_verified {
|
||||||
|
"hardware_pipeline_verified"
|
||||||
|
} else if self.desktop_duplication
|
||||||
|
|| self.hardware_h264_mft
|
||||||
|
|| self.hardware_hevc_mft
|
||||||
|
|| self.hardware_av1_mft
|
||||||
|
{
|
||||||
|
"hardware_probe_partial"
|
||||||
|
} else {
|
||||||
|
"hardware_probe_failed"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[must_use]
|
||||||
|
pub(crate) fn encoder_available(&self, codec: &str) -> bool {
|
||||||
|
match codec {
|
||||||
|
"h264" => self.hardware_h264_mft,
|
||||||
|
"hevc" => self.hardware_hevc_mft,
|
||||||
|
"av1" => self.hardware_av1_mft,
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static SNAPSHOT: OnceLock<CapabilitySnapshot> = OnceLock::new();
|
||||||
|
|
||||||
|
pub(crate) fn snapshot() -> &'static CapabilitySnapshot {
|
||||||
|
SNAPSHOT.get_or_init(probe)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(windows)]
|
||||||
|
pub(crate) fn probe() -> CapabilitySnapshot {
|
||||||
|
CapabilitySnapshot {
|
||||||
|
desktop_duplication: crate::capture::DdaCapture::new().is_ok(),
|
||||||
|
hardware_h264_mft: probe_hardware_encoder(
|
||||||
|
windows::Win32::Media::MediaFoundation::MFVideoFormat_H264,
|
||||||
|
),
|
||||||
|
hardware_h264_encoder_verified: probe_hardware_h264_encoder(),
|
||||||
|
hardware_hevc_mft: probe_hardware_encoder(
|
||||||
|
windows::Win32::Media::MediaFoundation::MFVideoFormat_HEVC,
|
||||||
|
),
|
||||||
|
hardware_av1_mft: probe_hardware_encoder(
|
||||||
|
windows::Win32::Media::MediaFoundation::MFVideoFormat_AV1,
|
||||||
|
),
|
||||||
|
hardware_pipeline_verified: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(windows))]
|
||||||
|
pub(crate) const fn probe() -> CapabilitySnapshot {
|
||||||
|
CapabilitySnapshot {
|
||||||
|
desktop_duplication: false,
|
||||||
|
hardware_h264_mft: false,
|
||||||
|
hardware_h264_encoder_verified: false,
|
||||||
|
hardware_hevc_mft: false,
|
||||||
|
hardware_av1_mft: false,
|
||||||
|
hardware_pipeline_verified: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(windows)]
|
||||||
|
fn probe_hardware_h264_encoder() -> bool {
|
||||||
|
let Ok(capture) = crate::capture::DdaCapture::new() else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let (width, height) = capture.dimensions();
|
||||||
|
if width < 2 || height < 2 || width % 2 != 0 || height % 2 != 0 {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let Ok(mut encoder) = crate::encoder::H264Encoder::new(width, height, 30, true) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let Some(bytes) = usize::try_from(width)
|
||||||
|
.ok()
|
||||||
|
.and_then(|w| usize::try_from(height).ok().and_then(|h| w.checked_mul(h)))
|
||||||
|
.and_then(|pixels| pixels.checked_mul(4))
|
||||||
|
else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let bgra = vec![0_u8; bytes];
|
||||||
|
let Ok(nv12) = crate::encoder::bgra_to_nv12(width, height, &bgra) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
for _ in 0..3 {
|
||||||
|
if encoder.encode(&nv12).is_err() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(windows)]
|
||||||
|
fn probe_hardware_encoder(codec: windows::core::GUID) -> bool {
|
||||||
|
use windows::Win32::Media::MediaFoundation::{
|
||||||
|
MF_VERSION, MFSTARTUP_FULL, MFStartup, MFT_CATEGORY_VIDEO_ENCODER, MFT_ENUM_FLAG_HARDWARE,
|
||||||
|
MFT_ENUM_FLAG_SORTANDFILTER_APPROVED_ONLY, MFT_REGISTER_TYPE_INFO, MFTEnumEx,
|
||||||
|
MFVideoFormat_NV12,
|
||||||
|
};
|
||||||
|
use windows::Win32::System::Com::{COINIT_MULTITHREADED, CoInitializeEx, CoTaskMemFree};
|
||||||
|
|
||||||
|
unsafe {
|
||||||
|
if CoInitializeEx(None, COINIT_MULTITHREADED).is_err()
|
||||||
|
|| MFStartup(MF_VERSION, MFSTARTUP_FULL).is_err()
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let input = MFT_REGISTER_TYPE_INFO {
|
||||||
|
guidMajorType: windows::Win32::Media::MediaFoundation::MFMediaType_Video,
|
||||||
|
guidSubtype: MFVideoFormat_NV12,
|
||||||
|
};
|
||||||
|
let output = MFT_REGISTER_TYPE_INFO {
|
||||||
|
guidMajorType: windows::Win32::Media::MediaFoundation::MFMediaType_Video,
|
||||||
|
guidSubtype: codec,
|
||||||
|
};
|
||||||
|
let mut activations: *mut Option<windows::Win32::Media::MediaFoundation::IMFActivate> =
|
||||||
|
std::ptr::null_mut();
|
||||||
|
let mut count = 0_u32;
|
||||||
|
let result = MFTEnumEx(
|
||||||
|
MFT_CATEGORY_VIDEO_ENCODER,
|
||||||
|
MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_SORTANDFILTER_APPROVED_ONLY,
|
||||||
|
Some(&raw const input),
|
||||||
|
Some(&raw const output),
|
||||||
|
&raw mut activations,
|
||||||
|
&raw mut count,
|
||||||
|
)
|
||||||
|
.is_ok()
|
||||||
|
&& count > 0;
|
||||||
|
if !activations.is_null() {
|
||||||
|
let entries = std::slice::from_raw_parts_mut(activations, count as usize);
|
||||||
|
for entry in entries {
|
||||||
|
*entry = None;
|
||||||
|
}
|
||||||
|
CoTaskMemFree(Some(activations.cast()));
|
||||||
|
}
|
||||||
|
let _ = windows::Win32::Media::MediaFoundation::MFShutdown();
|
||||||
|
result
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,302 @@
|
|||||||
|
//! Windows Desktop Duplication compatibility capture.
|
||||||
|
//!
|
||||||
|
//! The staging/readback path is deliberately reported as software/cpu_upload.
|
||||||
|
//! It is a compatibility path, not an end-to-end GPU zero-copy pipeline. An
|
||||||
|
//! IDD swap-chain consumer can replace this module without changing the agent
|
||||||
|
//! control protocol or frame envelope.
|
||||||
|
|
||||||
|
#[cfg(windows)]
|
||||||
|
mod windows_capture {
|
||||||
|
use anyhow::Context as _;
|
||||||
|
use flate2::{Compression, write::ZlibEncoder};
|
||||||
|
use std::io::Write as _;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
use windows::Win32::Graphics::Direct3D::D3D_DRIVER_TYPE_UNKNOWN;
|
||||||
|
use windows::Win32::Graphics::Direct3D11::{
|
||||||
|
D3D11_BIND_FLAG, D3D11_CPU_ACCESS_READ, D3D11_CREATE_DEVICE_FLAG, D3D11_MAP_READ,
|
||||||
|
D3D11_RESOURCE_MISC_FLAG, D3D11_TEXTURE2D_DESC, D3D11_USAGE_STAGING, D3D11CreateDevice,
|
||||||
|
ID3D11DeviceContext, ID3D11Texture2D,
|
||||||
|
};
|
||||||
|
use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_SAMPLE_DESC};
|
||||||
|
use windows::Win32::Graphics::Dxgi::{
|
||||||
|
CreateDXGIFactory1, DXGI_ERROR_WAIT_TIMEOUT, DXGI_OUTDUPL_FRAME_INFO, IDXGIAdapter1,
|
||||||
|
IDXGIOutput1, IDXGIOutputDuplication, IDXGIResource,
|
||||||
|
};
|
||||||
|
use windows::core::Interface as _;
|
||||||
|
|
||||||
|
const FRAME_MAGIC: &[u8; 4] = b"RDWF";
|
||||||
|
const FRAME_VERSION: u8 = 2;
|
||||||
|
const FRAME_HEADER_LEN: usize = 40;
|
||||||
|
const CODEC_BGRA_ZLIB: u8 = 1;
|
||||||
|
const MAX_DIMENSION: u32 = 16_384;
|
||||||
|
const MAX_RAW_FRAME_BYTES: usize = 256 * 1024 * 1024;
|
||||||
|
|
||||||
|
pub struct EncodedFrame {
|
||||||
|
pub bytes: Vec<u8>,
|
||||||
|
pub capture_latency: Duration,
|
||||||
|
pub encode_latency: Duration,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct CapturedFrame {
|
||||||
|
pub width: u32,
|
||||||
|
pub height: u32,
|
||||||
|
pub pixels: Vec<u8>,
|
||||||
|
pub capture_latency: Duration,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct DdaCapture {
|
||||||
|
context: ID3D11DeviceContext,
|
||||||
|
duplication: IDXGIOutputDuplication,
|
||||||
|
staging: ID3D11Texture2D,
|
||||||
|
width: u32,
|
||||||
|
height: u32,
|
||||||
|
pixels: Vec<u8>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DdaCapture {
|
||||||
|
pub fn dimensions(&self) -> (u32, u32) {
|
||||||
|
(self.width, self.height)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn new() -> anyhow::Result<Self> {
|
||||||
|
unsafe {
|
||||||
|
let factory = CreateDXGIFactory1::<windows::Win32::Graphics::Dxgi::IDXGIFactory1>()
|
||||||
|
.context("CreateDXGIFactory1 failed")?;
|
||||||
|
let mut selected: Option<(
|
||||||
|
IDXGIAdapter1,
|
||||||
|
windows::Win32::Graphics::Dxgi::IDXGIOutput,
|
||||||
|
)> = None;
|
||||||
|
for adapter_index in 0..32 {
|
||||||
|
let Ok(adapter) = factory.EnumAdapters1(adapter_index) else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
for output_index in 0..16 {
|
||||||
|
let Ok(output) = adapter.EnumOutputs(output_index) else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
let description =
|
||||||
|
output.GetDesc().context("IDXGIOutput::GetDesc failed")?;
|
||||||
|
if description.AttachedToDesktop.as_bool() {
|
||||||
|
selected = Some((adapter, output));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if selected.is_some() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let (adapter, output) =
|
||||||
|
selected.ok_or_else(|| anyhow::anyhow!("no attached DXGI output"))?;
|
||||||
|
let mut device = None;
|
||||||
|
let mut context = None;
|
||||||
|
D3D11CreateDevice(
|
||||||
|
&adapter,
|
||||||
|
D3D_DRIVER_TYPE_UNKNOWN,
|
||||||
|
windows::Win32::Foundation::HMODULE::default(),
|
||||||
|
D3D11_CREATE_DEVICE_FLAG(0),
|
||||||
|
None,
|
||||||
|
7,
|
||||||
|
Some(&raw mut device),
|
||||||
|
None,
|
||||||
|
Some(&raw mut context),
|
||||||
|
)
|
||||||
|
.context("D3D11CreateDevice failed")?;
|
||||||
|
let device = device.ok_or_else(|| anyhow::anyhow!("D3D11 returned no device"))?;
|
||||||
|
let context =
|
||||||
|
context.ok_or_else(|| anyhow::anyhow!("D3D11 returned no context"))?;
|
||||||
|
let output1: IDXGIOutput1 =
|
||||||
|
output.cast().context("DXGI output has no IDXGIOutput1")?;
|
||||||
|
let duplication = output1
|
||||||
|
.DuplicateOutput(&device)
|
||||||
|
.context("DuplicateOutput failed")?;
|
||||||
|
let desc = duplication.GetDesc();
|
||||||
|
let width = desc.ModeDesc.Width;
|
||||||
|
let height = desc.ModeDesc.Height;
|
||||||
|
validate_geometry(width, height)?;
|
||||||
|
let texture_desc = D3D11_TEXTURE2D_DESC {
|
||||||
|
Width: width,
|
||||||
|
Height: height,
|
||||||
|
MipLevels: 1,
|
||||||
|
ArraySize: 1,
|
||||||
|
Format: DXGI_FORMAT_B8G8R8A8_UNORM,
|
||||||
|
SampleDesc: DXGI_SAMPLE_DESC {
|
||||||
|
Count: 1,
|
||||||
|
Quality: 0,
|
||||||
|
},
|
||||||
|
Usage: D3D11_USAGE_STAGING,
|
||||||
|
BindFlags: D3D11_BIND_FLAG(0).0 as u32,
|
||||||
|
CPUAccessFlags: D3D11_CPU_ACCESS_READ.0 as u32,
|
||||||
|
MiscFlags: D3D11_RESOURCE_MISC_FLAG(0).0 as u32,
|
||||||
|
};
|
||||||
|
let mut staging = None;
|
||||||
|
device
|
||||||
|
.CreateTexture2D(&texture_desc, None, Some(&raw mut staging))
|
||||||
|
.context("CreateTexture2D staging resource failed")?;
|
||||||
|
let staging =
|
||||||
|
staging.ok_or_else(|| anyhow::anyhow!("D3D11 returned no staging texture"))?;
|
||||||
|
let bytes = usize::try_from(width)
|
||||||
|
.ok()
|
||||||
|
.and_then(|w| usize::try_from(height).ok().and_then(|h| w.checked_mul(h)))
|
||||||
|
.and_then(|pixels| pixels.checked_mul(4))
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("desktop frame size overflow"))?;
|
||||||
|
Ok(Self {
|
||||||
|
context,
|
||||||
|
duplication,
|
||||||
|
staging,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
pixels: vec![0; bytes],
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn capture_frame(&mut self) -> anyhow::Result<EncodedFrame> {
|
||||||
|
let captured = self.capture_bgra_frame()?;
|
||||||
|
let encode_started = Instant::now();
|
||||||
|
let bytes = encode_frame(
|
||||||
|
captured.width,
|
||||||
|
captured.height,
|
||||||
|
&captured.pixels,
|
||||||
|
captured.capture_latency,
|
||||||
|
Duration::ZERO,
|
||||||
|
)?;
|
||||||
|
Ok(EncodedFrame {
|
||||||
|
bytes,
|
||||||
|
capture_latency: captured.capture_latency,
|
||||||
|
encode_latency: encode_started.elapsed(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn capture_bgra_frame(&mut self) -> anyhow::Result<CapturedFrame> {
|
||||||
|
let capture_started = Instant::now();
|
||||||
|
let mut info = DXGI_OUTDUPL_FRAME_INFO::default();
|
||||||
|
let mut resource: Option<IDXGIResource> = None;
|
||||||
|
unsafe {
|
||||||
|
if let Err(error) =
|
||||||
|
self.duplication
|
||||||
|
.AcquireNextFrame(100, &raw mut info, &raw mut resource)
|
||||||
|
{
|
||||||
|
if error.code() == DXGI_ERROR_WAIT_TIMEOUT.into() {
|
||||||
|
anyhow::bail!("dda_timeout");
|
||||||
|
}
|
||||||
|
return Err(error).context("AcquireNextFrame failed");
|
||||||
|
}
|
||||||
|
let mut mapped = false;
|
||||||
|
let result = (|| {
|
||||||
|
let resource = resource
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("DDA returned no frame resource"))?;
|
||||||
|
let source: ID3D11Texture2D = resource
|
||||||
|
.cast()
|
||||||
|
.context("DDA resource is not a D3D11 texture")?;
|
||||||
|
self.context.CopyResource(&self.staging, &source);
|
||||||
|
let mut mapped_resource =
|
||||||
|
windows::Win32::Graphics::Direct3D11::D3D11_MAPPED_SUBRESOURCE::default();
|
||||||
|
self.context
|
||||||
|
.Map(
|
||||||
|
&self.staging,
|
||||||
|
0,
|
||||||
|
D3D11_MAP_READ,
|
||||||
|
0,
|
||||||
|
Some(&raw mut mapped_resource),
|
||||||
|
)
|
||||||
|
.context("Map staging texture failed")?;
|
||||||
|
mapped = true;
|
||||||
|
let row_bytes = usize::try_from(self.width).unwrap() * 4;
|
||||||
|
let pitch = usize::try_from(mapped_resource.RowPitch)
|
||||||
|
.context("invalid DDA row pitch")?;
|
||||||
|
anyhow::ensure!(pitch >= row_bytes, "DDA row pitch is shorter than a row");
|
||||||
|
anyhow::ensure!(
|
||||||
|
!mapped_resource.pData.is_null(),
|
||||||
|
"DDA map returned a null pointer"
|
||||||
|
);
|
||||||
|
let source_ptr = mapped_resource.pData.cast::<u8>();
|
||||||
|
for row in 0..usize::try_from(self.height).unwrap() {
|
||||||
|
let source_row =
|
||||||
|
std::slice::from_raw_parts(source_ptr.add(row * pitch), row_bytes);
|
||||||
|
self.pixels[row * row_bytes..(row + 1) * row_bytes]
|
||||||
|
.copy_from_slice(source_row);
|
||||||
|
}
|
||||||
|
Ok::<(), anyhow::Error>(())
|
||||||
|
})();
|
||||||
|
if mapped {
|
||||||
|
self.context.Unmap(&self.staging, 0);
|
||||||
|
}
|
||||||
|
let _ = self.duplication.ReleaseFrame();
|
||||||
|
result?;
|
||||||
|
}
|
||||||
|
Ok(CapturedFrame {
|
||||||
|
width: self.width,
|
||||||
|
height: self.height,
|
||||||
|
pixels: self.pixels.clone(),
|
||||||
|
capture_latency: capture_started.elapsed(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_geometry(width: u32, height: u32) -> anyhow::Result<()> {
|
||||||
|
anyhow::ensure!(
|
||||||
|
width > 0 && height > 0 && width <= MAX_DIMENSION && height <= MAX_DIMENSION,
|
||||||
|
"unsupported DDA output {width}x{height}"
|
||||||
|
);
|
||||||
|
let bytes = usize::try_from(width)
|
||||||
|
.ok()
|
||||||
|
.and_then(|w| usize::try_from(height).ok().and_then(|h| w.checked_mul(h)))
|
||||||
|
.and_then(|pixels| pixels.checked_mul(4))
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("DDA frame size overflow"))?;
|
||||||
|
anyhow::ensure!(bytes <= MAX_RAW_FRAME_BYTES, "DDA frame exceeds 256 MiB");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn encode_frame(
|
||||||
|
width: u32,
|
||||||
|
height: u32,
|
||||||
|
pixels: &[u8],
|
||||||
|
capture: Duration,
|
||||||
|
encode: Duration,
|
||||||
|
) -> anyhow::Result<Vec<u8>> {
|
||||||
|
let mut compressor = ZlibEncoder::new(Vec::new(), Compression::fast());
|
||||||
|
compressor.write_all(pixels)?;
|
||||||
|
let compressed = compressor.finish()?;
|
||||||
|
let raw_len = u32::try_from(pixels.len()).context("DDA raw frame is too large")?;
|
||||||
|
let compressed_len =
|
||||||
|
u32::try_from(compressed.len()).context("DDA compressed frame is too large")?;
|
||||||
|
let mut frame = Vec::with_capacity(FRAME_HEADER_LEN + compressed.len());
|
||||||
|
frame.extend_from_slice(FRAME_MAGIC);
|
||||||
|
frame.push(FRAME_VERSION);
|
||||||
|
frame.push(CODEC_BGRA_ZLIB);
|
||||||
|
frame.extend_from_slice(&0_u16.to_le_bytes());
|
||||||
|
frame.extend_from_slice(&width.to_le_bytes());
|
||||||
|
frame.extend_from_slice(&height.to_le_bytes());
|
||||||
|
frame.extend_from_slice(&raw_len.to_le_bytes());
|
||||||
|
frame.extend_from_slice(&compressed_len.to_le_bytes());
|
||||||
|
frame.extend_from_slice(
|
||||||
|
&u64::try_from(capture.as_micros())
|
||||||
|
.unwrap_or(u64::MAX)
|
||||||
|
.to_le_bytes(),
|
||||||
|
);
|
||||||
|
frame.extend_from_slice(
|
||||||
|
&u64::try_from(encode.as_micros())
|
||||||
|
.unwrap_or(u64::MAX)
|
||||||
|
.to_le_bytes(),
|
||||||
|
);
|
||||||
|
frame.extend_from_slice(&compressed);
|
||||||
|
Ok(frame)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(windows)]
|
||||||
|
pub(super) use windows_capture::DdaCapture;
|
||||||
|
|
||||||
|
#[cfg(not(windows))]
|
||||||
|
pub(super) struct DdaCapture;
|
||||||
|
|
||||||
|
#[cfg(not(windows))]
|
||||||
|
impl DdaCapture {
|
||||||
|
pub(super) fn dimensions(&self) -> (u32, u32) {
|
||||||
|
(0, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn new() -> anyhow::Result<Self> {
|
||||||
|
anyhow::bail!("desktop_duplication_windows_only")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,657 @@
|
|||||||
|
//! Windows H.264 encoding and CPU-side compatibility conversion.
|
||||||
|
//!
|
||||||
|
//! The encoder accepts an NV12 frame and returns one bounded Annex-B access
|
||||||
|
//! unit. The current Desktop Duplication path is a CPU readback, so callers
|
||||||
|
//! must report that upload/copy boundary separately; this module does not
|
||||||
|
//! claim end-to-end zero-copy merely because the MFT is hardware-backed.
|
||||||
|
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
const MAX_DIMENSION: u32 = 16_384;
|
||||||
|
const MAX_ACCESS_UNIT_BYTES: usize = 16 * 1024 * 1024;
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
|
pub(crate) struct EncodedAccessUnit {
|
||||||
|
pub(crate) data: Vec<u8>,
|
||||||
|
pub(crate) pts_100ns: i64,
|
||||||
|
pub(crate) duration: Duration,
|
||||||
|
pub(crate) encode_latency: Duration,
|
||||||
|
pub(crate) keyframe: bool,
|
||||||
|
pub(crate) encoder: &'static str,
|
||||||
|
pub(crate) hardware: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
|
||||||
|
pub(crate) enum EncoderError {
|
||||||
|
InvalidConfiguration,
|
||||||
|
InvalidFrame,
|
||||||
|
RuntimeUnavailable,
|
||||||
|
EncoderUnavailable,
|
||||||
|
InputRejected,
|
||||||
|
OutputUnavailable,
|
||||||
|
InvalidOutput,
|
||||||
|
KeyFrameUnavailable,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for EncoderError {
|
||||||
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
formatter.write_str(match self {
|
||||||
|
Self::InvalidConfiguration => "invalid_encoder_configuration",
|
||||||
|
Self::InvalidFrame => "invalid_nv12_frame",
|
||||||
|
Self::RuntimeUnavailable => "media_foundation_unavailable",
|
||||||
|
Self::EncoderUnavailable => "h264_encoder_unavailable",
|
||||||
|
Self::InputRejected => "h264_encoder_input_rejected",
|
||||||
|
Self::OutputUnavailable => "h264_encoder_output_unavailable",
|
||||||
|
Self::InvalidOutput => "invalid_h264_access_unit",
|
||||||
|
Self::KeyFrameUnavailable => "h264_keyframe_request_unavailable",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::error::Error for EncoderError {}
|
||||||
|
|
||||||
|
pub(crate) fn validate_configuration(
|
||||||
|
width: u32,
|
||||||
|
height: u32,
|
||||||
|
fps: u32,
|
||||||
|
) -> Result<(), EncoderError> {
|
||||||
|
if !(2..=MAX_DIMENSION).contains(&width)
|
||||||
|
|| !(2..=MAX_DIMENSION).contains(&height)
|
||||||
|
|| fps == 0
|
||||||
|
|| fps > 240
|
||||||
|
|| width % 2 != 0
|
||||||
|
|| height % 2 != 0
|
||||||
|
{
|
||||||
|
return Err(EncoderError::InvalidConfiguration);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn expected_nv12_bytes(width: u32, height: u32) -> Option<usize> {
|
||||||
|
usize::try_from(width)
|
||||||
|
.ok()
|
||||||
|
.and_then(|width| {
|
||||||
|
usize::try_from(height)
|
||||||
|
.ok()
|
||||||
|
.and_then(|height| width.checked_mul(height))
|
||||||
|
})
|
||||||
|
.and_then(|y| y.checked_add(y / 2))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Converts top-down BGRA8 into an 8-bit SDR NV12 frame.
|
||||||
|
pub(crate) fn bgra_to_nv12(width: u32, height: u32, bgra: &[u8]) -> Result<Vec<u8>, EncoderError> {
|
||||||
|
validate_configuration(width, height, 1)?;
|
||||||
|
let expected = expected_nv12_bytes(width, height).ok_or(EncoderError::InvalidFrame)?;
|
||||||
|
let bgra_len = usize::try_from(width)
|
||||||
|
.ok()
|
||||||
|
.and_then(|width| {
|
||||||
|
usize::try_from(height)
|
||||||
|
.ok()
|
||||||
|
.and_then(|height| width.checked_mul(height))
|
||||||
|
})
|
||||||
|
.and_then(|pixels| pixels.checked_mul(4))
|
||||||
|
.ok_or(EncoderError::InvalidFrame)?;
|
||||||
|
if bgra.len() != bgra_len {
|
||||||
|
return Err(EncoderError::InvalidFrame);
|
||||||
|
}
|
||||||
|
let width = usize::try_from(width).map_err(|_| EncoderError::InvalidFrame)?;
|
||||||
|
let height = usize::try_from(height).map_err(|_| EncoderError::InvalidFrame)?;
|
||||||
|
let y_plane_len = width
|
||||||
|
.checked_mul(height)
|
||||||
|
.ok_or(EncoderError::InvalidFrame)?;
|
||||||
|
let mut output = vec![0_u8; expected];
|
||||||
|
let (y_plane, uv_plane) = output.split_at_mut(y_plane_len);
|
||||||
|
|
||||||
|
for y in 0..height {
|
||||||
|
for x in 0..width {
|
||||||
|
let source = (y * width + x) * 4;
|
||||||
|
let b = i32::from(bgra[source]);
|
||||||
|
let g = i32::from(bgra[source + 1]);
|
||||||
|
let r = i32::from(bgra[source + 2]);
|
||||||
|
y_plane[y * width + x] = clamp_u8((66 * r + 129 * g + 25 * b + 128) / 256 + 16);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for y in (0..height).step_by(2) {
|
||||||
|
for x in (0..width).step_by(2) {
|
||||||
|
let mut u = 0_i32;
|
||||||
|
let mut v = 0_i32;
|
||||||
|
for dy in 0..2 {
|
||||||
|
for dx in 0..2 {
|
||||||
|
let source = ((y + dy) * width + x + dx) * 4;
|
||||||
|
let b = i32::from(bgra[source]);
|
||||||
|
let g = i32::from(bgra[source + 1]);
|
||||||
|
let r = i32::from(bgra[source + 2]);
|
||||||
|
u += (-38 * r - 74 * g + 112 * b + 512) / 1024 + 128;
|
||||||
|
v += (112 * r - 94 * g - 18 * b + 512) / 1024 + 128;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let destination = (y / 2) * width + x;
|
||||||
|
uv_plane[destination] = clamp_u8(u / 4);
|
||||||
|
uv_plane[destination + 1] = clamp_u8(v / 4);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(output)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn clamp_u8(value: i32) -> u8 {
|
||||||
|
value.clamp(0, 255) as u8
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn annex_b_contains_idr(data: &[u8]) -> bool {
|
||||||
|
let mut index = 0;
|
||||||
|
while index + 4 <= data.len() {
|
||||||
|
let start = if data[index..].starts_with(&[0, 0, 0, 1]) {
|
||||||
|
index + 4
|
||||||
|
} else if index + 3 <= data.len() && data[index..].starts_with(&[0, 0, 1]) {
|
||||||
|
index + 3
|
||||||
|
} else {
|
||||||
|
index += 1;
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if start < data.len() && data[start] & 0x1f == 5 {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
index = start;
|
||||||
|
}
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Media Foundation may expose H.264 samples as byte-stream Annex-B or as
|
||||||
|
/// AVC length-prefixed NAL units. Normalize both forms before the access unit
|
||||||
|
/// crosses the process boundary.
|
||||||
|
pub(crate) fn normalize_h264_access_unit(data: &[u8]) -> Result<Vec<u8>, EncoderError> {
|
||||||
|
if data.is_empty() {
|
||||||
|
return Err(EncoderError::InvalidOutput);
|
||||||
|
}
|
||||||
|
if data.starts_with(&[0, 0, 0, 1]) || data.starts_with(&[0, 0, 1]) {
|
||||||
|
return Ok(data.to_vec());
|
||||||
|
}
|
||||||
|
let mut offset = 0_usize;
|
||||||
|
let mut output = Vec::with_capacity(data.len() + 16);
|
||||||
|
while offset + 4 <= data.len() {
|
||||||
|
let length = u32::from_be_bytes(data[offset..offset + 4].try_into().unwrap()) as usize;
|
||||||
|
offset += 4;
|
||||||
|
if length == 0 || length > data.len().saturating_sub(offset) {
|
||||||
|
return Err(EncoderError::InvalidOutput);
|
||||||
|
}
|
||||||
|
output.extend_from_slice(&[0, 0, 0, 1]);
|
||||||
|
output.extend_from_slice(&data[offset..offset + length]);
|
||||||
|
offset += length;
|
||||||
|
}
|
||||||
|
if offset != data.len() || output.is_empty() {
|
||||||
|
return Err(EncoderError::InvalidOutput);
|
||||||
|
}
|
||||||
|
Ok(output)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(windows)]
|
||||||
|
mod windows_mft {
|
||||||
|
use super::{
|
||||||
|
EncodedAccessUnit, EncoderError, MAX_ACCESS_UNIT_BYTES, annex_b_contains_idr,
|
||||||
|
validate_configuration,
|
||||||
|
};
|
||||||
|
use core::mem::ManuallyDrop;
|
||||||
|
use std::collections::VecDeque;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
use windows::Win32::Media::MediaFoundation::{
|
||||||
|
CLSID_MSH264EncoderMFT, CODECAPI_AVEncVideoForceKeyFrame, ICodecAPI, IMFActivate,
|
||||||
|
IMFMediaType, IMFTransform, MF_E_NOTACCEPTING, MF_E_TRANSFORM_NEED_MORE_INPUT,
|
||||||
|
MF_MT_AVG_BITRATE, MF_MT_FRAME_RATE, MF_MT_FRAME_SIZE, MF_MT_INTERLACE_MODE,
|
||||||
|
MF_MT_MAJOR_TYPE, MF_MT_SUBTYPE, MF_VERSION, MFCreateMediaType, MFCreateMemoryBuffer,
|
||||||
|
MFCreateSample, MFMediaType_Video, MFSTARTUP_FULL, MFShutdown, MFStartup,
|
||||||
|
MFT_CATEGORY_VIDEO_ENCODER, MFT_ENUM_FLAG_HARDWARE,
|
||||||
|
MFT_ENUM_FLAG_SORTANDFILTER_APPROVED_ONLY, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING,
|
||||||
|
MFT_MESSAGE_NOTIFY_START_OF_STREAM, MFT_OUTPUT_DATA_BUFFER, MFT_REGISTER_TYPE_INFO,
|
||||||
|
MFVideoFormat_H264, MFVideoFormat_NV12, MFVideoInterlace_Progressive,
|
||||||
|
};
|
||||||
|
use windows::Win32::System::Com::{
|
||||||
|
CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED, CoCreateInstance, CoInitializeEx, CoTaskMemFree,
|
||||||
|
};
|
||||||
|
use windows::Win32::System::Variant::VARIANT;
|
||||||
|
use windows::core::Interface as _;
|
||||||
|
|
||||||
|
const MAX_OUTPUTS_PER_INPUT: usize = 8;
|
||||||
|
const MAX_PENDING_ACCESS_UNITS: usize = 16;
|
||||||
|
|
||||||
|
struct ComGuard;
|
||||||
|
impl Drop for ComGuard {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
unsafe { windows::Win32::System::Com::CoUninitialize() };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct MfGuard;
|
||||||
|
impl Drop for MfGuard {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
unsafe {
|
||||||
|
let _ = MFShutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) struct H264Encoder {
|
||||||
|
transform: IMFTransform,
|
||||||
|
width: u32,
|
||||||
|
height: u32,
|
||||||
|
fps: u32,
|
||||||
|
next_pts_100ns: i64,
|
||||||
|
last_explicit_pts_100ns: Option<i64>,
|
||||||
|
encoder: &'static str,
|
||||||
|
hardware: bool,
|
||||||
|
pending: VecDeque<EncodedAccessUnit>,
|
||||||
|
_com: ComGuard,
|
||||||
|
_mf: MfGuard,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl H264Encoder {
|
||||||
|
pub(crate) fn new(
|
||||||
|
width: u32,
|
||||||
|
height: u32,
|
||||||
|
fps: u32,
|
||||||
|
hardware_only: bool,
|
||||||
|
) -> Result<Self, EncoderError> {
|
||||||
|
validate_configuration(width, height, fps)?;
|
||||||
|
let com_result = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) };
|
||||||
|
if com_result.is_err() && com_result != windows::Win32::Foundation::S_FALSE {
|
||||||
|
return Err(EncoderError::RuntimeUnavailable);
|
||||||
|
}
|
||||||
|
let com = ComGuard;
|
||||||
|
unsafe { MFStartup(MF_VERSION, MFSTARTUP_FULL) }
|
||||||
|
.map_err(|_| EncoderError::RuntimeUnavailable)?;
|
||||||
|
let mf = MfGuard;
|
||||||
|
let (transform, encoder, hardware) = if hardware_only {
|
||||||
|
activate_hardware_encoder()?.ok_or(EncoderError::EncoderUnavailable)?
|
||||||
|
} else {
|
||||||
|
let transform: IMFTransform = unsafe {
|
||||||
|
CoCreateInstance(&CLSID_MSH264EncoderMFT, None, CLSCTX_INPROC_SERVER)
|
||||||
|
}
|
||||||
|
.map_err(|_| EncoderError::EncoderUnavailable)?;
|
||||||
|
(transform, "mfh264enc", false)
|
||||||
|
};
|
||||||
|
configure_transform(&transform, width, height, fps)?;
|
||||||
|
unsafe { transform.ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0) }
|
||||||
|
.map_err(|_| EncoderError::EncoderUnavailable)?;
|
||||||
|
unsafe { transform.ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0) }
|
||||||
|
.map_err(|_| EncoderError::EncoderUnavailable)?;
|
||||||
|
Ok(Self {
|
||||||
|
transform,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
fps,
|
||||||
|
next_pts_100ns: 0,
|
||||||
|
last_explicit_pts_100ns: None,
|
||||||
|
encoder,
|
||||||
|
hardware,
|
||||||
|
pending: VecDeque::new(),
|
||||||
|
_com: com,
|
||||||
|
_mf: mf,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn encode(
|
||||||
|
&mut self,
|
||||||
|
nv12: &[u8],
|
||||||
|
) -> Result<Option<EncodedAccessUnit>, EncoderError> {
|
||||||
|
let duration = Duration::from_nanos(1_000_000_000_u64 / u64::from(self.fps));
|
||||||
|
let timestamp = self.next_pts_100ns;
|
||||||
|
self.next_pts_100ns = self
|
||||||
|
.next_pts_100ns
|
||||||
|
.saturating_add(i64::try_from(duration.as_nanos() / 100).unwrap_or(i64::MAX));
|
||||||
|
self.encode_timestamped(nv12, timestamp, duration)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn encode_at(
|
||||||
|
&mut self,
|
||||||
|
nv12: &[u8],
|
||||||
|
requested_pts_100ns: i64,
|
||||||
|
) -> Result<Option<EncodedAccessUnit>, EncoderError> {
|
||||||
|
if requested_pts_100ns < 0 {
|
||||||
|
return Err(EncoderError::InvalidFrame);
|
||||||
|
}
|
||||||
|
let timestamp = self
|
||||||
|
.last_explicit_pts_100ns
|
||||||
|
.map_or(requested_pts_100ns, |last| {
|
||||||
|
requested_pts_100ns.max(last.saturating_add(1))
|
||||||
|
});
|
||||||
|
self.last_explicit_pts_100ns = Some(timestamp);
|
||||||
|
let duration = Duration::from_nanos(1_000_000_000_u64 / u64::from(self.fps));
|
||||||
|
self.encode_timestamped(nv12, timestamp, duration)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn encode_timestamped(
|
||||||
|
&mut self,
|
||||||
|
nv12: &[u8],
|
||||||
|
timestamp: i64,
|
||||||
|
duration: Duration,
|
||||||
|
) -> Result<Option<EncodedAccessUnit>, EncoderError> {
|
||||||
|
let started = Instant::now();
|
||||||
|
let expected = super::expected_nv12_bytes(self.width, self.height)
|
||||||
|
.ok_or(EncoderError::InvalidFrame)?;
|
||||||
|
if nv12.len() != expected {
|
||||||
|
return Err(EncoderError::InvalidFrame);
|
||||||
|
}
|
||||||
|
let sample = create_sample(nv12, timestamp, duration)?;
|
||||||
|
let input = unsafe { self.transform.ProcessInput(0, &sample, 0) };
|
||||||
|
if let Err(error) = input {
|
||||||
|
if error.code() != MF_E_NOTACCEPTING {
|
||||||
|
return Err(EncoderError::InputRejected);
|
||||||
|
}
|
||||||
|
let outputs = self.drain_outputs(started)?;
|
||||||
|
self.queue_outputs(outputs)?;
|
||||||
|
unsafe { self.transform.ProcessInput(0, &sample, 0) }
|
||||||
|
.map_err(|_| EncoderError::InputRejected)?;
|
||||||
|
}
|
||||||
|
let outputs = self.drain_outputs(started)?;
|
||||||
|
self.queue_outputs(outputs)?;
|
||||||
|
Ok(self.pending.pop_front())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn request_keyframe(&self) -> Result<(), EncoderError> {
|
||||||
|
let codec_api = self
|
||||||
|
.transform
|
||||||
|
.cast::<ICodecAPI>()
|
||||||
|
.map_err(|_| EncoderError::KeyFrameUnavailable)?;
|
||||||
|
let value = VARIANT::from(true);
|
||||||
|
let force_keyframe = CODECAPI_AVEncVideoForceKeyFrame;
|
||||||
|
unsafe { codec_api.SetValue(&raw const force_keyframe, &raw const value) }
|
||||||
|
.map_err(|_| EncoderError::KeyFrameUnavailable)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn queue_outputs(&mut self, outputs: Vec<EncodedAccessUnit>) -> Result<(), EncoderError> {
|
||||||
|
if self.pending.len().saturating_add(outputs.len()) > MAX_PENDING_ACCESS_UNITS {
|
||||||
|
return Err(EncoderError::OutputUnavailable);
|
||||||
|
}
|
||||||
|
self.pending.extend(outputs);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drain_outputs(
|
||||||
|
&mut self,
|
||||||
|
started: Instant,
|
||||||
|
) -> Result<Vec<EncodedAccessUnit>, EncoderError> {
|
||||||
|
let mut access_units = Vec::new();
|
||||||
|
for _ in 0..MAX_OUTPUTS_PER_INPUT {
|
||||||
|
let mut output = MFT_OUTPUT_DATA_BUFFER::default();
|
||||||
|
output.pSample = ManuallyDrop::new(None);
|
||||||
|
output.pEvents = ManuallyDrop::new(None);
|
||||||
|
let mut status = 0_u32;
|
||||||
|
let result = unsafe {
|
||||||
|
self.transform.ProcessOutput(
|
||||||
|
0,
|
||||||
|
std::slice::from_mut(&mut output),
|
||||||
|
&raw mut status,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let sample = unsafe { ManuallyDrop::take(&mut output.pSample) };
|
||||||
|
let _events = unsafe { ManuallyDrop::take(&mut output.pEvents) };
|
||||||
|
match result {
|
||||||
|
Ok(()) => {
|
||||||
|
let sample = sample.ok_or(EncoderError::OutputUnavailable)?;
|
||||||
|
access_units.push(self.access_unit_from_sample(sample, started.elapsed())?);
|
||||||
|
}
|
||||||
|
Err(error) if error.code() == MF_E_TRANSFORM_NEED_MORE_INPUT => break,
|
||||||
|
Err(_) => return Err(EncoderError::OutputUnavailable),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(access_units)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn access_unit_from_sample(
|
||||||
|
&self,
|
||||||
|
sample: windows::Win32::Media::MediaFoundation::IMFSample,
|
||||||
|
encode_latency: Duration,
|
||||||
|
) -> Result<EncodedAccessUnit, EncoderError> {
|
||||||
|
let buffer_count =
|
||||||
|
unsafe { sample.GetBufferCount() }.map_err(|_| EncoderError::InvalidOutput)?;
|
||||||
|
if buffer_count != 1 {
|
||||||
|
return Err(EncoderError::InvalidOutput);
|
||||||
|
}
|
||||||
|
let buffer =
|
||||||
|
unsafe { sample.GetBufferByIndex(0) }.map_err(|_| EncoderError::InvalidOutput)?;
|
||||||
|
let current_length =
|
||||||
|
unsafe { buffer.GetCurrentLength() }.map_err(|_| EncoderError::InvalidOutput)?;
|
||||||
|
let mut data = vec![0_u8; MAX_ACCESS_UNIT_BYTES.min(current_length as usize)];
|
||||||
|
let mut length = 0_u32;
|
||||||
|
let mut locked = core::ptr::null_mut();
|
||||||
|
unsafe { buffer.Lock(&raw mut locked, None, Some(&raw mut length)) }
|
||||||
|
.map_err(|_| EncoderError::InvalidOutput)?;
|
||||||
|
let result = if locked.is_null()
|
||||||
|
|| usize::try_from(length).unwrap_or(usize::MAX) > MAX_ACCESS_UNIT_BYTES
|
||||||
|
{
|
||||||
|
Err(EncoderError::InvalidOutput)
|
||||||
|
} else {
|
||||||
|
data.resize(length as usize, 0);
|
||||||
|
unsafe {
|
||||||
|
core::ptr::copy_nonoverlapping(
|
||||||
|
locked.cast::<u8>(),
|
||||||
|
data.as_mut_ptr(),
|
||||||
|
data.len(),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
Ok(())
|
||||||
|
};
|
||||||
|
let _ = unsafe { buffer.Unlock() };
|
||||||
|
result?;
|
||||||
|
if data.is_empty() {
|
||||||
|
return Err(EncoderError::InvalidOutput);
|
||||||
|
}
|
||||||
|
let data = super::normalize_h264_access_unit(&data)?;
|
||||||
|
let keyframe = annex_b_contains_idr(&data);
|
||||||
|
let pts_100ns = unsafe { sample.GetSampleTime() }.unwrap_or(0);
|
||||||
|
Ok(EncodedAccessUnit {
|
||||||
|
data,
|
||||||
|
pts_100ns,
|
||||||
|
duration: Duration::from_nanos(1_000_000_000 / u64::from(self.fps)),
|
||||||
|
encode_latency,
|
||||||
|
keyframe,
|
||||||
|
encoder: self.encoder,
|
||||||
|
hardware: self.hardware,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn activate_hardware_encoder()
|
||||||
|
-> Result<Option<(IMFTransform, &'static str, bool)>, EncoderError> {
|
||||||
|
let input = MFT_REGISTER_TYPE_INFO {
|
||||||
|
guidMajorType: MFMediaType_Video,
|
||||||
|
guidSubtype: MFVideoFormat_NV12,
|
||||||
|
};
|
||||||
|
let output = MFT_REGISTER_TYPE_INFO {
|
||||||
|
guidMajorType: MFMediaType_Video,
|
||||||
|
guidSubtype: MFVideoFormat_H264,
|
||||||
|
};
|
||||||
|
let mut activations: *mut Option<IMFActivate> = core::ptr::null_mut();
|
||||||
|
let mut count = 0_u32;
|
||||||
|
let result = unsafe {
|
||||||
|
windows::Win32::Media::MediaFoundation::MFTEnumEx(
|
||||||
|
MFT_CATEGORY_VIDEO_ENCODER,
|
||||||
|
MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_SORTANDFILTER_APPROVED_ONLY,
|
||||||
|
Some(&raw const input),
|
||||||
|
Some(&raw const output),
|
||||||
|
&raw mut activations,
|
||||||
|
&raw mut count,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
if result.is_err() || count == 0 {
|
||||||
|
free_activations(activations, count);
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let transform = unsafe {
|
||||||
|
(*activations)
|
||||||
|
.as_ref()
|
||||||
|
.ok_or(EncoderError::EncoderUnavailable)?
|
||||||
|
.ActivateObject::<IMFTransform>()
|
||||||
|
};
|
||||||
|
free_activations(activations, count);
|
||||||
|
Ok(Some((
|
||||||
|
transform.map_err(|_| EncoderError::EncoderUnavailable)?,
|
||||||
|
"hardware-mft",
|
||||||
|
true,
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn free_activations(pointer: *mut Option<IMFActivate>, count: u32) {
|
||||||
|
if pointer.is_null() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
unsafe {
|
||||||
|
for item in std::slice::from_raw_parts_mut(pointer, count as usize) {
|
||||||
|
*item = None;
|
||||||
|
}
|
||||||
|
CoTaskMemFree(Some(pointer.cast()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn configure_transform(
|
||||||
|
transform: &IMFTransform,
|
||||||
|
width: u32,
|
||||||
|
height: u32,
|
||||||
|
fps: u32,
|
||||||
|
) -> Result<(), EncoderError> {
|
||||||
|
let input = media_type(MFVideoFormat_NV12, width, height, fps, None)?;
|
||||||
|
let output = media_type(
|
||||||
|
MFVideoFormat_H264,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
fps,
|
||||||
|
Some(bitrate(width, height, fps)),
|
||||||
|
)?;
|
||||||
|
unsafe { transform.SetInputType(0, &input, 0) }
|
||||||
|
.map_err(|_| EncoderError::EncoderUnavailable)?;
|
||||||
|
unsafe { transform.SetOutputType(0, &output, 0) }
|
||||||
|
.map_err(|_| EncoderError::EncoderUnavailable)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn media_type(
|
||||||
|
subtype: windows::core::GUID,
|
||||||
|
width: u32,
|
||||||
|
height: u32,
|
||||||
|
fps: u32,
|
||||||
|
bitrate_value: Option<u32>,
|
||||||
|
) -> Result<IMFMediaType, EncoderError> {
|
||||||
|
let media_type =
|
||||||
|
unsafe { MFCreateMediaType() }.map_err(|_| EncoderError::RuntimeUnavailable)?;
|
||||||
|
unsafe { media_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video) }
|
||||||
|
.map_err(|_| EncoderError::EncoderUnavailable)?;
|
||||||
|
unsafe { media_type.SetGUID(&MF_MT_SUBTYPE, &subtype) }
|
||||||
|
.map_err(|_| EncoderError::EncoderUnavailable)?;
|
||||||
|
unsafe {
|
||||||
|
media_type.SetUINT64(
|
||||||
|
&MF_MT_FRAME_SIZE,
|
||||||
|
(u64::from(width) << 32) | u64::from(height),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
.map_err(|_| EncoderError::EncoderUnavailable)?;
|
||||||
|
unsafe { media_type.SetUINT64(&MF_MT_FRAME_RATE, (u64::from(fps) << 32) | 1) }
|
||||||
|
.map_err(|_| EncoderError::EncoderUnavailable)?;
|
||||||
|
unsafe {
|
||||||
|
media_type.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32)
|
||||||
|
}
|
||||||
|
.map_err(|_| EncoderError::EncoderUnavailable)?;
|
||||||
|
if let Some(value) = bitrate_value {
|
||||||
|
unsafe { media_type.SetUINT32(&MF_MT_AVG_BITRATE, value) }
|
||||||
|
.map_err(|_| EncoderError::EncoderUnavailable)?;
|
||||||
|
}
|
||||||
|
Ok(media_type)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bitrate(width: u32, height: u32, fps: u32) -> u32 {
|
||||||
|
let value = u64::from(width)
|
||||||
|
.saturating_mul(u64::from(height))
|
||||||
|
.saturating_mul(u64::from(fps))
|
||||||
|
.saturating_mul(2)
|
||||||
|
.saturating_div(100);
|
||||||
|
u32::try_from(value.clamp(2_000_000, 80_000_000)).unwrap_or(80_000_000)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_sample(
|
||||||
|
data: &[u8],
|
||||||
|
pts: i64,
|
||||||
|
duration: Duration,
|
||||||
|
) -> Result<windows::Win32::Media::MediaFoundation::IMFSample, EncoderError> {
|
||||||
|
let buffer = unsafe {
|
||||||
|
MFCreateMemoryBuffer(u32::try_from(data.len()).map_err(|_| EncoderError::InvalidFrame)?)
|
||||||
|
}
|
||||||
|
.map_err(|_| EncoderError::RuntimeUnavailable)?;
|
||||||
|
let mut destination = core::ptr::null_mut();
|
||||||
|
unsafe { buffer.Lock(&raw mut destination, None, None) }
|
||||||
|
.map_err(|_| EncoderError::InvalidFrame)?;
|
||||||
|
if destination.is_null() {
|
||||||
|
let _ = unsafe { buffer.Unlock() };
|
||||||
|
return Err(EncoderError::InvalidFrame);
|
||||||
|
}
|
||||||
|
unsafe {
|
||||||
|
core::ptr::copy_nonoverlapping(data.as_ptr(), destination.cast::<u8>(), data.len());
|
||||||
|
}
|
||||||
|
unsafe { buffer.Unlock() }.map_err(|_| EncoderError::InvalidFrame)?;
|
||||||
|
unsafe {
|
||||||
|
buffer.SetCurrentLength(
|
||||||
|
u32::try_from(data.len()).map_err(|_| EncoderError::InvalidFrame)?,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
.map_err(|_| EncoderError::InvalidFrame)?;
|
||||||
|
let sample = unsafe { MFCreateSample() }.map_err(|_| EncoderError::RuntimeUnavailable)?;
|
||||||
|
unsafe { sample.AddBuffer(&buffer) }.map_err(|_| EncoderError::InvalidFrame)?;
|
||||||
|
unsafe { sample.SetSampleTime(pts) }.map_err(|_| EncoderError::InvalidFrame)?;
|
||||||
|
unsafe {
|
||||||
|
sample.SetSampleDuration(i64::try_from(duration.as_nanos() / 100).unwrap_or(i64::MAX))
|
||||||
|
}
|
||||||
|
.map_err(|_| EncoderError::InvalidFrame)?;
|
||||||
|
Ok(sample)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(windows)]
|
||||||
|
#[allow(unused_imports)]
|
||||||
|
pub(crate) use windows_mft::H264Encoder;
|
||||||
|
|
||||||
|
#[cfg(not(windows))]
|
||||||
|
pub(crate) struct H264Encoder;
|
||||||
|
|
||||||
|
#[cfg(not(windows))]
|
||||||
|
impl H264Encoder {
|
||||||
|
pub(crate) fn new(_: u32, _: u32, _: u32, _: bool) -> Result<Self, EncoderError> {
|
||||||
|
Err(EncoderError::RuntimeUnavailable)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bgra_to_nv12_has_expected_layout() {
|
||||||
|
let bgra = vec![
|
||||||
|
0_u8, 0, 255, 255, 0, 255, 0, 255, 255, 0, 0, 255, 255, 255, 255, 255,
|
||||||
|
];
|
||||||
|
let nv12 = bgra_to_nv12(2, 2, &bgra).unwrap();
|
||||||
|
assert_eq!(nv12.len(), 6);
|
||||||
|
assert!(nv12[..4].iter().all(|value| *value > 0));
|
||||||
|
assert_eq!(nv12[4..].len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn odd_dimensions_and_wrong_lengths_fail_closed() {
|
||||||
|
assert_eq!(
|
||||||
|
validate_configuration(3, 2, 30),
|
||||||
|
Err(EncoderError::InvalidConfiguration)
|
||||||
|
);
|
||||||
|
assert_eq!(bgra_to_nv12(2, 2, &[0; 3]), Err(EncoderError::InvalidFrame));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn idr_detection_accepts_annex_b_five() {
|
||||||
|
assert!(annex_b_contains_idr(&[0, 0, 0, 1, 5, 1]));
|
||||||
|
assert!(!annex_b_contains_idr(&[0, 0, 0, 1, 1, 1]));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn avc_length_prefixed_output_is_normalized_to_annex_b() {
|
||||||
|
let avc = [0_u8, 0, 0, 2, 5, 1, 0, 0, 0, 1, 1];
|
||||||
|
assert_eq!(
|
||||||
|
normalize_h264_access_unit(&avc).unwrap(),
|
||||||
|
[0, 0, 0, 1, 5, 1, 0, 0, 0, 1, 1]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,186 @@
|
|||||||
|
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||||
|
use hmac::{Hmac, KeyInit as _, Mac as _};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use sha2::Sha256;
|
||||||
|
use std::io;
|
||||||
|
use tokio::io::{
|
||||||
|
AsyncBufReadExt as _, AsyncRead, AsyncReadExt as _, AsyncWrite, AsyncWriteExt as _, BufReader,
|
||||||
|
};
|
||||||
|
use zeroize::{Zeroize as _, Zeroizing};
|
||||||
|
|
||||||
|
pub const AGENT_PIPE_TOKEN_ENV: &str = "REMOTEDESK_AGENT_PIPE_TOKEN";
|
||||||
|
const AUTH_PROTOCOL_VERSION: u8 = 1;
|
||||||
|
const AUTH_KEY_BYTES: usize = 32;
|
||||||
|
const AUTH_NONCE_BYTES: usize = 32;
|
||||||
|
const AUTH_RESPONSE_MAX_BYTES: usize = 2 * 1024;
|
||||||
|
const AUTH_DOMAIN: &[u8] = b"RemoteDesk Windows Agent pipe auth v1\0";
|
||||||
|
|
||||||
|
type HmacSha256 = Hmac<Sha256>;
|
||||||
|
|
||||||
|
pub struct PipeAuthenticator {
|
||||||
|
key: Zeroizing<Vec<u8>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct Challenge<'a> {
|
||||||
|
kind: &'a str,
|
||||||
|
protocol_version: u8,
|
||||||
|
nonce: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct Authentication {
|
||||||
|
kind: String,
|
||||||
|
protocol_version: u8,
|
||||||
|
mac: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PipeAuthenticator {
|
||||||
|
pub fn from_environment() -> io::Result<Self> {
|
||||||
|
let encoded = std::env::var(AGENT_PIPE_TOKEN_ENV).map_err(|_| {
|
||||||
|
io::Error::new(
|
||||||
|
io::ErrorKind::PermissionDenied,
|
||||||
|
format!("{AGENT_PIPE_TOKEN_ENV} is required for authenticated Named Pipe mode"),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
// The bootstrap is needed only while constructing the in-memory key.
|
||||||
|
unsafe { std::env::remove_var(AGENT_PIPE_TOKEN_ENV) };
|
||||||
|
let mut encoded = Zeroizing::new(encoded);
|
||||||
|
let result = Self::from_encoded(&encoded);
|
||||||
|
encoded.zeroize();
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
fn from_encoded(encoded: &str) -> io::Result<Self> {
|
||||||
|
let key = Zeroizing::new(URL_SAFE_NO_PAD.decode(encoded).map_err(|_| {
|
||||||
|
io::Error::new(
|
||||||
|
io::ErrorKind::InvalidData,
|
||||||
|
"Agent Pipe token is not base64url",
|
||||||
|
)
|
||||||
|
})?);
|
||||||
|
if key.len() != AUTH_KEY_BYTES {
|
||||||
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::InvalidData,
|
||||||
|
"Agent Pipe token must decode to exactly 32 bytes",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(Self { key })
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn authenticate<S>(&self, stream: &mut S) -> io::Result<()>
|
||||||
|
where
|
||||||
|
S: AsyncRead + AsyncWrite + Unpin,
|
||||||
|
{
|
||||||
|
let mut nonce = [0_u8; AUTH_NONCE_BYTES];
|
||||||
|
getrandom::fill(&mut nonce).map_err(io::Error::other)?;
|
||||||
|
let challenge = serde_json::to_vec(&Challenge {
|
||||||
|
kind: "agent_auth_challenge",
|
||||||
|
protocol_version: AUTH_PROTOCOL_VERSION,
|
||||||
|
nonce: URL_SAFE_NO_PAD.encode(nonce),
|
||||||
|
})
|
||||||
|
.map_err(io::Error::other)?;
|
||||||
|
stream.write_all(&challenge).await?;
|
||||||
|
stream.write_all(b"\n").await?;
|
||||||
|
stream.flush().await?;
|
||||||
|
|
||||||
|
let mut line = String::new();
|
||||||
|
let mut reader = BufReader::new(stream).take((AUTH_RESPONSE_MAX_BYTES + 1) as u64);
|
||||||
|
let bytes = reader.read_line(&mut line).await?;
|
||||||
|
if bytes == 0
|
||||||
|
|| bytes > AUTH_RESPONSE_MAX_BYTES
|
||||||
|
|| !line.ends_with('\n')
|
||||||
|
|| !line.is_ascii()
|
||||||
|
{
|
||||||
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::InvalidData,
|
||||||
|
"Agent Pipe authentication response is invalid",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let authentication: Authentication = serde_json::from_str(&line)
|
||||||
|
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid Agent Pipe auth"))?;
|
||||||
|
if authentication.kind != "agent_auth"
|
||||||
|
|| authentication.protocol_version != AUTH_PROTOCOL_VERSION
|
||||||
|
{
|
||||||
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::PermissionDenied,
|
||||||
|
"Agent Pipe authentication protocol mismatch",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let supplied = URL_SAFE_NO_PAD.decode(authentication.mac).map_err(|_| {
|
||||||
|
io::Error::new(io::ErrorKind::InvalidData, "invalid Agent Pipe auth MAC")
|
||||||
|
})?;
|
||||||
|
let mut mac = HmacSha256::new_from_slice(&self.key)
|
||||||
|
.map_err(|_| io::Error::other("invalid Agent Pipe HMAC key"))?;
|
||||||
|
mac.update(AUTH_DOMAIN);
|
||||||
|
mac.update(&nonce);
|
||||||
|
mac.verify_slice(&supplied).map_err(|_| {
|
||||||
|
io::Error::new(
|
||||||
|
io::ErrorKind::PermissionDenied,
|
||||||
|
"Agent Pipe authentication was rejected",
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
fn authentication_mac(key: &[u8], nonce: &[u8]) -> io::Result<Vec<u8>> {
|
||||||
|
let mut mac = HmacSha256::new_from_slice(key)
|
||||||
|
.map_err(|_| io::Error::other("invalid Agent Pipe HMAC key"))?;
|
||||||
|
mac.update(AUTH_DOMAIN);
|
||||||
|
mac.update(nonce);
|
||||||
|
Ok(mac.finalize().into_bytes().to_vec())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn token_requires_exactly_32_base64url_bytes() {
|
||||||
|
assert!(PipeAuthenticator::from_encoded("not-base64!").is_err());
|
||||||
|
assert!(PipeAuthenticator::from_encoded(&URL_SAFE_NO_PAD.encode([7_u8; 31])).is_err());
|
||||||
|
assert!(PipeAuthenticator::from_encoded(&URL_SAFE_NO_PAD.encode([7_u8; 32])).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn authentication_mac_matches_go_wire_vector() {
|
||||||
|
let key = (0_u8..32).collect::<Vec<_>>();
|
||||||
|
let nonce = (32_u8..64).collect::<Vec<_>>();
|
||||||
|
let mac = authentication_mac(&key, &nonce).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
URL_SAFE_NO_PAD.encode(mac),
|
||||||
|
"KinBDOBvVhz0eRv37iahq8BO0z8QdiI0Qnx7HZmcxIk"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn challenge_accepts_the_matching_response() {
|
||||||
|
let key = [9_u8; 32];
|
||||||
|
let authenticator = PipeAuthenticator::from_encoded(&URL_SAFE_NO_PAD.encode(key)).unwrap();
|
||||||
|
let (mut server, client) = tokio::io::duplex(4 * 1024);
|
||||||
|
let server_task =
|
||||||
|
tokio::spawn(async move { authenticator.authenticate(&mut server).await });
|
||||||
|
let (reader, mut writer) = tokio::io::split(client);
|
||||||
|
let mut reader = BufReader::new(reader);
|
||||||
|
let mut line = String::new();
|
||||||
|
reader.read_line(&mut line).await.unwrap();
|
||||||
|
let challenge: Value = serde_json::from_str(&line).unwrap();
|
||||||
|
let nonce = URL_SAFE_NO_PAD
|
||||||
|
.decode(challenge["nonce"].as_str().unwrap())
|
||||||
|
.unwrap();
|
||||||
|
let mac = authentication_mac(&key, &nonce).unwrap();
|
||||||
|
let response = serde_json::json!({
|
||||||
|
"kind": "agent_auth",
|
||||||
|
"protocol_version": 1,
|
||||||
|
"mac": URL_SAFE_NO_PAD.encode(mac)
|
||||||
|
});
|
||||||
|
writer
|
||||||
|
.write_all(response.to_string().as_bytes())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
writer.write_all(b"\n").await.unwrap();
|
||||||
|
server_task.await.unwrap().unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
+1041
-59
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,439 @@
|
|||||||
|
//! Bounded SPSC ring for encoded media IPC.
|
||||||
|
//!
|
||||||
|
//! The ring carries compressed access units only. It deliberately does not
|
||||||
|
//! expose a GPU texture or a raw BGRA/NV12 surface to the Go service.
|
||||||
|
|
||||||
|
use memmap2::{MmapMut, MmapOptions};
|
||||||
|
use serde::Serialize;
|
||||||
|
use std::fs::{File, OpenOptions};
|
||||||
|
use std::io;
|
||||||
|
use std::mem::{align_of, size_of};
|
||||||
|
use std::path::Path;
|
||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
|
||||||
|
const MAGIC: [u8; 8] = *b"RDMRING1";
|
||||||
|
const VERSION: u32 = 1;
|
||||||
|
const HEADER_BYTES: usize = size_of::<RingHeader>();
|
||||||
|
const SLOT_HEADER_BYTES: usize = size_of::<SlotHeader>();
|
||||||
|
const MAX_SLOTS: u32 = 256;
|
||||||
|
const MAX_SLOT_BYTES: u32 = 16 * 1024 * 1024;
|
||||||
|
|
||||||
|
#[repr(C, align(8))]
|
||||||
|
struct RingHeader {
|
||||||
|
magic: [u8; 8],
|
||||||
|
version: u32,
|
||||||
|
slot_count: u32,
|
||||||
|
slot_bytes: u32,
|
||||||
|
reserved: u32,
|
||||||
|
write_seq: AtomicU64,
|
||||||
|
read_seq: AtomicU64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[repr(C)]
|
||||||
|
#[derive(Clone, Copy, Default)]
|
||||||
|
struct SlotHeader {
|
||||||
|
sequence: u64,
|
||||||
|
pts: u64,
|
||||||
|
kind: u32,
|
||||||
|
length: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct MediaRecord {
|
||||||
|
pub sequence: u64,
|
||||||
|
pub pts: u64,
|
||||||
|
pub kind: u32,
|
||||||
|
pub payload: Vec<u8>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Control-plane description for a ring owned by one authenticated session.
|
||||||
|
/// The ring contains encoded access units only; it is never a raw-surface
|
||||||
|
/// transport. The creator must bind this descriptor to its session ACL before
|
||||||
|
/// sending it to another process.
|
||||||
|
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
|
||||||
|
pub struct MediaRingDescriptor {
|
||||||
|
pub schema_version: u16,
|
||||||
|
pub session_id: String,
|
||||||
|
pub generation: u64,
|
||||||
|
pub owner_pid: u32,
|
||||||
|
pub ring_path: String,
|
||||||
|
pub slot_count: u32,
|
||||||
|
pub slot_bytes: u32,
|
||||||
|
pub video_transport: &'static str,
|
||||||
|
pub audio_transport: &'static str,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub audio_ring: Option<AudioRingDescriptor>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Control-plane description for a separate Opus packet ring.
|
||||||
|
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
|
||||||
|
pub struct AudioRingDescriptor {
|
||||||
|
pub schema_version: u16,
|
||||||
|
pub session_id: String,
|
||||||
|
pub generation: u64,
|
||||||
|
pub owner_pid: u32,
|
||||||
|
pub ring_path: String,
|
||||||
|
pub slot_count: u32,
|
||||||
|
pub slot_bytes: u32,
|
||||||
|
pub audio_transport: &'static str,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AudioRingDescriptor {
|
||||||
|
pub fn validate(&self) -> io::Result<()> {
|
||||||
|
if self.schema_version != VERSION as u16
|
||||||
|
|| self.generation == 0
|
||||||
|
|| self.owner_pid == 0
|
||||||
|
|| self.session_id.is_empty()
|
||||||
|
|| self.session_id.len() > 128
|
||||||
|
|| !self
|
||||||
|
.session_id
|
||||||
|
.bytes()
|
||||||
|
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_'))
|
||||||
|
{
|
||||||
|
return invalid_ring("audio ring descriptor identity is invalid");
|
||||||
|
}
|
||||||
|
if !Path::new(&self.ring_path).is_absolute() {
|
||||||
|
return invalid_ring("audio ring path must be absolute");
|
||||||
|
}
|
||||||
|
if self.audio_transport != "shared_memory_opus_packets" {
|
||||||
|
return invalid_ring("audio ring transport contract is invalid");
|
||||||
|
}
|
||||||
|
validate_geometry(self.slot_count, self.slot_bytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MediaRingDescriptor {
|
||||||
|
pub fn validate(&self) -> io::Result<()> {
|
||||||
|
if self.schema_version != VERSION as u16
|
||||||
|
|| self.generation == 0
|
||||||
|
|| self.owner_pid == 0
|
||||||
|
|| self.session_id.is_empty()
|
||||||
|
|| self.session_id.len() > 128
|
||||||
|
|| !self
|
||||||
|
.session_id
|
||||||
|
.bytes()
|
||||||
|
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_'))
|
||||||
|
{
|
||||||
|
return invalid_ring("media ring descriptor identity is invalid");
|
||||||
|
}
|
||||||
|
if !Path::new(&self.ring_path).is_absolute() {
|
||||||
|
return invalid_ring("media ring path must be absolute");
|
||||||
|
}
|
||||||
|
if self.video_transport != "shared_memory_encoded_access_units"
|
||||||
|
|| self.audio_transport != "separate_transport"
|
||||||
|
{
|
||||||
|
return invalid_ring("media ring transport contract is invalid");
|
||||||
|
}
|
||||||
|
validate_geometry(self.slot_count, self.slot_bytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct MediaRing {
|
||||||
|
file: File,
|
||||||
|
map: MmapMut,
|
||||||
|
slot_count: u32,
|
||||||
|
slot_bytes: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MediaRing {
|
||||||
|
pub fn descriptor(
|
||||||
|
&self,
|
||||||
|
session_id: impl Into<String>,
|
||||||
|
generation: u64,
|
||||||
|
owner_pid: u32,
|
||||||
|
ring_path: impl AsRef<Path>,
|
||||||
|
) -> io::Result<MediaRingDescriptor> {
|
||||||
|
let descriptor = MediaRingDescriptor {
|
||||||
|
schema_version: VERSION as u16,
|
||||||
|
session_id: session_id.into(),
|
||||||
|
generation,
|
||||||
|
owner_pid,
|
||||||
|
ring_path: ring_path.as_ref().to_string_lossy().into_owned(),
|
||||||
|
slot_count: self.slot_count,
|
||||||
|
slot_bytes: self.slot_bytes,
|
||||||
|
video_transport: "shared_memory_encoded_access_units",
|
||||||
|
audio_transport: "separate_transport",
|
||||||
|
audio_ring: None,
|
||||||
|
};
|
||||||
|
descriptor.validate()?;
|
||||||
|
Ok(descriptor)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn audio_descriptor(
|
||||||
|
&self,
|
||||||
|
session_id: impl Into<String>,
|
||||||
|
generation: u64,
|
||||||
|
owner_pid: u32,
|
||||||
|
ring_path: impl AsRef<Path>,
|
||||||
|
) -> io::Result<AudioRingDescriptor> {
|
||||||
|
let descriptor = AudioRingDescriptor {
|
||||||
|
schema_version: VERSION as u16,
|
||||||
|
session_id: session_id.into(),
|
||||||
|
generation,
|
||||||
|
owner_pid,
|
||||||
|
ring_path: ring_path.as_ref().to_string_lossy().into_owned(),
|
||||||
|
slot_count: self.slot_count,
|
||||||
|
slot_bytes: self.slot_bytes,
|
||||||
|
audio_transport: "shared_memory_opus_packets",
|
||||||
|
};
|
||||||
|
descriptor.validate()?;
|
||||||
|
Ok(descriptor)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn create(path: impl AsRef<Path>, slot_count: u32, slot_bytes: u32) -> io::Result<Self> {
|
||||||
|
validate_geometry(slot_count, slot_bytes)?;
|
||||||
|
let path = path.as_ref();
|
||||||
|
#[cfg(windows)]
|
||||||
|
let file = crate::windows_security::create_current_user_file(path)?;
|
||||||
|
#[cfg(not(windows))]
|
||||||
|
let file = OpenOptions::new()
|
||||||
|
.read(true)
|
||||||
|
.write(true)
|
||||||
|
.create_new(true)
|
||||||
|
.open(path)?;
|
||||||
|
let length = ring_bytes(slot_count, slot_bytes)?;
|
||||||
|
file.set_len(length as u64)?;
|
||||||
|
let mut map = unsafe { MmapOptions::new().len(length).map_mut(&file)? };
|
||||||
|
unsafe {
|
||||||
|
std::ptr::write(
|
||||||
|
map.as_mut_ptr().cast::<RingHeader>(),
|
||||||
|
RingHeader {
|
||||||
|
magic: MAGIC,
|
||||||
|
version: VERSION,
|
||||||
|
slot_count,
|
||||||
|
slot_bytes,
|
||||||
|
reserved: 0,
|
||||||
|
write_seq: AtomicU64::new(0),
|
||||||
|
read_seq: AtomicU64::new(0),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(Self {
|
||||||
|
file,
|
||||||
|
map,
|
||||||
|
slot_count,
|
||||||
|
slot_bytes,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
|
pub fn open(path: impl AsRef<Path>) -> io::Result<Self> {
|
||||||
|
let file = OpenOptions::new().read(true).write(true).open(path)?;
|
||||||
|
let length = usize::try_from(file.metadata()?.len())
|
||||||
|
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "ring file is too large"))?;
|
||||||
|
if length < HEADER_BYTES {
|
||||||
|
return invalid_ring("ring file is shorter than its header");
|
||||||
|
}
|
||||||
|
let map = unsafe { MmapOptions::new().len(length).map_mut(&file)? };
|
||||||
|
let header = unsafe { &*map.as_ptr().cast::<RingHeader>() };
|
||||||
|
if header.magic != MAGIC || header.version != VERSION {
|
||||||
|
return invalid_ring("ring header magic or version is invalid");
|
||||||
|
}
|
||||||
|
validate_geometry(header.slot_count, header.slot_bytes)?;
|
||||||
|
if ring_bytes(header.slot_count, header.slot_bytes)? != length {
|
||||||
|
return invalid_ring("ring file size does not match its header");
|
||||||
|
}
|
||||||
|
Ok(Self {
|
||||||
|
file,
|
||||||
|
map,
|
||||||
|
slot_count: header.slot_count,
|
||||||
|
slot_bytes: header.slot_bytes,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pushes one encoded access unit. A full ring returns false so the
|
||||||
|
/// caller can drop this video packet without blocking input or audio.
|
||||||
|
pub fn push(&mut self, record: &MediaRecord) -> io::Result<bool> {
|
||||||
|
if record.payload.is_empty() || record.payload.len() > self.slot_bytes as usize {
|
||||||
|
return invalid_ring("media record exceeds ring slot capacity");
|
||||||
|
}
|
||||||
|
let write = self.header().write_seq.load(Ordering::Relaxed);
|
||||||
|
let read = self.header().read_seq.load(Ordering::Acquire);
|
||||||
|
if write.wrapping_sub(read) >= u64::from(self.slot_count) {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
let slot = self.slot_mut(write);
|
||||||
|
slot.payload[..record.payload.len()].copy_from_slice(&record.payload);
|
||||||
|
*slot.header = SlotHeader {
|
||||||
|
sequence: record.sequence,
|
||||||
|
pts: record.pts,
|
||||||
|
kind: record.kind,
|
||||||
|
length: record.payload.len() as u32,
|
||||||
|
};
|
||||||
|
self.header()
|
||||||
|
.write_seq
|
||||||
|
.store(write.wrapping_add(1), Ordering::Release);
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
|
pub fn pop(&mut self) -> io::Result<Option<MediaRecord>> {
|
||||||
|
let read = self.header().read_seq.load(Ordering::Relaxed);
|
||||||
|
let write = self.header().write_seq.load(Ordering::Acquire);
|
||||||
|
if read == write {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let slot_bytes = self.slot_bytes as usize;
|
||||||
|
let slot = self.slot_mut(read);
|
||||||
|
let length = usize::try_from(slot.header.length).map_err(|_| {
|
||||||
|
io::Error::new(io::ErrorKind::InvalidData, "ring record length overflow")
|
||||||
|
})?;
|
||||||
|
if length == 0 || length > slot_bytes {
|
||||||
|
return invalid_ring("ring record length is invalid");
|
||||||
|
}
|
||||||
|
let record = MediaRecord {
|
||||||
|
sequence: slot.header.sequence,
|
||||||
|
pts: slot.header.pts,
|
||||||
|
kind: slot.header.kind,
|
||||||
|
payload: slot.payload[..length].to_vec(),
|
||||||
|
};
|
||||||
|
self.header()
|
||||||
|
.read_seq
|
||||||
|
.store(read.wrapping_add(1), Ordering::Release);
|
||||||
|
Ok(Some(record))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn header(&self) -> &RingHeader {
|
||||||
|
debug_assert_eq!(self.map.as_ptr().align_offset(align_of::<RingHeader>()), 0);
|
||||||
|
unsafe { &*self.map.as_ptr().cast::<RingHeader>() }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn slot_mut(&mut self, sequence: u64) -> Slot<'_> {
|
||||||
|
let offset =
|
||||||
|
HEADER_BYTES + (sequence as usize % self.slot_count as usize) * self.slot_stride();
|
||||||
|
let pointer = unsafe { self.map.as_mut_ptr().add(offset) };
|
||||||
|
Slot {
|
||||||
|
header: unsafe { &mut *pointer.cast::<SlotHeader>() },
|
||||||
|
payload: unsafe {
|
||||||
|
std::slice::from_raw_parts_mut(
|
||||||
|
pointer.add(SLOT_HEADER_BYTES),
|
||||||
|
self.slot_bytes as usize,
|
||||||
|
)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn slot_stride(&self) -> usize {
|
||||||
|
SLOT_HEADER_BYTES + self.slot_bytes as usize
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Slot<'a> {
|
||||||
|
header: &'a mut SlotHeader,
|
||||||
|
payload: &'a mut [u8],
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_geometry(slot_count: u32, slot_bytes: u32) -> io::Result<()> {
|
||||||
|
if !(2..=MAX_SLOTS).contains(&slot_count) || !slot_count.is_power_of_two() {
|
||||||
|
return invalid_ring("slot count must be a power of two between 2 and 256");
|
||||||
|
}
|
||||||
|
if slot_bytes == 0 || slot_bytes > MAX_SLOT_BYTES {
|
||||||
|
return invalid_ring("slot capacity is invalid");
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ring_bytes(slot_count: u32, slot_bytes: u32) -> io::Result<usize> {
|
||||||
|
let slots = usize::try_from(slot_count)
|
||||||
|
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "slot count overflow"))?;
|
||||||
|
let capacity = usize::try_from(slot_bytes)
|
||||||
|
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "slot capacity overflow"))?;
|
||||||
|
HEADER_BYTES
|
||||||
|
.checked_add(
|
||||||
|
slots
|
||||||
|
.checked_mul(SLOT_HEADER_BYTES.checked_add(capacity).ok_or_else(|| {
|
||||||
|
io::Error::new(io::ErrorKind::InvalidInput, "ring size overflow")
|
||||||
|
})?)
|
||||||
|
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "ring size overflow"))?,
|
||||||
|
)
|
||||||
|
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "ring size overflow"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn invalid_ring<T>(message: &str) -> io::Result<T> {
|
||||||
|
Err(io::Error::new(io::ErrorKind::InvalidData, message))
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for MediaRing {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let _ = self.map.flush();
|
||||||
|
let _ = self.file.sync_data();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
fn temp_path() -> std::path::PathBuf {
|
||||||
|
let nonce = SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap()
|
||||||
|
.as_nanos();
|
||||||
|
std::env::temp_dir().join(format!("remotedesk-media-ring-{nonce}.bin"))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn producer_and_consumer_share_encoded_records() {
|
||||||
|
let path = temp_path();
|
||||||
|
let mut producer = MediaRing::create(&path, 2, 64).unwrap();
|
||||||
|
let mut consumer = MediaRing::open(&path).unwrap();
|
||||||
|
let record = MediaRecord {
|
||||||
|
sequence: 7,
|
||||||
|
pts: 11,
|
||||||
|
kind: 1,
|
||||||
|
payload: b"encoded-access-unit".to_vec(),
|
||||||
|
};
|
||||||
|
assert!(producer.push(&record).unwrap());
|
||||||
|
assert_eq!(consumer.pop().unwrap(), Some(record));
|
||||||
|
assert_eq!(consumer.pop().unwrap(), None);
|
||||||
|
drop(consumer);
|
||||||
|
drop(producer);
|
||||||
|
std::fs::remove_file(path).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn full_ring_does_not_block() {
|
||||||
|
let path = temp_path();
|
||||||
|
let mut ring = MediaRing::create(&path, 2, 8).unwrap();
|
||||||
|
let record = MediaRecord {
|
||||||
|
sequence: 1,
|
||||||
|
pts: 0,
|
||||||
|
kind: 1,
|
||||||
|
payload: vec![1],
|
||||||
|
};
|
||||||
|
assert!(ring.push(&record).unwrap());
|
||||||
|
assert!(ring.push(&record).unwrap());
|
||||||
|
assert!(!ring.push(&record).unwrap());
|
||||||
|
drop(ring);
|
||||||
|
std::fs::remove_file(path).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn descriptor_binds_session_and_encoded_only_transport() {
|
||||||
|
let path = temp_path();
|
||||||
|
let ring = MediaRing::create(&path, 2, 64).unwrap();
|
||||||
|
let descriptor = ring.descriptor("session-1", 3, 42, &path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
descriptor.video_transport,
|
||||||
|
"shared_memory_encoded_access_units"
|
||||||
|
);
|
||||||
|
assert!(descriptor.validate().is_ok());
|
||||||
|
drop(ring);
|
||||||
|
std::fs::remove_file(path).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn descriptor_rejects_relative_or_unbound_identity() {
|
||||||
|
let path = temp_path();
|
||||||
|
let ring = MediaRing::create(&path, 2, 64).unwrap();
|
||||||
|
assert!(ring.descriptor("bad session", 1, 42, &path).is_err());
|
||||||
|
assert!(ring.descriptor("session", 0, 42, &path).is_err());
|
||||||
|
assert!(
|
||||||
|
ring.descriptor("session", 1, 42, "relative-ring.bin")
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
drop(ring);
|
||||||
|
std::fs::remove_file(path).unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
use std::ffi::{OsStr, c_void};
|
||||||
|
use std::fs::File;
|
||||||
|
use std::io;
|
||||||
|
use std::mem;
|
||||||
|
use std::os::windows::ffi::OsStrExt as _;
|
||||||
|
use std::os::windows::io::{AsRawHandle as _, FromRawHandle as _, RawHandle};
|
||||||
|
use std::path::Path;
|
||||||
|
use std::ptr;
|
||||||
|
use tokio::net::windows::named_pipe::{NamedPipeServer, PipeMode, ServerOptions};
|
||||||
|
use windows_sys::Win32::Foundation::{
|
||||||
|
ERROR_INSUFFICIENT_BUFFER, GENERIC_READ, GENERIC_WRITE, HANDLE, INVALID_HANDLE_VALUE, LocalFree,
|
||||||
|
};
|
||||||
|
use windows_sys::Win32::Security::Authorization::{
|
||||||
|
ConvertSidToStringSidW, ConvertStringSecurityDescriptorToSecurityDescriptorW, SDDL_REVISION_1,
|
||||||
|
};
|
||||||
|
use windows_sys::Win32::Security::{
|
||||||
|
GetTokenInformation, PSECURITY_DESCRIPTOR, SECURITY_ATTRIBUTES, TOKEN_QUERY, TOKEN_USER,
|
||||||
|
TokenUser,
|
||||||
|
};
|
||||||
|
use windows_sys::Win32::Storage::FileSystem::{
|
||||||
|
CREATE_NEW, CreateFileW, FILE_ATTRIBUTE_TEMPORARY, FILE_SHARE_DELETE, FILE_SHARE_READ,
|
||||||
|
FILE_SHARE_WRITE,
|
||||||
|
};
|
||||||
|
use windows_sys::Win32::System::Threading::{GetCurrentProcess, OpenProcessToken};
|
||||||
|
|
||||||
|
struct CurrentUserSecurity {
|
||||||
|
descriptor: PSECURITY_DESCRIPTOR,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CurrentUserSecurity {
|
||||||
|
fn new() -> io::Result<Self> {
|
||||||
|
let sid = current_user_sid()?;
|
||||||
|
let sddl = wide(OsStr::new(&format!("D:P(A;;GA;;;{sid})")));
|
||||||
|
let mut descriptor: PSECURITY_DESCRIPTOR = ptr::null_mut();
|
||||||
|
// SAFETY: the SDDL is terminated and descriptor is writable output storage.
|
||||||
|
if unsafe {
|
||||||
|
ConvertStringSecurityDescriptorToSecurityDescriptorW(
|
||||||
|
sddl.as_ptr(),
|
||||||
|
SDDL_REVISION_1,
|
||||||
|
&raw mut descriptor,
|
||||||
|
ptr::null_mut(),
|
||||||
|
)
|
||||||
|
} == 0
|
||||||
|
{
|
||||||
|
return Err(io::Error::last_os_error());
|
||||||
|
}
|
||||||
|
Ok(Self { descriptor })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn attributes(&mut self) -> io::Result<SECURITY_ATTRIBUTES> {
|
||||||
|
Ok(SECURITY_ATTRIBUTES {
|
||||||
|
nLength: u32::try_from(mem::size_of::<SECURITY_ATTRIBUTES>())
|
||||||
|
.map_err(|_| io::Error::other("SECURITY_ATTRIBUTES size is not representable"))?,
|
||||||
|
lpSecurityDescriptor: self.descriptor.cast(),
|
||||||
|
bInheritHandle: 0,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for CurrentUserSecurity {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if !self.descriptor.is_null() {
|
||||||
|
// SAFETY: the descriptor was allocated by the SDDL conversion API.
|
||||||
|
unsafe { LocalFree(self.descriptor.cast()) };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn create_current_user_pipe(name: &str, first_instance: bool) -> io::Result<NamedPipeServer> {
|
||||||
|
let mut security = CurrentUserSecurity::new()?;
|
||||||
|
let mut attributes = security.attributes()?;
|
||||||
|
let mut options = ServerOptions::new();
|
||||||
|
options
|
||||||
|
.pipe_mode(PipeMode::Byte)
|
||||||
|
.max_instances(16)
|
||||||
|
.reject_remote_clients(true)
|
||||||
|
.first_pipe_instance(first_instance);
|
||||||
|
// SAFETY: attributes and its descriptor remain alive for the complete creation call.
|
||||||
|
unsafe {
|
||||||
|
options.create_with_security_attributes_raw(name, (&raw mut attributes).cast::<c_void>())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn create_current_user_file(path: &Path) -> io::Result<File> {
|
||||||
|
let mut security = CurrentUserSecurity::new()?;
|
||||||
|
let mut attributes = security.attributes()?;
|
||||||
|
let path = wide(path.as_os_str());
|
||||||
|
// SAFETY: all pointers are valid for the duration of CreateFileW and the
|
||||||
|
// security descriptor is owned by `security` until the call returns.
|
||||||
|
let handle = unsafe {
|
||||||
|
CreateFileW(
|
||||||
|
path.as_ptr(),
|
||||||
|
GENERIC_READ | GENERIC_WRITE,
|
||||||
|
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
|
||||||
|
&raw mut attributes,
|
||||||
|
CREATE_NEW,
|
||||||
|
FILE_ATTRIBUTE_TEMPORARY,
|
||||||
|
ptr::null_mut(),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
if handle == INVALID_HANDLE_VALUE {
|
||||||
|
return Err(io::Error::last_os_error());
|
||||||
|
}
|
||||||
|
// SAFETY: CreateFileW returned a new owned kernel handle.
|
||||||
|
Ok(unsafe { File::from_raw_handle(handle as RawHandle) })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn current_user_sid() -> io::Result<String> {
|
||||||
|
let mut token: HANDLE = ptr::null_mut();
|
||||||
|
// SAFETY: token points to writable storage and the process pseudo-handle is valid.
|
||||||
|
if unsafe { OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &raw mut token) } == 0 {
|
||||||
|
return Err(io::Error::last_os_error());
|
||||||
|
}
|
||||||
|
// SAFETY: OpenProcessToken returned an owned token handle.
|
||||||
|
let token = unsafe { File::from_raw_handle(token as RawHandle) };
|
||||||
|
let mut required = 0_u32;
|
||||||
|
// SAFETY: a null buffer with zero length is the documented size query.
|
||||||
|
unsafe {
|
||||||
|
GetTokenInformation(
|
||||||
|
token.as_raw_handle() as HANDLE,
|
||||||
|
TokenUser,
|
||||||
|
ptr::null_mut(),
|
||||||
|
0,
|
||||||
|
&raw mut required,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let size_error = io::Error::last_os_error();
|
||||||
|
if size_error.raw_os_error().map(i32::cast_unsigned) != Some(ERROR_INSUFFICIENT_BUFFER)
|
||||||
|
|| required == 0
|
||||||
|
{
|
||||||
|
return Err(size_error);
|
||||||
|
}
|
||||||
|
let words = (required as usize).div_ceil(mem::size_of::<usize>());
|
||||||
|
let mut buffer = vec![0_usize; words];
|
||||||
|
// SAFETY: buffer contains at least `required` writable bytes.
|
||||||
|
if unsafe {
|
||||||
|
GetTokenInformation(
|
||||||
|
token.as_raw_handle() as HANDLE,
|
||||||
|
TokenUser,
|
||||||
|
buffer.as_mut_ptr().cast::<c_void>(),
|
||||||
|
required,
|
||||||
|
&raw mut required,
|
||||||
|
)
|
||||||
|
} == 0
|
||||||
|
{
|
||||||
|
return Err(io::Error::last_os_error());
|
||||||
|
}
|
||||||
|
// SAFETY: TokenUser queries return a TOKEN_USER at the start of the buffer.
|
||||||
|
let token_user = unsafe { &*buffer.as_ptr().cast::<TOKEN_USER>() };
|
||||||
|
let mut sid_string = ptr::null_mut();
|
||||||
|
// SAFETY: the SID is owned by the token information buffer and output is writable.
|
||||||
|
if unsafe { ConvertSidToStringSidW(token_user.User.Sid, &raw mut sid_string) } == 0 {
|
||||||
|
return Err(io::Error::last_os_error());
|
||||||
|
}
|
||||||
|
let mut length = 0_usize;
|
||||||
|
// SAFETY: ConvertSidToStringSidW returns a terminated LocalAlloc string.
|
||||||
|
unsafe {
|
||||||
|
while *sid_string.add(length) != 0 {
|
||||||
|
length += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// SAFETY: the preceding scan established the initialized UTF-16 range.
|
||||||
|
let sid = String::from_utf16(unsafe { std::slice::from_raw_parts(sid_string, length) })
|
||||||
|
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "current SID is invalid"));
|
||||||
|
// SAFETY: the SID string was allocated by ConvertSidToStringSidW.
|
||||||
|
unsafe { LocalFree(sid_string.cast()) };
|
||||||
|
sid
|
||||||
|
}
|
||||||
|
|
||||||
|
fn wide(value: &OsStr) -> Vec<u16> {
|
||||||
|
value.encode_wide().chain([0]).collect()
|
||||||
|
}
|
||||||
@@ -1065,7 +1065,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
request.gpu_policy = GpuPolicy {
|
request.gpu_policy = GpuPolicy {
|
||||||
selection: GpuSelection::Manual(adapter("gpu-1")),
|
selection: GpuSelection::Manual(adapter("gpu-1")),
|
||||||
zero_copy_policy: ZeroCopyPolicy::Compatibility,
|
zero_copy_policy: ZeroCopyPolicy::RequiredEndToEnd,
|
||||||
};
|
};
|
||||||
request.decoder_adapter = adapter("gpu-1");
|
request.decoder_adapter = adapter("gpu-1");
|
||||||
let plan = planner.plan(request).unwrap();
|
let plan = planner.plan(request).unwrap();
|
||||||
|
|||||||
@@ -220,7 +220,7 @@ mod tests {
|
|||||||
let plan = plan_d3d11_pipeline(PipelineRequest {
|
let plan = plan_d3d11_pipeline(PipelineRequest {
|
||||||
policy: GpuPolicy {
|
policy: GpuPolicy {
|
||||||
selection: GpuSelection::WindowDisplayAdapter,
|
selection: GpuSelection::WindowDisplayAdapter,
|
||||||
zero_copy_policy: ZeroCopyPolicy::Compatibility,
|
zero_copy_policy: ZeroCopyPolicy::RequiredEndToEnd,
|
||||||
},
|
},
|
||||||
window_display_adapter: adapter("render"),
|
window_display_adapter: adapter("render"),
|
||||||
decoder_adapter: adapter("decode"),
|
decoder_adapter: adapter("decode"),
|
||||||
@@ -229,7 +229,7 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
assert_eq!(plan.planned_memory_path, MemoryPathStatus::CrossAdapterCopy);
|
assert_eq!(plan.planned_memory_path, MemoryPathStatus::CrossAdapterCopy);
|
||||||
assert_eq!(plan.zero_copy_policy, ZeroCopyPolicy::Compatibility);
|
assert_eq!(plan.zero_copy_policy, ZeroCopyPolicy::RequiredEndToEnd);
|
||||||
assert!(!plan.is_zero_copy_candidate());
|
assert!(!plan.is_zero_copy_candidate());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -238,7 +238,7 @@ mod tests {
|
|||||||
let plan = plan_d3d11_pipeline(PipelineRequest {
|
let plan = plan_d3d11_pipeline(PipelineRequest {
|
||||||
policy: GpuPolicy {
|
policy: GpuPolicy {
|
||||||
selection: GpuSelection::Manual(adapter("manual-gpu")),
|
selection: GpuSelection::Manual(adapter("manual-gpu")),
|
||||||
zero_copy_policy: ZeroCopyPolicy::Compatibility,
|
zero_copy_policy: ZeroCopyPolicy::RequiredEndToEnd,
|
||||||
},
|
},
|
||||||
window_display_adapter: adapter("display-gpu"),
|
window_display_adapter: adapter("display-gpu"),
|
||||||
decoder_adapter: adapter("manual-gpu"),
|
decoder_adapter: adapter("manual-gpu"),
|
||||||
|
|||||||
@@ -0,0 +1,2 @@
|
|||||||
|
#[cfg(windows)]
|
||||||
|
pub mod stream;
|
||||||
@@ -112,7 +112,6 @@ fn parse_args(mut args: impl Iterator<Item = String>) -> Result<DryRunArgs, Stri
|
|||||||
manual_adapter = Some(AdapterId::new(required_value(&mut args, &argument)?));
|
manual_adapter = Some(AdapterId::new(required_value(&mut args, &argument)?));
|
||||||
}
|
}
|
||||||
"--required-end-to-end" => zero_copy_policy = ZeroCopyPolicy::RequiredEndToEnd,
|
"--required-end-to-end" => zero_copy_policy = ZeroCopyPolicy::RequiredEndToEnd,
|
||||||
"--compatibility" => zero_copy_policy = ZeroCopyPolicy::Compatibility,
|
|
||||||
"--help" | "-h" => return Err(usage().to_owned()),
|
"--help" | "-h" => return Err(usage().to_owned()),
|
||||||
other => return Err(format!("unknown argument {other}")),
|
other => return Err(format!("unknown argument {other}")),
|
||||||
}
|
}
|
||||||
@@ -141,7 +140,7 @@ fn required_value(args: &mut impl Iterator<Item = String>, option: &str) -> Resu
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn usage() -> &'static str {
|
fn usage() -> &'static str {
|
||||||
"usage: remotedesk-native-video --capabilities | --probe-d3d11 | --probe-h264-decoder | --probe-h264-file <local.mp4> | --play-h264-file <local.mp4> | --dry-run --window-adapter <id> [--decode-adapter <id>] [--manual-adapter <id>] [--required-end-to-end|--compatibility]"
|
"usage: remotedesk-native-video --capabilities | --probe-d3d11 | --probe-h264-decoder | --probe-h264-file <local.mp4> | --play-h264-file <local.mp4> | --dry-run --window-adapter <id> [--decode-adapter <id>] [--manual-adapter <id>]"
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(serde::Serialize)]
|
#[derive(serde::Serialize)]
|
||||||
@@ -817,7 +816,7 @@ mod tests {
|
|||||||
"gpu-0",
|
"gpu-0",
|
||||||
"--manual-adapter",
|
"--manual-adapter",
|
||||||
"gpu-1",
|
"gpu-1",
|
||||||
"--compatibility",
|
"--required-end-to-end",
|
||||||
]))
|
]))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
@@ -841,12 +840,12 @@ mod tests {
|
|||||||
"--dry-run",
|
"--dry-run",
|
||||||
"--window-adapter",
|
"--window-adapter",
|
||||||
"gpu-0",
|
"gpu-0",
|
||||||
"--compatibility",
|
"--required-end-to-end",
|
||||||
]))
|
]))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
compatibility.zero_copy_policy,
|
compatibility.zero_copy_policy,
|
||||||
ZeroCopyPolicy::Compatibility
|
ZeroCopyPolicy::RequiredEndToEnd
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,797 @@
|
|||||||
|
use core::mem::ManuallyDrop;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use raw_window_handle::{HasWindowHandle as _, RawWindowHandle};
|
||||||
|
use windows::Win32::Foundation::{HWND, RECT};
|
||||||
|
use windows::Win32::Graphics::Direct3D::{
|
||||||
|
D3D_DRIVER_TYPE_HARDWARE, D3D_FEATURE_LEVEL_11_0, D3D_FEATURE_LEVEL_11_1,
|
||||||
|
};
|
||||||
|
use windows::Win32::Graphics::Direct3D11::{
|
||||||
|
D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_SDK_VERSION,
|
||||||
|
D3D11_TEX2D_VPIV, D3D11_TEX2D_VPOV, D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE,
|
||||||
|
D3D11_VIDEO_PROCESSOR_COLOR_SPACE, D3D11_VIDEO_PROCESSOR_CONTENT_DESC,
|
||||||
|
D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_INPUT, D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT,
|
||||||
|
D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC, D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC_0,
|
||||||
|
D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC, D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC_0,
|
||||||
|
D3D11_VIDEO_PROCESSOR_STREAM, D3D11_VIDEO_USAGE_PLAYBACK_NORMAL,
|
||||||
|
D3D11_VPIV_DIMENSION_TEXTURE2D, D3D11_VPOV_DIMENSION_TEXTURE2D, D3D11CreateDevice,
|
||||||
|
ID3D11Device, ID3D11DeviceContext, ID3D11RenderTargetView, ID3D11Texture2D, ID3D11VideoContext,
|
||||||
|
ID3D11VideoDevice, ID3D11VideoProcessor, ID3D11VideoProcessorEnumerator,
|
||||||
|
ID3D11VideoProcessorOutputView,
|
||||||
|
};
|
||||||
|
use windows::Win32::Graphics::Dxgi::Common::{
|
||||||
|
DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_NV12, DXGI_FORMAT_UNKNOWN, DXGI_MODE_DESC,
|
||||||
|
DXGI_RATIONAL, DXGI_SAMPLE_DESC,
|
||||||
|
};
|
||||||
|
use windows::Win32::Graphics::Dxgi::{
|
||||||
|
DXGI_MWA_NO_ALT_ENTER, DXGI_PRESENT, DXGI_SWAP_CHAIN_DESC, DXGI_SWAP_CHAIN_FLAG,
|
||||||
|
DXGI_SWAP_EFFECT_DISCARD, DXGI_USAGE_RENDER_TARGET_OUTPUT, IDXGIAdapter, IDXGIDevice,
|
||||||
|
IDXGIFactory, IDXGISwapChain,
|
||||||
|
};
|
||||||
|
use windows::Win32::Media::MediaFoundation::{
|
||||||
|
CLSID_MSH264DecoderMFT, IMFAttributes, IMFDXGIBuffer, IMFDXGIDeviceManager, IMFMediaType,
|
||||||
|
IMFSample, IMFTransform, MF_E_NO_MORE_TYPES, MF_E_NOTACCEPTING, MF_E_TRANSFORM_NEED_MORE_INPUT,
|
||||||
|
MF_E_TRANSFORM_STREAM_CHANGE, MF_MT_FRAME_SIZE, MF_MT_MAJOR_TYPE, MF_MT_SUBTYPE,
|
||||||
|
MF_MT_VIDEO_NOMINAL_RANGE, MF_MT_YUV_MATRIX, MF_SA_D3D11_AWARE, MF_VERSION,
|
||||||
|
MFCreateDXGIDeviceManager, MFCreateMediaType, MFCreateMemoryBuffer, MFCreateSample,
|
||||||
|
MFMediaType_Video, MFNominalRange_0_255, MFSTARTUP_FULL, MFStartup,
|
||||||
|
MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, MFT_MESSAGE_NOTIFY_START_OF_STREAM,
|
||||||
|
MFT_MESSAGE_SET_D3D_MANAGER, MFT_OUTPUT_DATA_BUFFER, MFT_OUTPUT_STREAM_CAN_PROVIDE_SAMPLES,
|
||||||
|
MFT_OUTPUT_STREAM_PROVIDES_SAMPLES, MFVideoFormat_H264, MFVideoFormat_H264_ES,
|
||||||
|
MFVideoFormat_NV12, MFVideoTransferMatrix_BT709,
|
||||||
|
};
|
||||||
|
use windows::Win32::System::Com::{
|
||||||
|
CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED, CoCreateInstance, CoInitializeEx,
|
||||||
|
};
|
||||||
|
use windows::core::Interface as _;
|
||||||
|
use winit::window::Window;
|
||||||
|
|
||||||
|
const MAX_OUTPUT_TYPES: u32 = 64;
|
||||||
|
const MAX_OUTPUTS_PER_INPUT: usize = 8;
|
||||||
|
|
||||||
|
pub struct H264AccessUnit {
|
||||||
|
pub data: Vec<u8>,
|
||||||
|
pub duration: std::time::Duration,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct NativeH264Renderer {
|
||||||
|
decoder: StreamingDecoder,
|
||||||
|
renderer: VideoRenderer,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct NativeVideoPresentation {
|
||||||
|
pub decode_latency_us: u64,
|
||||||
|
pub presentation_latency_us: u64,
|
||||||
|
pub frames: usize,
|
||||||
|
pub width: u32,
|
||||||
|
pub height: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NativeH264Renderer {
|
||||||
|
pub fn new(window: Arc<Window>) -> Result<Self, String> {
|
||||||
|
let decoder = StreamingDecoder::new()?;
|
||||||
|
let renderer = VideoRenderer::new(window, decoder.device.clone(), decoder.context.clone())?;
|
||||||
|
Ok(Self { decoder, renderer })
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn decode_and_present(
|
||||||
|
&mut self,
|
||||||
|
access_unit: &H264AccessUnit,
|
||||||
|
) -> Result<Option<NativeVideoPresentation>, String> {
|
||||||
|
let decode_started = Instant::now();
|
||||||
|
let frames = self.decoder.decode(access_unit)?;
|
||||||
|
let decode_latency_us = elapsed_microseconds(decode_started);
|
||||||
|
if frames.is_empty() {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let presentation_started = Instant::now();
|
||||||
|
let frame_count = frames.len();
|
||||||
|
let (width, height) = frames
|
||||||
|
.last()
|
||||||
|
.map(|frame| (frame.width, frame.height))
|
||||||
|
.expect("non-empty decoded frame list");
|
||||||
|
for frame in frames {
|
||||||
|
self.renderer.present(&frame)?;
|
||||||
|
}
|
||||||
|
Ok(Some(NativeVideoPresentation {
|
||||||
|
decode_latency_us,
|
||||||
|
presentation_latency_us: elapsed_microseconds(presentation_started),
|
||||||
|
frames: frame_count,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ComGuard;
|
||||||
|
|
||||||
|
impl Drop for ComGuard {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
unsafe { windows::Win32::System::Com::CoUninitialize() };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct MediaFoundationGuard;
|
||||||
|
|
||||||
|
impl Drop for MediaFoundationGuard {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let _ = unsafe { windows::Win32::Media::MediaFoundation::MFShutdown() };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct StreamingDecoder {
|
||||||
|
transform: IMFTransform,
|
||||||
|
_input_type: IMFMediaType,
|
||||||
|
_manager: IMFDXGIDeviceManager,
|
||||||
|
_attributes: IMFAttributes,
|
||||||
|
device: ID3D11Device,
|
||||||
|
context: ID3D11DeviceContext,
|
||||||
|
next_timestamp_100ns: i64,
|
||||||
|
_media_foundation_guard: MediaFoundationGuard,
|
||||||
|
_com_guard: ComGuard,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StreamingDecoder {
|
||||||
|
fn new() -> Result<Self, String> {
|
||||||
|
unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) }
|
||||||
|
.ok()
|
||||||
|
.map_err(|error| format!("unable to initialize COM for remote H.264: {error}"))?;
|
||||||
|
let com_guard = ComGuard;
|
||||||
|
unsafe { MFStartup(MF_VERSION, MFSTARTUP_FULL) }
|
||||||
|
.map_err(|error| format!("unable to start Media Foundation: {error}"))?;
|
||||||
|
let media_foundation_guard = MediaFoundationGuard;
|
||||||
|
let (device, context) = create_hardware_d3d11_device()?;
|
||||||
|
let mut reset_token = 0_u32;
|
||||||
|
let mut manager = None;
|
||||||
|
unsafe { MFCreateDXGIDeviceManager(&raw mut reset_token, &raw mut manager) }
|
||||||
|
.map_err(|error| format!("unable to create remote-video DXGI manager: {error}"))?;
|
||||||
|
let manager =
|
||||||
|
manager.ok_or_else(|| "Media Foundation returned no DXGI manager".to_owned())?;
|
||||||
|
unsafe { manager.ResetDevice(&device, reset_token) }
|
||||||
|
.map_err(|error| format!("unable to bind the remote-video D3D11 device: {error}"))?;
|
||||||
|
|
||||||
|
let transform: IMFTransform =
|
||||||
|
unsafe { CoCreateInstance(&CLSID_MSH264DecoderMFT, None, CLSCTX_INPROC_SERVER) }
|
||||||
|
.map_err(|error| {
|
||||||
|
format!("unable to create the Windows H.264 decoder MFT: {error}")
|
||||||
|
})?;
|
||||||
|
let attributes = unsafe { transform.GetAttributes() }
|
||||||
|
.map_err(|error| format!("unable to read H.264 decoder attributes: {error}"))?;
|
||||||
|
let d3d11_aware_key = MF_SA_D3D11_AWARE;
|
||||||
|
if unsafe { attributes.GetUINT32(&d3d11_aware_key) }.unwrap_or_default() != 1 {
|
||||||
|
return Err("the Windows H.264 decoder is not D3D11-aware".to_owned());
|
||||||
|
}
|
||||||
|
unsafe { transform.ProcessMessage(MFT_MESSAGE_SET_D3D_MANAGER, manager.as_raw() as usize) }
|
||||||
|
.map_err(|error| format!("unable to attach D3D11 to the H.264 decoder: {error}"))?;
|
||||||
|
|
||||||
|
let input_type = select_h264_input(&transform)?;
|
||||||
|
select_nv12_output(&transform)?;
|
||||||
|
let output_info = unsafe { transform.GetOutputStreamInfo(0) }.map_err(|error| {
|
||||||
|
format!("unable to read H.264 decoder output requirements: {error}")
|
||||||
|
})?;
|
||||||
|
let sample_flags = u32::try_from(
|
||||||
|
(MFT_OUTPUT_STREAM_PROVIDES_SAMPLES.0 | MFT_OUTPUT_STREAM_CAN_PROVIDE_SAMPLES.0).max(0),
|
||||||
|
)
|
||||||
|
.unwrap_or_default();
|
||||||
|
if output_info.dwFlags & sample_flags == 0 {
|
||||||
|
return Err("the D3D11 H.264 decoder requires CPU-allocated output samples".to_owned());
|
||||||
|
}
|
||||||
|
unsafe { transform.ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0) }
|
||||||
|
.map_err(|error| format!("unable to begin H.264 streaming: {error}"))?;
|
||||||
|
unsafe { transform.ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0) }
|
||||||
|
.map_err(|error| format!("unable to start H.264 input: {error}"))?;
|
||||||
|
Ok(Self {
|
||||||
|
transform,
|
||||||
|
_input_type: input_type,
|
||||||
|
_manager: manager,
|
||||||
|
_attributes: attributes,
|
||||||
|
device,
|
||||||
|
context,
|
||||||
|
next_timestamp_100ns: 0,
|
||||||
|
_media_foundation_guard: media_foundation_guard,
|
||||||
|
_com_guard: com_guard,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode(&mut self, access_unit: &H264AccessUnit) -> Result<Vec<DecodedFrame>, String> {
|
||||||
|
if access_unit.data.is_empty() || access_unit.data.len() > 16 * 1024 * 1024 {
|
||||||
|
return Err("remote H.264 access unit is outside the decoder bounds".to_owned());
|
||||||
|
}
|
||||||
|
let mut frames = self.drain_output()?;
|
||||||
|
let input = create_input_sample(
|
||||||
|
&access_unit.data,
|
||||||
|
self.next_timestamp_100ns,
|
||||||
|
access_unit.duration,
|
||||||
|
)?;
|
||||||
|
let duration_100ns = i64::try_from(access_unit.duration.as_nanos() / 100)
|
||||||
|
.map_err(|_| "remote H.264 duration overflow".to_owned())?
|
||||||
|
.max(1);
|
||||||
|
self.next_timestamp_100ns = self.next_timestamp_100ns.saturating_add(duration_100ns);
|
||||||
|
match unsafe { self.transform.ProcessInput(0, &input, 0) } {
|
||||||
|
Ok(()) => {}
|
||||||
|
Err(error) if error.code() == MF_E_NOTACCEPTING => {
|
||||||
|
frames.extend(self.drain_output()?);
|
||||||
|
unsafe { self.transform.ProcessInput(0, &input, 0) }
|
||||||
|
.map_err(|retry| format!("H.264 decoder still rejects input: {retry}"))?;
|
||||||
|
}
|
||||||
|
Err(error) => return Err(format!("H.264 decoder rejected input: {error}")),
|
||||||
|
}
|
||||||
|
frames.extend(self.drain_output()?);
|
||||||
|
Ok(frames)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drain_output(&mut self) -> Result<Vec<DecodedFrame>, String> {
|
||||||
|
let mut frames = Vec::new();
|
||||||
|
for _ in 0..MAX_OUTPUTS_PER_INPUT {
|
||||||
|
let mut output = MFT_OUTPUT_DATA_BUFFER::default();
|
||||||
|
output.pSample = ManuallyDrop::new(None);
|
||||||
|
let mut status = 0_u32;
|
||||||
|
let result = unsafe {
|
||||||
|
self.transform
|
||||||
|
.ProcessOutput(0, std::slice::from_mut(&mut output), &raw mut status)
|
||||||
|
};
|
||||||
|
let sample = unsafe { ManuallyDrop::take(&mut output.pSample) };
|
||||||
|
let _events = unsafe { ManuallyDrop::take(&mut output.pEvents) };
|
||||||
|
match result {
|
||||||
|
Ok(()) => {
|
||||||
|
let sample = sample.ok_or_else(|| {
|
||||||
|
"H.264 decoder returned success without a DXGI sample".to_owned()
|
||||||
|
})?;
|
||||||
|
frames.push(decoded_frame(sample, &self.transform, &self.device)?);
|
||||||
|
}
|
||||||
|
Err(error) if error.code() == MF_E_TRANSFORM_NEED_MORE_INPUT => break,
|
||||||
|
Err(error) if error.code() == MF_E_TRANSFORM_STREAM_CHANGE => {
|
||||||
|
select_nv12_output(&self.transform)?;
|
||||||
|
}
|
||||||
|
Err(error) => return Err(format!("H.264 decoder output failed: {error}")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(frames)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn select_h264_input(transform: &IMFTransform) -> Result<IMFMediaType, String> {
|
||||||
|
let mut last_error = None;
|
||||||
|
for subtype in [MFVideoFormat_H264_ES, MFVideoFormat_H264] {
|
||||||
|
let media_type = unsafe { MFCreateMediaType() }
|
||||||
|
.map_err(|error| format!("unable to create the H.264 input media type: {error}"))?;
|
||||||
|
let major_type_key = MF_MT_MAJOR_TYPE;
|
||||||
|
let video_type = MFMediaType_Video;
|
||||||
|
unsafe { media_type.SetGUID(&major_type_key, &video_type) }
|
||||||
|
.map_err(|error| format!("unable to set the H.264 input major type: {error}"))?;
|
||||||
|
let subtype_key = MF_MT_SUBTYPE;
|
||||||
|
unsafe { media_type.SetGUID(&subtype_key, &subtype) }
|
||||||
|
.map_err(|error| format!("unable to set the H.264 input subtype: {error}"))?;
|
||||||
|
match unsafe { transform.SetInputType(0, &media_type, 0) } {
|
||||||
|
Ok(()) => return Ok(media_type),
|
||||||
|
Err(error) => last_error = Some(error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(format!(
|
||||||
|
"Windows H.264 decoder rejected Annex-B input: {}",
|
||||||
|
last_error
|
||||||
|
.map(|error| error.to_string())
|
||||||
|
.unwrap_or_else(|| "no supported input subtype".to_owned())
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_input_sample(
|
||||||
|
data: &[u8],
|
||||||
|
timestamp_100ns: i64,
|
||||||
|
duration: std::time::Duration,
|
||||||
|
) -> Result<IMFSample, String> {
|
||||||
|
let length = u32::try_from(data.len()).map_err(|_| "H.264 input is too large".to_owned())?;
|
||||||
|
let buffer = unsafe { MFCreateMemoryBuffer(length) }
|
||||||
|
.map_err(|error| format!("unable to allocate H.264 input buffer: {error}"))?;
|
||||||
|
let mut destination = core::ptr::null_mut();
|
||||||
|
unsafe { buffer.Lock(&raw mut destination, None, None) }
|
||||||
|
.map_err(|error| format!("unable to lock H.264 input buffer: {error}"))?;
|
||||||
|
if destination.is_null() {
|
||||||
|
let _ = unsafe { buffer.Unlock() };
|
||||||
|
return Err("Media Foundation returned a null H.264 input buffer".to_owned());
|
||||||
|
}
|
||||||
|
unsafe { core::ptr::copy_nonoverlapping(data.as_ptr(), destination, data.len()) };
|
||||||
|
unsafe { buffer.Unlock() }
|
||||||
|
.map_err(|error| format!("unable to unlock H.264 input buffer: {error}"))?;
|
||||||
|
unsafe { buffer.SetCurrentLength(length) }
|
||||||
|
.map_err(|error| format!("unable to commit H.264 input bytes: {error}"))?;
|
||||||
|
let sample = unsafe { MFCreateSample() }
|
||||||
|
.map_err(|error| format!("unable to create H.264 input sample: {error}"))?;
|
||||||
|
unsafe { sample.AddBuffer(&buffer) }
|
||||||
|
.map_err(|error| format!("unable to attach H.264 input bytes: {error}"))?;
|
||||||
|
unsafe { sample.SetSampleTime(timestamp_100ns) }
|
||||||
|
.map_err(|error| format!("unable to set H.264 sample time: {error}"))?;
|
||||||
|
let duration_100ns = i64::try_from(duration.as_nanos() / 100)
|
||||||
|
.map_err(|_| "H.264 sample duration overflow".to_owned())?
|
||||||
|
.max(1);
|
||||||
|
unsafe { sample.SetSampleDuration(duration_100ns) }
|
||||||
|
.map_err(|error| format!("unable to set H.264 sample duration: {error}"))?;
|
||||||
|
Ok(sample)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn select_nv12_output(transform: &IMFTransform) -> Result<(), String> {
|
||||||
|
for index in 0..MAX_OUTPUT_TYPES {
|
||||||
|
let media_type = match unsafe { transform.GetOutputAvailableType(0, index) } {
|
||||||
|
Ok(media_type) => media_type,
|
||||||
|
Err(error) if error.code() == MF_E_NO_MORE_TYPES => break,
|
||||||
|
Err(error) => {
|
||||||
|
return Err(format!(
|
||||||
|
"unable to enumerate decoder output {index}: {error}"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let subtype_key = MF_MT_SUBTYPE;
|
||||||
|
let subtype = unsafe { media_type.GetGUID(&subtype_key) };
|
||||||
|
if subtype.is_ok_and(|value| value == MFVideoFormat_NV12)
|
||||||
|
&& unsafe { transform.SetOutputType(0, &media_type, 0) }.is_ok()
|
||||||
|
{
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err("the Windows H.264 decoder exposes no NV12 output".to_owned())
|
||||||
|
}
|
||||||
|
|
||||||
|
struct DecodedFrame {
|
||||||
|
_sample: IMFSample,
|
||||||
|
texture: ID3D11Texture2D,
|
||||||
|
subresource: u32,
|
||||||
|
mip_levels: u32,
|
||||||
|
width: u32,
|
||||||
|
height: u32,
|
||||||
|
input_color_space: D3D11_VIDEO_PROCESSOR_COLOR_SPACE,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decoded_frame(
|
||||||
|
sample: IMFSample,
|
||||||
|
transform: &IMFTransform,
|
||||||
|
expected_device: &ID3D11Device,
|
||||||
|
) -> Result<DecodedFrame, String> {
|
||||||
|
let buffer_count = unsafe { sample.GetBufferCount() }
|
||||||
|
.map_err(|error| format!("unable to count decoded H.264 buffers: {error}"))?;
|
||||||
|
if buffer_count != 1 {
|
||||||
|
return Err("decoded H.264 frame must contain one DXGI buffer".to_owned());
|
||||||
|
}
|
||||||
|
let buffer = unsafe { sample.GetBufferByIndex(0) }
|
||||||
|
.map_err(|error| format!("unable to read decoded H.264 buffer: {error}"))?;
|
||||||
|
let dxgi_buffer = buffer
|
||||||
|
.cast::<IMFDXGIBuffer>()
|
||||||
|
.map_err(|_| "H.264 decoder returned a CPU media buffer".to_owned())?;
|
||||||
|
let mut raw_texture = core::ptr::null_mut();
|
||||||
|
unsafe { dxgi_buffer.GetResource(&ID3D11Texture2D::IID, &raw mut raw_texture) }
|
||||||
|
.map_err(|error| format!("decoded H.264 buffer has no D3D11 texture: {error}"))?;
|
||||||
|
if raw_texture.is_null() {
|
||||||
|
return Err("decoded H.264 buffer returned a null texture".to_owned());
|
||||||
|
}
|
||||||
|
let texture = unsafe { ID3D11Texture2D::from_raw(raw_texture) };
|
||||||
|
let mut descriptor = windows::Win32::Graphics::Direct3D11::D3D11_TEXTURE2D_DESC::default();
|
||||||
|
unsafe { texture.GetDesc(&raw mut descriptor) };
|
||||||
|
if descriptor.Format != DXGI_FORMAT_NV12 || descriptor.Width == 0 || descriptor.Height == 0 {
|
||||||
|
return Err("decoded H.264 texture is not NV12".to_owned());
|
||||||
|
}
|
||||||
|
let texture_device = unsafe { texture.GetDevice() }
|
||||||
|
.map_err(|error| format!("unable to read decoded texture device: {error}"))?;
|
||||||
|
if texture_device != *expected_device {
|
||||||
|
return Err("decoded H.264 texture belongs to another D3D11 device".to_owned());
|
||||||
|
}
|
||||||
|
let subresource = unsafe { dxgi_buffer.GetSubresourceIndex() }
|
||||||
|
.map_err(|error| format!("unable to read decoded H.264 subresource: {error}"))?;
|
||||||
|
let subresources = descriptor.MipLevels.saturating_mul(descriptor.ArraySize);
|
||||||
|
if descriptor.MipLevels == 0 || subresources == 0 || subresource >= subresources {
|
||||||
|
return Err("decoded H.264 subresource is outside its texture".to_owned());
|
||||||
|
}
|
||||||
|
let media_type = unsafe { transform.GetOutputCurrentType(0) }
|
||||||
|
.map_err(|error| format!("unable to read current H.264 output type: {error}"))?;
|
||||||
|
let (width, height) =
|
||||||
|
media_type_frame_size(&media_type).unwrap_or((descriptor.Width, descriptor.Height));
|
||||||
|
if width > descriptor.Width || height > descriptor.Height {
|
||||||
|
return Err("visible H.264 frame exceeds its NV12 texture".to_owned());
|
||||||
|
}
|
||||||
|
Ok(DecodedFrame {
|
||||||
|
_sample: sample,
|
||||||
|
texture,
|
||||||
|
subresource,
|
||||||
|
mip_levels: descriptor.MipLevels,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
input_color_space: media_type_color_space(&media_type, height),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_hardware_d3d11_device() -> Result<(ID3D11Device, ID3D11DeviceContext), String> {
|
||||||
|
let levels = [D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0];
|
||||||
|
let mut device = None;
|
||||||
|
let mut context = None;
|
||||||
|
unsafe {
|
||||||
|
D3D11CreateDevice(
|
||||||
|
None::<&IDXGIAdapter>,
|
||||||
|
D3D_DRIVER_TYPE_HARDWARE,
|
||||||
|
windows::Win32::Foundation::HMODULE::default(),
|
||||||
|
D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT,
|
||||||
|
Some(&levels),
|
||||||
|
D3D11_SDK_VERSION,
|
||||||
|
Some(&raw mut device),
|
||||||
|
None,
|
||||||
|
Some(&raw mut context),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
.map_err(|error| format!("unable to create remote-video D3D11 device: {error}"))?;
|
||||||
|
Ok((
|
||||||
|
device.ok_or_else(|| "D3D11 returned no remote-video device".to_owned())?,
|
||||||
|
context.ok_or_else(|| "D3D11 returned no remote-video context".to_owned())?,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
struct VideoRenderer {
|
||||||
|
window: Arc<Window>,
|
||||||
|
device: ID3D11Device,
|
||||||
|
context: ID3D11DeviceContext,
|
||||||
|
video_device: ID3D11VideoDevice,
|
||||||
|
video_context: ID3D11VideoContext,
|
||||||
|
swap_chain: IDXGISwapChain,
|
||||||
|
pipeline: Option<ProcessorPipeline>,
|
||||||
|
output_size: (u32, u32),
|
||||||
|
output_frame: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ProcessorPipeline {
|
||||||
|
input_size: (u32, u32),
|
||||||
|
output_size: (u32, u32),
|
||||||
|
enumerator: ID3D11VideoProcessorEnumerator,
|
||||||
|
processor: ID3D11VideoProcessor,
|
||||||
|
output_view: ID3D11VideoProcessorOutputView,
|
||||||
|
render_target: ID3D11RenderTargetView,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl VideoRenderer {
|
||||||
|
fn new(
|
||||||
|
window: Arc<Window>,
|
||||||
|
device: ID3D11Device,
|
||||||
|
context: ID3D11DeviceContext,
|
||||||
|
) -> Result<Self, String> {
|
||||||
|
let RawWindowHandle::Win32(handle) = window
|
||||||
|
.window_handle()
|
||||||
|
.map_err(|error| format!("unable to read remote-video window handle: {error}"))?
|
||||||
|
.as_raw()
|
||||||
|
else {
|
||||||
|
return Err("remote H.264 presentation requires a Win32 window".to_owned());
|
||||||
|
};
|
||||||
|
let hwnd = HWND(handle.hwnd.get() as *mut core::ffi::c_void);
|
||||||
|
let size = window.inner_size();
|
||||||
|
let width = size.width.max(1);
|
||||||
|
let height = size.height.max(1);
|
||||||
|
let descriptor = swap_chain_descriptor(hwnd, width, height);
|
||||||
|
let dxgi_device: IDXGIDevice = device
|
||||||
|
.cast()
|
||||||
|
.map_err(|error| format!("remote-video device has no DXGI interface: {error}"))?;
|
||||||
|
let adapter = unsafe { dxgi_device.GetAdapter() }
|
||||||
|
.map_err(|error| format!("unable to read remote-video adapter: {error}"))?;
|
||||||
|
let factory: IDXGIFactory = unsafe { adapter.GetParent() }
|
||||||
|
.map_err(|error| format!("unable to read remote-video DXGI factory: {error}"))?;
|
||||||
|
unsafe { factory.MakeWindowAssociation(hwnd, DXGI_MWA_NO_ALT_ENTER) }
|
||||||
|
.map_err(|error| format!("unable to configure remote-video window: {error}"))?;
|
||||||
|
let mut swap_chain = None;
|
||||||
|
unsafe { factory.CreateSwapChain(&device, &raw const descriptor, &raw mut swap_chain) }
|
||||||
|
.ok()
|
||||||
|
.map_err(|error| format!("unable to create remote-video swap chain: {error}"))?;
|
||||||
|
Ok(Self {
|
||||||
|
window,
|
||||||
|
video_device: device
|
||||||
|
.cast()
|
||||||
|
.map_err(|error| format!("D3D11 device has no video interface: {error}"))?,
|
||||||
|
video_context: context
|
||||||
|
.cast()
|
||||||
|
.map_err(|error| format!("D3D11 context has no video interface: {error}"))?,
|
||||||
|
device,
|
||||||
|
context,
|
||||||
|
swap_chain: swap_chain.ok_or_else(|| "DXGI returned no swap chain".to_owned())?,
|
||||||
|
pipeline: None,
|
||||||
|
output_size: (width, height),
|
||||||
|
output_frame: 0,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn present(&mut self, frame: &DecodedFrame) -> Result<(), String> {
|
||||||
|
let size = self.window.inner_size();
|
||||||
|
if size.width == 0 || size.height == 0 {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
self.resize(size.width, size.height)?;
|
||||||
|
self.ensure_pipeline(frame.width, frame.height)?;
|
||||||
|
let pipeline = self
|
||||||
|
.pipeline
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| "video processor is missing".to_owned())?;
|
||||||
|
let input_descriptor = D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC {
|
||||||
|
FourCC: 0,
|
||||||
|
ViewDimension: D3D11_VPIV_DIMENSION_TEXTURE2D,
|
||||||
|
Anonymous: D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC_0 {
|
||||||
|
Texture2D: D3D11_TEX2D_VPIV {
|
||||||
|
MipSlice: frame.subresource % frame.mip_levels,
|
||||||
|
ArraySlice: frame.subresource / frame.mip_levels,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let mut input_view = None;
|
||||||
|
unsafe {
|
||||||
|
self.video_device.CreateVideoProcessorInputView(
|
||||||
|
&frame.texture,
|
||||||
|
&pipeline.enumerator,
|
||||||
|
&raw const input_descriptor,
|
||||||
|
Some(&raw mut input_view),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
.map_err(|error| format!("unable to create NV12 input view: {error}"))?;
|
||||||
|
let source = RECT {
|
||||||
|
left: 0,
|
||||||
|
top: 0,
|
||||||
|
right: i32::try_from(frame.width).map_err(|_| "video width overflow")?,
|
||||||
|
bottom: i32::try_from(frame.height).map_err(|_| "video height overflow")?,
|
||||||
|
};
|
||||||
|
let destination = aspect_fit_rect(frame.width, frame.height, size.width, size.height)?;
|
||||||
|
let target = RECT {
|
||||||
|
left: 0,
|
||||||
|
top: 0,
|
||||||
|
right: i32::try_from(size.width).map_err(|_| "window width overflow")?,
|
||||||
|
bottom: i32::try_from(size.height).map_err(|_| "window height overflow")?,
|
||||||
|
};
|
||||||
|
unsafe {
|
||||||
|
self.context
|
||||||
|
.ClearRenderTargetView(&pipeline.render_target, &[0.0, 0.0, 0.0, 1.0]);
|
||||||
|
self.video_context.VideoProcessorSetOutputTargetRect(
|
||||||
|
&pipeline.processor,
|
||||||
|
true,
|
||||||
|
Some(&raw const target),
|
||||||
|
);
|
||||||
|
self.video_context.VideoProcessorSetStreamSourceRect(
|
||||||
|
&pipeline.processor,
|
||||||
|
0,
|
||||||
|
true,
|
||||||
|
Some(&raw const source),
|
||||||
|
);
|
||||||
|
self.video_context.VideoProcessorSetStreamDestRect(
|
||||||
|
&pipeline.processor,
|
||||||
|
0,
|
||||||
|
true,
|
||||||
|
Some(&raw const destination),
|
||||||
|
);
|
||||||
|
self.video_context.VideoProcessorSetStreamColorSpace(
|
||||||
|
&pipeline.processor,
|
||||||
|
0,
|
||||||
|
&raw const frame.input_color_space,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let mut stream = D3D11_VIDEO_PROCESSOR_STREAM {
|
||||||
|
Enable: true.into(),
|
||||||
|
pInputSurface: ManuallyDrop::new(input_view),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let result = unsafe {
|
||||||
|
self.video_context.VideoProcessorBlt(
|
||||||
|
&pipeline.processor,
|
||||||
|
&pipeline.output_view,
|
||||||
|
self.output_frame,
|
||||||
|
std::slice::from_ref(&stream),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let _input_view = unsafe { ManuallyDrop::take(&mut stream.pInputSurface) };
|
||||||
|
result.map_err(|error| format!("unable to process remote NV12 frame: {error}"))?;
|
||||||
|
unsafe { self.swap_chain.Present(1, DXGI_PRESENT(0)) }
|
||||||
|
.ok()
|
||||||
|
.map_err(|error| format!("unable to present remote H.264 frame: {error}"))?;
|
||||||
|
self.output_frame = self.output_frame.wrapping_add(1);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resize(&mut self, width: u32, height: u32) -> Result<(), String> {
|
||||||
|
if self.output_size == (width, height) {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
self.pipeline = None;
|
||||||
|
unsafe {
|
||||||
|
self.swap_chain.ResizeBuffers(
|
||||||
|
0,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
DXGI_FORMAT_UNKNOWN,
|
||||||
|
DXGI_SWAP_CHAIN_FLAG(0),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
.map_err(|error| format!("unable to resize remote-video swap chain: {error}"))?;
|
||||||
|
self.output_size = (width, height);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ensure_pipeline(&mut self, input_width: u32, input_height: u32) -> Result<(), String> {
|
||||||
|
if self.pipeline.as_ref().is_some_and(|pipeline| {
|
||||||
|
pipeline.input_size == (input_width, input_height)
|
||||||
|
&& pipeline.output_size == self.output_size
|
||||||
|
}) {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let content = D3D11_VIDEO_PROCESSOR_CONTENT_DESC {
|
||||||
|
InputFrameFormat: D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE,
|
||||||
|
InputFrameRate: DXGI_RATIONAL {
|
||||||
|
Numerator: 60,
|
||||||
|
Denominator: 1,
|
||||||
|
},
|
||||||
|
InputWidth: input_width,
|
||||||
|
InputHeight: input_height,
|
||||||
|
OutputFrameRate: DXGI_RATIONAL {
|
||||||
|
Numerator: 60,
|
||||||
|
Denominator: 1,
|
||||||
|
},
|
||||||
|
OutputWidth: self.output_size.0,
|
||||||
|
OutputHeight: self.output_size.1,
|
||||||
|
Usage: D3D11_VIDEO_USAGE_PLAYBACK_NORMAL,
|
||||||
|
};
|
||||||
|
let enumerator = unsafe {
|
||||||
|
self.video_device
|
||||||
|
.CreateVideoProcessorEnumerator(&raw const content)
|
||||||
|
}
|
||||||
|
.map_err(|error| format!("unable to create video processor enumerator: {error}"))?;
|
||||||
|
let input_support = unsafe { enumerator.CheckVideoProcessorFormat(DXGI_FORMAT_NV12) }
|
||||||
|
.map_err(|error| format!("unable to verify NV12 processor input: {error}"))?;
|
||||||
|
let output_support =
|
||||||
|
unsafe { enumerator.CheckVideoProcessorFormat(DXGI_FORMAT_B8G8R8A8_UNORM) }
|
||||||
|
.map_err(|error| format!("unable to verify BGRA processor output: {error}"))?;
|
||||||
|
let required_input = u32::try_from(D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_INPUT.0)
|
||||||
|
.map_err(|_| "invalid D3D11 input support flag".to_owned())?;
|
||||||
|
let required_output = u32::try_from(D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT.0)
|
||||||
|
.map_err(|_| "invalid D3D11 output support flag".to_owned())?;
|
||||||
|
if input_support & required_input == 0 || output_support & required_output == 0 {
|
||||||
|
return Err("D3D11 VideoProcessor lacks NV12/BGRA support".to_owned());
|
||||||
|
}
|
||||||
|
let processor = unsafe { self.video_device.CreateVideoProcessor(&enumerator, 0) }
|
||||||
|
.map_err(|error| format!("unable to create D3D11 video processor: {error}"))?;
|
||||||
|
let back_buffer: ID3D11Texture2D = unsafe { self.swap_chain.GetBuffer(0) }
|
||||||
|
.map_err(|error| format!("unable to read remote-video back buffer: {error}"))?;
|
||||||
|
let mut render_target = None;
|
||||||
|
unsafe {
|
||||||
|
self.device
|
||||||
|
.CreateRenderTargetView(&back_buffer, None, Some(&raw mut render_target))
|
||||||
|
}
|
||||||
|
.map_err(|error| format!("unable to create remote-video render target: {error}"))?;
|
||||||
|
let render_target =
|
||||||
|
render_target.ok_or_else(|| "D3D11 returned no render target view".to_owned())?;
|
||||||
|
let output_descriptor = D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC {
|
||||||
|
ViewDimension: D3D11_VPOV_DIMENSION_TEXTURE2D,
|
||||||
|
Anonymous: D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC_0 {
|
||||||
|
Texture2D: D3D11_TEX2D_VPOV { MipSlice: 0 },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let mut output_view = None;
|
||||||
|
unsafe {
|
||||||
|
self.video_device.CreateVideoProcessorOutputView(
|
||||||
|
&back_buffer,
|
||||||
|
&enumerator,
|
||||||
|
&raw const output_descriptor,
|
||||||
|
Some(&raw mut output_view),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
.map_err(|error| format!("unable to create processor output view: {error}"))?;
|
||||||
|
self.pipeline = Some(ProcessorPipeline {
|
||||||
|
input_size: (input_width, input_height),
|
||||||
|
output_size: self.output_size,
|
||||||
|
enumerator,
|
||||||
|
processor,
|
||||||
|
output_view: output_view
|
||||||
|
.ok_or_else(|| "D3D11 returned no processor output view".to_owned())?,
|
||||||
|
render_target,
|
||||||
|
});
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn swap_chain_descriptor(hwnd: HWND, width: u32, height: u32) -> DXGI_SWAP_CHAIN_DESC {
|
||||||
|
DXGI_SWAP_CHAIN_DESC {
|
||||||
|
BufferDesc: DXGI_MODE_DESC {
|
||||||
|
Width: width,
|
||||||
|
Height: height,
|
||||||
|
RefreshRate: DXGI_RATIONAL {
|
||||||
|
Numerator: 0,
|
||||||
|
Denominator: 1,
|
||||||
|
},
|
||||||
|
Format: DXGI_FORMAT_B8G8R8A8_UNORM,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
SampleDesc: DXGI_SAMPLE_DESC {
|
||||||
|
Count: 1,
|
||||||
|
Quality: 0,
|
||||||
|
},
|
||||||
|
BufferUsage: DXGI_USAGE_RENDER_TARGET_OUTPUT,
|
||||||
|
BufferCount: 2,
|
||||||
|
OutputWindow: hwnd,
|
||||||
|
Windowed: true.into(),
|
||||||
|
SwapEffect: DXGI_SWAP_EFFECT_DISCARD,
|
||||||
|
Flags: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn aspect_fit_rect(
|
||||||
|
source_width: u32,
|
||||||
|
source_height: u32,
|
||||||
|
target_width: u32,
|
||||||
|
target_height: u32,
|
||||||
|
) -> Result<RECT, String> {
|
||||||
|
if source_width == 0 || source_height == 0 || target_width == 0 || target_height == 0 {
|
||||||
|
return Err("video dimensions must be non-zero".to_owned());
|
||||||
|
}
|
||||||
|
let source_aspect = f64::from(source_width) / f64::from(source_height);
|
||||||
|
let target_aspect = f64::from(target_width) / f64::from(target_height);
|
||||||
|
let (width, height) = if source_aspect > target_aspect {
|
||||||
|
(
|
||||||
|
target_width,
|
||||||
|
(f64::from(target_width) / source_aspect).round() as u32,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
(
|
||||||
|
(f64::from(target_height) * source_aspect).round() as u32,
|
||||||
|
target_height,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let left = (target_width - width) / 2;
|
||||||
|
let top = (target_height - height) / 2;
|
||||||
|
Ok(RECT {
|
||||||
|
left: i32::try_from(left).map_err(|_| "video left overflow")?,
|
||||||
|
top: i32::try_from(top).map_err(|_| "video top overflow")?,
|
||||||
|
right: i32::try_from(left + width).map_err(|_| "video right overflow")?,
|
||||||
|
bottom: i32::try_from(top + height).map_err(|_| "video bottom overflow")?,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn media_type_frame_size(media_type: &IMFMediaType) -> Option<(u32, u32)> {
|
||||||
|
let frame_size_key = MF_MT_FRAME_SIZE;
|
||||||
|
let packed = unsafe { media_type.GetUINT64(&frame_size_key) }.ok()?;
|
||||||
|
let width = u32::try_from(packed >> 32).ok()?;
|
||||||
|
let height = u32::try_from(packed & u64::from(u32::MAX)).ok()?;
|
||||||
|
(width > 0 && height > 0).then_some((width, height))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn media_type_color_space(
|
||||||
|
media_type: &IMFMediaType,
|
||||||
|
visible_height: u32,
|
||||||
|
) -> D3D11_VIDEO_PROCESSOR_COLOR_SPACE {
|
||||||
|
let bt709_value = u32::try_from(MFVideoTransferMatrix_BT709.0).unwrap_or(1);
|
||||||
|
let full_range_value = u32::try_from(MFNominalRange_0_255.0).unwrap_or(1);
|
||||||
|
let yuv_matrix_key = MF_MT_YUV_MATRIX;
|
||||||
|
let bt709 = unsafe { media_type.GetUINT32(&yuv_matrix_key) }
|
||||||
|
.map_or(visible_height >= 720, |value| value == bt709_value);
|
||||||
|
let nominal_range_key = MF_MT_VIDEO_NOMINAL_RANGE;
|
||||||
|
let full_range = unsafe { media_type.GetUINT32(&nominal_range_key) }
|
||||||
|
.is_ok_and(|value| value == full_range_value);
|
||||||
|
D3D11_VIDEO_PROCESSOR_COLOR_SPACE {
|
||||||
|
_bitfield: (u32::from(bt709) << 2) | ((if full_range { 2 } else { 1 }) << 4),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn elapsed_microseconds(started: Instant) -> u64 {
|
||||||
|
started.elapsed().as_micros().try_into().unwrap_or(u64::MAX)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn aspect_fit_centers_letterbox_and_pillarbox_output() {
|
||||||
|
assert_eq!(
|
||||||
|
aspect_fit_rect(1_920, 1_080, 1_000, 1_000).unwrap(),
|
||||||
|
RECT {
|
||||||
|
left: 0,
|
||||||
|
top: 218,
|
||||||
|
right: 1_000,
|
||||||
|
bottom: 781
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
aspect_fit_rect(1_000, 1_000, 1_920, 1_080).unwrap(),
|
||||||
|
RECT {
|
||||||
|
left: 420,
|
||||||
|
top: 0,
|
||||||
|
right: 1_500,
|
||||||
|
bottom: 1_080
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,12 +7,16 @@ rust-version.workspace = true
|
|||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
flate2 = "1.1"
|
flate2 = "1.1"
|
||||||
|
remotedesk-native-video = { path = "../native-video" }
|
||||||
|
remotedesk-protocol = { path = "../../../protocol" }
|
||||||
serde = { version = "1.0", features = ["derive"] }
|
serde = { version = "1.0", features = ["derive"] }
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
softbuffer = "0.4"
|
softbuffer = "0.4"
|
||||||
winit = "0.30"
|
winit = "0.30"
|
||||||
|
|
||||||
[target.'cfg(windows)'.dependencies]
|
[target.'cfg(windows)'.dependencies]
|
||||||
|
cpal = "0.17.3"
|
||||||
|
opus2 = { version = "0.4.0", features = ["bundled"] }
|
||||||
windows = { version = "0.62.2", features = ["Win32_Foundation", "Win32_UI_WindowsAndMessaging"] }
|
windows = { version = "0.62.2", features = ["Win32_Foundation", "Win32_UI_WindowsAndMessaging"] }
|
||||||
|
|
||||||
[lints.rust]
|
[lints.rust]
|
||||||
|
|||||||
@@ -0,0 +1,633 @@
|
|||||||
|
//! Bounded RDA1 jitter buffering, Opus decoding, and WASAPI playback.
|
||||||
|
//!
|
||||||
|
//! Audio is independent from the video event mailbox. A slow or unavailable
|
||||||
|
//! audio device only disables this worker; it never terminates video or input.
|
||||||
|
|
||||||
|
use remotedesk_protocol::AudioPacket;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
const AUDIO_TARGET_BUFFER: Duration = Duration::from_millis(60);
|
||||||
|
const VIDEO_LATE_TOLERANCE_100NS: u64 = 800_000;
|
||||||
|
const AUDIO_SYNC_DEADBAND_100NS: i64 = 50_000;
|
||||||
|
const AUDIO_RATE_LIMIT_PPM: i32 = 5_000;
|
||||||
|
|
||||||
|
/// Shared media timeline used to make the video path follow audio playout.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct MediaClock {
|
||||||
|
base_audio_pts: Option<u64>,
|
||||||
|
base_local_time: Option<Instant>,
|
||||||
|
last_audio_pts: Option<u64>,
|
||||||
|
source_rate_adjustment_ppm: i32,
|
||||||
|
buffer_rate_adjustment_ppm: i32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MediaClock {
|
||||||
|
pub(crate) fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
base_audio_pts: None,
|
||||||
|
base_local_time: None,
|
||||||
|
last_audio_pts: None,
|
||||||
|
source_rate_adjustment_ppm: 0,
|
||||||
|
buffer_rate_adjustment_ppm: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn observe_audio(&mut self, pts: u64, duration_ms: u16) {
|
||||||
|
if self.base_audio_pts.is_none() {
|
||||||
|
self.base_audio_pts = Some(pts);
|
||||||
|
self.base_local_time = Some(Instant::now() + AUDIO_TARGET_BUFFER);
|
||||||
|
}
|
||||||
|
let is_forward = self.last_audio_pts.is_none_or(|last_pts| pts > last_pts);
|
||||||
|
if is_forward {
|
||||||
|
if let Some(last_pts) = self.last_audio_pts {
|
||||||
|
let delta = pts.saturating_sub(last_pts);
|
||||||
|
let expected_duration = u64::from(duration_ms.max(1)).saturating_mul(10_000);
|
||||||
|
let error = i64::try_from(delta).unwrap_or(i64::MAX)
|
||||||
|
- i64::try_from(expected_duration).unwrap_or(i64::MAX);
|
||||||
|
if error.abs() <= AUDIO_SYNC_DEADBAND_100NS {
|
||||||
|
self.source_rate_adjustment_ppm =
|
||||||
|
self.source_rate_adjustment_ppm.saturating_mul(7) / 8;
|
||||||
|
} else {
|
||||||
|
let target = (error.saturating_mul(1_000_000)
|
||||||
|
/ i64::try_from(expected_duration.max(1)).unwrap_or(1))
|
||||||
|
.clamp(
|
||||||
|
-i64::from(AUDIO_RATE_LIMIT_PPM),
|
||||||
|
i64::from(AUDIO_RATE_LIMIT_PPM),
|
||||||
|
);
|
||||||
|
let current = i64::from(self.source_rate_adjustment_ppm);
|
||||||
|
self.source_rate_adjustment_ppm =
|
||||||
|
i32::try_from(current + (target - current) / 8).unwrap_or(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.last_audio_pts = Some(pts);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn reset_audio(&mut self, pts: u64) {
|
||||||
|
self.base_audio_pts = Some(pts);
|
||||||
|
self.base_local_time = Some(Instant::now() + AUDIO_TARGET_BUFFER);
|
||||||
|
self.last_audio_pts = Some(pts);
|
||||||
|
self.source_rate_adjustment_ppm = 0;
|
||||||
|
self.buffer_rate_adjustment_ppm = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn update_buffer_rate_adjustment(&mut self, buffered_frames: usize) {
|
||||||
|
const TARGET_AUDIO_FRAMES: i64 = 2_880;
|
||||||
|
let error = i64::try_from(buffered_frames).unwrap_or(i64::MAX) - TARGET_AUDIO_FRAMES;
|
||||||
|
self.buffer_rate_adjustment_ppm = error.saturating_mul(2).clamp(
|
||||||
|
-i64::from(AUDIO_RATE_LIMIT_PPM),
|
||||||
|
i64::from(AUDIO_RATE_LIMIT_PPM),
|
||||||
|
) as i32;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn rate_adjustment_ppm(&self) -> i32 {
|
||||||
|
self.source_rate_adjustment_ppm
|
||||||
|
.saturating_add(self.buffer_rate_adjustment_ppm)
|
||||||
|
.clamp(-AUDIO_RATE_LIMIT_PPM, AUDIO_RATE_LIMIT_PPM)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn audio_playout_pts(&self) -> Option<u64> {
|
||||||
|
let (Some(base_pts), Some(base_time)) = (self.base_audio_pts, self.base_local_time) else {
|
||||||
|
return None;
|
||||||
|
};
|
||||||
|
let elapsed = Instant::now().saturating_duration_since(base_time);
|
||||||
|
let elapsed_100ns = elapsed.as_nanos().min(u128::from(u64::MAX) * 100) / 100;
|
||||||
|
base_pts.checked_add(u64::try_from(elapsed_100ns).unwrap_or(u64::MAX))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns false only for video frames already too far behind audio.
|
||||||
|
pub(crate) fn should_present_video(&self, video_pts: u64) -> bool {
|
||||||
|
self.audio_playout_pts().is_none_or(|audio_pts| {
|
||||||
|
video_pts.saturating_add(VIDEO_LATE_TOLERANCE_100NS) >= audio_pts
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(windows)]
|
||||||
|
mod windows_audio {
|
||||||
|
use super::*;
|
||||||
|
use cpal::SizedSample;
|
||||||
|
use cpal::traits::{DeviceTrait as _, HostTrait as _, StreamTrait as _};
|
||||||
|
use opus2::{Channels, Decoder};
|
||||||
|
use std::collections::{BTreeMap, VecDeque};
|
||||||
|
use std::sync::atomic::{AtomicI32, Ordering};
|
||||||
|
use std::sync::mpsc::Receiver;
|
||||||
|
|
||||||
|
const MAX_JITTER_PACKETS: usize = 64;
|
||||||
|
const MAX_AUDIO_FRAMES: usize = 48_000;
|
||||||
|
const MAX_DECODE_SAMPLES_PER_CHANNEL: usize = 5_760;
|
||||||
|
|
||||||
|
struct PlaybackBuffer {
|
||||||
|
frames: VecDeque<[f32; 2]>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PlaybackBuffer {
|
||||||
|
fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
frames: VecDeque::with_capacity(MAX_AUDIO_FRAMES),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push(&mut self, samples: &[f32]) {
|
||||||
|
if samples.len() / 2 > MAX_AUDIO_FRAMES {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if self.frames.len() + samples.len() / 2 > MAX_AUDIO_FRAMES {
|
||||||
|
self.frames.clear();
|
||||||
|
}
|
||||||
|
self.frames
|
||||||
|
.extend(samples.chunks_exact(2).map(|frame| [frame[0], frame[1]]));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn pop(&mut self) -> [f32; 2] {
|
||||||
|
self.frames.pop_front().unwrap_or([0.0; 2])
|
||||||
|
}
|
||||||
|
|
||||||
|
fn clear(&mut self) {
|
||||||
|
self.frames.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn len(&self) -> usize {
|
||||||
|
self.frames.len()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct JitterDecoder {
|
||||||
|
decoder: Decoder,
|
||||||
|
packets: BTreeMap<u64, AudioPacket>,
|
||||||
|
expected: Option<u64>,
|
||||||
|
generation: u64,
|
||||||
|
stream_id: u32,
|
||||||
|
primed: bool,
|
||||||
|
last_decoded_sequence: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl JitterDecoder {
|
||||||
|
fn new(packet: &AudioPacket) -> Result<Self, String> {
|
||||||
|
Ok(Self {
|
||||||
|
decoder: Decoder::new(48_000, Channels::Stereo)
|
||||||
|
.map_err(|error| format!("Opus decoder initialization failed: {error}"))?,
|
||||||
|
packets: BTreeMap::new(),
|
||||||
|
expected: Some(packet.sequence),
|
||||||
|
generation: packet.generation,
|
||||||
|
stream_id: packet.stream_id,
|
||||||
|
primed: false,
|
||||||
|
last_decoded_sequence: None,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reset(&mut self, packet: &AudioPacket) -> Result<(), String> {
|
||||||
|
self.decoder = Decoder::new(48_000, Channels::Stereo)
|
||||||
|
.map_err(|error| format!("Opus decoder reset failed: {error}"))?;
|
||||||
|
self.packets.clear();
|
||||||
|
self.expected = Some(packet.sequence);
|
||||||
|
self.generation = packet.generation;
|
||||||
|
self.stream_id = packet.stream_id;
|
||||||
|
self.primed = false;
|
||||||
|
self.last_decoded_sequence = None;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push(
|
||||||
|
&mut self,
|
||||||
|
packet: AudioPacket,
|
||||||
|
output: &mut PlaybackBuffer,
|
||||||
|
clock: &Arc<Mutex<MediaClock>>,
|
||||||
|
rate_adjustment_ppm: &Arc<AtomicI32>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let reset = packet.stream_id != self.stream_id
|
||||||
|
|| packet.generation != self.generation
|
||||||
|
|| packet
|
||||||
|
.flags
|
||||||
|
.contains(remotedesk_protocol::AudioPacketFlags::DISCONTINUITY);
|
||||||
|
if reset {
|
||||||
|
self.reset(&packet)?;
|
||||||
|
}
|
||||||
|
let mut clock_guard = clock
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| "audio clock lock poisoned".to_owned())?;
|
||||||
|
if reset {
|
||||||
|
output.clear();
|
||||||
|
clock_guard.reset_audio(packet.pts);
|
||||||
|
rate_adjustment_ppm.store(0, Ordering::Release);
|
||||||
|
}
|
||||||
|
drop(clock_guard);
|
||||||
|
if self.packets.len() >= MAX_JITTER_PACKETS {
|
||||||
|
self.packets.pop_first();
|
||||||
|
}
|
||||||
|
self.packets.entry(packet.sequence).or_insert(packet);
|
||||||
|
if !self.primed {
|
||||||
|
if self.packets.len() < 3 {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
self.primed = true;
|
||||||
|
}
|
||||||
|
self.drain(output, clock)?;
|
||||||
|
let mut clock = clock
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| "audio clock lock poisoned".to_owned())?;
|
||||||
|
clock.update_buffer_rate_adjustment(output.len());
|
||||||
|
rate_adjustment_ppm.store(clock.rate_adjustment_ppm(), Ordering::Release);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drain(
|
||||||
|
&mut self,
|
||||||
|
output: &mut PlaybackBuffer,
|
||||||
|
clock: &Arc<Mutex<MediaClock>>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let Some(mut expected) = self.expected else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
let mut missing_budget = 3;
|
||||||
|
loop {
|
||||||
|
if let Some(packet) = self.packets.remove(&expected) {
|
||||||
|
self.decode(&packet.payload, false, output)?;
|
||||||
|
let mut clock = clock
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| "audio clock lock poisoned".to_owned())?;
|
||||||
|
if self
|
||||||
|
.last_decoded_sequence
|
||||||
|
.is_some_and(|sequence| sequence.saturating_add(1) == packet.sequence)
|
||||||
|
{
|
||||||
|
clock.observe_audio(packet.pts, packet.duration_ms);
|
||||||
|
}
|
||||||
|
self.last_decoded_sequence = Some(packet.sequence);
|
||||||
|
expected = expected.saturating_add(1);
|
||||||
|
missing_budget = 3;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some((&next, _)) = self.packets.first_key_value() else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
if next <= expected {
|
||||||
|
self.packets.remove(&next);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if missing_budget == 0 {
|
||||||
|
expected = next;
|
||||||
|
missing_budget = 3;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// The packet after the gap may contain in-band FEC for one
|
||||||
|
// missing packet. Fall back to Opus PLC if FEC is unavailable.
|
||||||
|
let fec_payload = (next == expected.saturating_add(1))
|
||||||
|
.then(|| self.packets.get(&next).map(|packet| packet.payload.clone()))
|
||||||
|
.flatten();
|
||||||
|
if fec_payload
|
||||||
|
.as_deref()
|
||||||
|
.is_none_or(|payload| self.decode(payload, true, output).is_err())
|
||||||
|
{
|
||||||
|
self.decode(&[], false, output)?;
|
||||||
|
}
|
||||||
|
expected = expected.saturating_add(1);
|
||||||
|
missing_budget -= 1;
|
||||||
|
}
|
||||||
|
self.expected = Some(expected);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode(
|
||||||
|
&mut self,
|
||||||
|
payload: &[u8],
|
||||||
|
fec: bool,
|
||||||
|
output: &mut PlaybackBuffer,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let mut pcm = vec![0_i16; MAX_DECODE_SAMPLES_PER_CHANNEL * 2];
|
||||||
|
let samples = self
|
||||||
|
.decoder
|
||||||
|
.decode(payload, &mut pcm, fec)
|
||||||
|
.map_err(|error| format!("Opus decode failed: {error}"))?;
|
||||||
|
let interleaved = pcm
|
||||||
|
.iter()
|
||||||
|
.take(samples.saturating_mul(2))
|
||||||
|
.map(|sample| f32::from(*sample) / f32::from(i16::MAX))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
output.push(&interleaved);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn start(receiver: Receiver<AudioPacket>, clock: Arc<Mutex<MediaClock>>) {
|
||||||
|
let _ = std::thread::Builder::new()
|
||||||
|
.name("windows-agent-opus-playback".to_owned())
|
||||||
|
.spawn(move || {
|
||||||
|
if let Err(error) = run(receiver, clock) {
|
||||||
|
eprintln!("Windows Agent audio disabled: {error}");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run(receiver: Receiver<AudioPacket>, clock: Arc<Mutex<MediaClock>>) -> Result<(), String> {
|
||||||
|
let host = cpal::default_host();
|
||||||
|
let device = host
|
||||||
|
.default_output_device()
|
||||||
|
.ok_or_else(|| "WASAPI default output device unavailable".to_owned())?;
|
||||||
|
let supported = device
|
||||||
|
.default_output_config()
|
||||||
|
.map_err(|error| format!("WASAPI output configuration failed: {error}"))?;
|
||||||
|
let config = supported.config();
|
||||||
|
let channels = usize::from(config.channels);
|
||||||
|
let output_rate = config.sample_rate;
|
||||||
|
if channels == 0 || channels > 8 {
|
||||||
|
return Err("WASAPI output channel count is unsupported".to_owned());
|
||||||
|
}
|
||||||
|
let buffer = Arc::new(Mutex::new(PlaybackBuffer::new()));
|
||||||
|
let rate_adjustment_ppm = Arc::new(AtomicI32::new(0));
|
||||||
|
let stream = match supported.sample_format() {
|
||||||
|
cpal::SampleFormat::I8 => build_output::<i8>(
|
||||||
|
&device,
|
||||||
|
&config,
|
||||||
|
channels,
|
||||||
|
output_rate,
|
||||||
|
Arc::clone(&buffer),
|
||||||
|
Arc::clone(&rate_adjustment_ppm),
|
||||||
|
)?,
|
||||||
|
cpal::SampleFormat::I16 => build_output::<i16>(
|
||||||
|
&device,
|
||||||
|
&config,
|
||||||
|
channels,
|
||||||
|
output_rate,
|
||||||
|
Arc::clone(&buffer),
|
||||||
|
Arc::clone(&rate_adjustment_ppm),
|
||||||
|
)?,
|
||||||
|
cpal::SampleFormat::I24 => build_output::<cpal::I24>(
|
||||||
|
&device,
|
||||||
|
&config,
|
||||||
|
channels,
|
||||||
|
output_rate,
|
||||||
|
Arc::clone(&buffer),
|
||||||
|
Arc::clone(&rate_adjustment_ppm),
|
||||||
|
)?,
|
||||||
|
cpal::SampleFormat::I32 => build_output::<i32>(
|
||||||
|
&device,
|
||||||
|
&config,
|
||||||
|
channels,
|
||||||
|
output_rate,
|
||||||
|
Arc::clone(&buffer),
|
||||||
|
Arc::clone(&rate_adjustment_ppm),
|
||||||
|
)?,
|
||||||
|
cpal::SampleFormat::I64 => build_output::<i64>(
|
||||||
|
&device,
|
||||||
|
&config,
|
||||||
|
channels,
|
||||||
|
output_rate,
|
||||||
|
Arc::clone(&buffer),
|
||||||
|
Arc::clone(&rate_adjustment_ppm),
|
||||||
|
)?,
|
||||||
|
cpal::SampleFormat::U8 => build_output::<u8>(
|
||||||
|
&device,
|
||||||
|
&config,
|
||||||
|
channels,
|
||||||
|
output_rate,
|
||||||
|
Arc::clone(&buffer),
|
||||||
|
Arc::clone(&rate_adjustment_ppm),
|
||||||
|
)?,
|
||||||
|
cpal::SampleFormat::U16 => build_output::<u16>(
|
||||||
|
&device,
|
||||||
|
&config,
|
||||||
|
channels,
|
||||||
|
output_rate,
|
||||||
|
Arc::clone(&buffer),
|
||||||
|
Arc::clone(&rate_adjustment_ppm),
|
||||||
|
)?,
|
||||||
|
cpal::SampleFormat::U24 => build_output::<cpal::U24>(
|
||||||
|
&device,
|
||||||
|
&config,
|
||||||
|
channels,
|
||||||
|
output_rate,
|
||||||
|
Arc::clone(&buffer),
|
||||||
|
Arc::clone(&rate_adjustment_ppm),
|
||||||
|
)?,
|
||||||
|
cpal::SampleFormat::U32 => build_output::<u32>(
|
||||||
|
&device,
|
||||||
|
&config,
|
||||||
|
channels,
|
||||||
|
output_rate,
|
||||||
|
Arc::clone(&buffer),
|
||||||
|
Arc::clone(&rate_adjustment_ppm),
|
||||||
|
)?,
|
||||||
|
cpal::SampleFormat::U64 => build_output::<u64>(
|
||||||
|
&device,
|
||||||
|
&config,
|
||||||
|
channels,
|
||||||
|
output_rate,
|
||||||
|
Arc::clone(&buffer),
|
||||||
|
Arc::clone(&rate_adjustment_ppm),
|
||||||
|
)?,
|
||||||
|
cpal::SampleFormat::F32 => build_output::<f32>(
|
||||||
|
&device,
|
||||||
|
&config,
|
||||||
|
channels,
|
||||||
|
output_rate,
|
||||||
|
Arc::clone(&buffer),
|
||||||
|
Arc::clone(&rate_adjustment_ppm),
|
||||||
|
)?,
|
||||||
|
cpal::SampleFormat::F64 => build_output::<f64>(
|
||||||
|
&device,
|
||||||
|
&config,
|
||||||
|
channels,
|
||||||
|
output_rate,
|
||||||
|
Arc::clone(&buffer),
|
||||||
|
Arc::clone(&rate_adjustment_ppm),
|
||||||
|
)?,
|
||||||
|
_ => return Err("WASAPI output sample format is unsupported".to_owned()),
|
||||||
|
};
|
||||||
|
stream
|
||||||
|
.play()
|
||||||
|
.map_err(|error| format!("WASAPI output start failed: {error}"))?;
|
||||||
|
|
||||||
|
let first = receiver
|
||||||
|
.recv()
|
||||||
|
.map_err(|_| "audio stream closed".to_owned())?;
|
||||||
|
let mut decoder = JitterDecoder::new(&first)?;
|
||||||
|
{
|
||||||
|
let mut playback = buffer
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| "audio buffer lock poisoned".to_owned())?;
|
||||||
|
decoder.push(first, &mut playback, &clock, &rate_adjustment_ppm)?;
|
||||||
|
}
|
||||||
|
loop {
|
||||||
|
match receiver.recv() {
|
||||||
|
Ok(packet) => {
|
||||||
|
let mut playback = buffer
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| "audio buffer lock poisoned".to_owned())?;
|
||||||
|
decoder.push(packet, &mut playback, &clock, &rate_adjustment_ppm)?;
|
||||||
|
}
|
||||||
|
Err(_) => return Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_output<T>(
|
||||||
|
device: &cpal::Device,
|
||||||
|
config: &cpal::StreamConfig,
|
||||||
|
channels: usize,
|
||||||
|
output_rate: u32,
|
||||||
|
buffer: Arc<Mutex<PlaybackBuffer>>,
|
||||||
|
rate_adjustment_ppm: Arc<AtomicI32>,
|
||||||
|
) -> Result<cpal::Stream, String>
|
||||||
|
where
|
||||||
|
T: SizedSample + cpal::FromSample<f32>,
|
||||||
|
{
|
||||||
|
let step = 48_000.0 / f64::from(output_rate.max(1));
|
||||||
|
let mut phase = 1.0;
|
||||||
|
let mut current = [0.0_f32; 2];
|
||||||
|
device
|
||||||
|
.build_output_stream(
|
||||||
|
config,
|
||||||
|
move |output: &mut [T], _: &cpal::OutputCallbackInfo| {
|
||||||
|
if let Ok(mut buffer) = buffer.try_lock() {
|
||||||
|
for frame in output.chunks_mut(channels) {
|
||||||
|
while phase >= 1.0 {
|
||||||
|
current = buffer.pop();
|
||||||
|
phase -= 1.0;
|
||||||
|
}
|
||||||
|
let adjustment = f64::from(rate_adjustment_ppm.load(Ordering::Acquire))
|
||||||
|
/ 1_000_000.0;
|
||||||
|
phase += step * (1.0 + adjustment);
|
||||||
|
for (channel, value) in frame.iter_mut().enumerate() {
|
||||||
|
let sample = match channel {
|
||||||
|
0 => current[0],
|
||||||
|
1 => current[1],
|
||||||
|
_ => (current[0] + current[1]) * 0.5,
|
||||||
|
};
|
||||||
|
*value = T::from_sample(sample);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for value in output {
|
||||||
|
*value = T::from_sample(0.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|error| eprintln!("WASAPI output stream error: {error}"),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.map_err(|error| format!("WASAPI output stream creation failed: {error}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use opus2::{Application, Encoder};
|
||||||
|
|
||||||
|
fn packet(encoder: &mut Encoder, sequence: u64) -> remotedesk_protocol::AudioPacket {
|
||||||
|
let payload = encoder
|
||||||
|
.encode_vec_float(&vec![0.0; 960 * 2], 4 * 1024)
|
||||||
|
.unwrap();
|
||||||
|
remotedesk_protocol::AudioPacket {
|
||||||
|
stream_id: 1,
|
||||||
|
generation: 1,
|
||||||
|
sequence,
|
||||||
|
pts: (sequence - 1) * 200_000,
|
||||||
|
duration_ms: 20,
|
||||||
|
flags: remotedesk_protocol::AudioPacketFlags::from_bits(0).unwrap(),
|
||||||
|
payload,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn jitter_startup_reorders_three_packets_before_decode() {
|
||||||
|
let mut encoder = Encoder::new(48_000, Channels::Stereo, Application::Audio).unwrap();
|
||||||
|
let first = packet(&mut encoder, 1);
|
||||||
|
let second = packet(&mut encoder, 2);
|
||||||
|
let third = packet(&mut encoder, 3);
|
||||||
|
let clock = Arc::new(Mutex::new(MediaClock::new()));
|
||||||
|
let rate_adjustment_ppm = Arc::new(AtomicI32::new(0));
|
||||||
|
let mut output = PlaybackBuffer::new();
|
||||||
|
let mut jitter = JitterDecoder::new(&first).unwrap();
|
||||||
|
|
||||||
|
jitter
|
||||||
|
.push(first, &mut output, &clock, &rate_adjustment_ppm)
|
||||||
|
.unwrap();
|
||||||
|
jitter
|
||||||
|
.push(third, &mut output, &clock, &rate_adjustment_ppm)
|
||||||
|
.unwrap();
|
||||||
|
assert!(output.frames.is_empty());
|
||||||
|
jitter
|
||||||
|
.push(second, &mut output, &clock, &rate_adjustment_ppm)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(output.frames.len(), 3 * 960);
|
||||||
|
assert_eq!(jitter.expected, Some(4));
|
||||||
|
assert!(jitter.packets.is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(windows)]
|
||||||
|
pub(crate) use windows_audio::start;
|
||||||
|
|
||||||
|
#[cfg(not(windows))]
|
||||||
|
pub(crate) fn start(
|
||||||
|
_receiver: std::sync::mpsc::Receiver<AudioPacket>,
|
||||||
|
_clock: Arc<Mutex<MediaClock>>,
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn media_clock_is_unlocked_before_audio_and_accepts_on_time_video() {
|
||||||
|
let mut clock = MediaClock::new();
|
||||||
|
assert!(clock.should_present_video(1));
|
||||||
|
clock.observe_audio(1_000_000, 20);
|
||||||
|
assert!(clock.should_present_video(1_000_000));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn media_clock_keeps_nominal_audio_rate_near_zero() {
|
||||||
|
let mut clock = MediaClock::new();
|
||||||
|
clock.observe_audio(1_000_000, 20);
|
||||||
|
for sequence in 1..32 {
|
||||||
|
clock.observe_audio(1_000_000 + sequence * 200_000, 20);
|
||||||
|
}
|
||||||
|
assert_eq!(clock.rate_adjustment_ppm(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn media_clock_bounds_drift_correction_and_decays_in_deadband() {
|
||||||
|
let mut clock = MediaClock::new();
|
||||||
|
clock.observe_audio(1_000_000, 20);
|
||||||
|
for sequence in 1..128 {
|
||||||
|
clock.observe_audio(1_000_000 + sequence * 300_000, 30);
|
||||||
|
}
|
||||||
|
assert!(clock.rate_adjustment_ppm().abs() <= AUDIO_RATE_LIMIT_PPM);
|
||||||
|
let corrected = clock.rate_adjustment_ppm();
|
||||||
|
for sequence in 128..160 {
|
||||||
|
clock.observe_audio(
|
||||||
|
1_000_000 + 127 * 300_000 + (sequence - 127) * 200_000,
|
||||||
|
20,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(clock.rate_adjustment_ppm().abs() < corrected.abs());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn media_clock_reset_clears_drift_and_buffer_correction() {
|
||||||
|
let mut clock = MediaClock::new();
|
||||||
|
clock.observe_audio(1_000_000, 20);
|
||||||
|
clock.observe_audio(1_300_000, 20);
|
||||||
|
clock.update_buffer_rate_adjustment(48_000);
|
||||||
|
assert_ne!(clock.rate_adjustment_ppm(), 0);
|
||||||
|
|
||||||
|
clock.reset_audio(9_000_000);
|
||||||
|
assert_eq!(clock.rate_adjustment_ppm(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn media_clock_buffer_feedback_is_bounded_and_zero_at_target() {
|
||||||
|
let mut clock = MediaClock::new();
|
||||||
|
clock.update_buffer_rate_adjustment(2_880);
|
||||||
|
assert_eq!(clock.rate_adjustment_ppm(), 0);
|
||||||
|
clock.update_buffer_rate_adjustment(48_000);
|
||||||
|
assert_eq!(clock.rate_adjustment_ppm(), AUDIO_RATE_LIMIT_PPM);
|
||||||
|
clock.update_buffer_rate_adjustment(0);
|
||||||
|
assert_eq!(clock.rate_adjustment_ppm(), -AUDIO_RATE_LIMIT_PPM);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,13 +1,21 @@
|
|||||||
use flate2::read::ZlibDecoder;
|
use flate2::read::ZlibDecoder;
|
||||||
|
mod audio;
|
||||||
|
#[cfg(windows)]
|
||||||
|
use remotedesk_native_video::stream::{H264AccessUnit, NativeH264Renderer};
|
||||||
|
use remotedesk_protocol::{
|
||||||
|
AUDIO_DATAGRAM_MAX_BYTES, VIDEO_DATAGRAM_MAX_BYTES, VideoAccessUnit, VideoReassembler,
|
||||||
|
decode_audio_datagram,
|
||||||
|
};
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use std::env;
|
use std::env;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::io::{BufRead as _, BufReader, Read as _, Write as _};
|
use std::io::{BufRead as _, BufReader, Read as _, Write as _};
|
||||||
use std::net::{TcpStream, ToSocketAddrs};
|
use std::net::{SocketAddr, TcpStream, ToSocketAddrs, UdpSocket};
|
||||||
use std::num::NonZeroU32;
|
use std::num::NonZeroU32;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::process::ExitCode;
|
use std::process::ExitCode;
|
||||||
use std::sync::Arc;
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
use std::sync::{Arc, Mutex, mpsc};
|
||||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||||
use winit::application::ApplicationHandler;
|
use winit::application::ApplicationHandler;
|
||||||
use winit::dpi::PhysicalSize;
|
use winit::dpi::PhysicalSize;
|
||||||
@@ -22,6 +30,7 @@ const DEFAULT_PORT: u16 = 39_501;
|
|||||||
const MAX_DIMENSION: u32 = 16_384;
|
const MAX_DIMENSION: u32 = 16_384;
|
||||||
const MAX_RAW_BYTES: usize = 256 * 1024 * 1024;
|
const MAX_RAW_BYTES: usize = 256 * 1024 * 1024;
|
||||||
const MAX_COMPRESSED_BYTES: usize = 256 * 1024 * 1024;
|
const MAX_COMPRESSED_BYTES: usize = 256 * 1024 * 1024;
|
||||||
|
const INPUT_QUEUE_CAPACITY: usize = 256;
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
struct Args {
|
struct Args {
|
||||||
@@ -31,6 +40,7 @@ struct Args {
|
|||||||
fullscreen: bool,
|
fullscreen: bool,
|
||||||
mode: String,
|
mode: String,
|
||||||
application: Option<String>,
|
application: Option<String>,
|
||||||
|
media_udp: Option<SocketAddr>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
@@ -47,9 +57,15 @@ struct Frame {
|
|||||||
enum ViewerEvent {
|
enum ViewerEvent {
|
||||||
Connected,
|
Connected,
|
||||||
Frame(Frame),
|
Frame(Frame),
|
||||||
|
Encoded(EncodedViewerFrame),
|
||||||
Failed(String),
|
Failed(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct EncodedViewerFrame {
|
||||||
|
access_unit: VideoAccessUnit,
|
||||||
|
pending: Arc<AtomicBool>,
|
||||||
|
}
|
||||||
|
|
||||||
fn main() -> ExitCode {
|
fn main() -> ExitCode {
|
||||||
match run() {
|
match run() {
|
||||||
Ok(()) => ExitCode::SUCCESS,
|
Ok(()) => ExitCode::SUCCESS,
|
||||||
@@ -68,16 +84,50 @@ fn run() -> Result<(), String> {
|
|||||||
.map_err(|error| error.to_string())?;
|
.map_err(|error| error.to_string())?;
|
||||||
let proxy = event_loop.create_proxy();
|
let proxy = event_loop.create_proxy();
|
||||||
let target = normalize_target(&args.target)?;
|
let target = normalize_target(&args.target)?;
|
||||||
|
let input = InputChannel::start(target.clone());
|
||||||
#[cfg(windows)]
|
#[cfg(windows)]
|
||||||
windows_keyboard_hook::install(target.clone());
|
windows_keyboard_hook::install(input.clone());
|
||||||
|
let frame_target = target.clone();
|
||||||
let fps = args.fps;
|
let fps = args.fps;
|
||||||
let mode = args.mode.clone();
|
let mode = args.mode.clone();
|
||||||
let application = args.application.clone();
|
let application = args.application.clone();
|
||||||
std::thread::Builder::new()
|
let encoded_media = args.media_udp.is_some();
|
||||||
.name("windows-agent-rdwf".to_owned())
|
if let Some(bind) = args.media_udp {
|
||||||
.spawn(move || receive_frames(&target, fps, &mode, application.as_deref(), &proxy))
|
let (audio_sender, audio_receiver) = mpsc::sync_channel(64);
|
||||||
.map_err(|error| error.to_string())?;
|
let media_clock = Arc::new(Mutex::new(audio::MediaClock::new()));
|
||||||
let mut app = ViewerApp::new(&event_loop, args.fullscreen, diagnostics, args.target.clone())?;
|
audio::start(audio_receiver, Arc::clone(&media_clock));
|
||||||
|
let media_input = input.clone();
|
||||||
|
let media_session_id = args.session_id.clone();
|
||||||
|
let media_clock_for_receiver = Arc::clone(&media_clock);
|
||||||
|
std::thread::Builder::new()
|
||||||
|
.name("windows-agent-rdv1".to_owned())
|
||||||
|
.spawn(move || {
|
||||||
|
receive_encoded_media(
|
||||||
|
bind,
|
||||||
|
&proxy,
|
||||||
|
&media_input,
|
||||||
|
&media_session_id,
|
||||||
|
audio_sender,
|
||||||
|
media_clock_for_receiver,
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.map_err(|error| error.to_string())?;
|
||||||
|
} else {
|
||||||
|
std::thread::Builder::new()
|
||||||
|
.name("windows-agent-rdwf".to_owned())
|
||||||
|
.spawn(move || {
|
||||||
|
receive_frames(&frame_target, fps, &mode, application.as_deref(), &proxy);
|
||||||
|
})
|
||||||
|
.map_err(|error| error.to_string())?;
|
||||||
|
}
|
||||||
|
let mut app = ViewerApp::new(
|
||||||
|
&event_loop,
|
||||||
|
args.fullscreen,
|
||||||
|
diagnostics,
|
||||||
|
input,
|
||||||
|
encoded_media,
|
||||||
|
fps,
|
||||||
|
)?;
|
||||||
event_loop
|
event_loop
|
||||||
.run_app(&mut app)
|
.run_app(&mut app)
|
||||||
.map_err(|error| error.to_string())
|
.map_err(|error| error.to_string())
|
||||||
@@ -90,6 +140,7 @@ fn parse_args(mut args: impl Iterator<Item = String>) -> Result<Args, String> {
|
|||||||
let mut fullscreen = false;
|
let mut fullscreen = false;
|
||||||
let mut mode = "desktop".to_owned();
|
let mut mode = "desktop".to_owned();
|
||||||
let mut application = None;
|
let mut application = None;
|
||||||
|
let mut media_udp = None;
|
||||||
while let Some(option) = args.next() {
|
while let Some(option) = args.next() {
|
||||||
match option.as_str() {
|
match option.as_str() {
|
||||||
"--target" => target = args.next(),
|
"--target" => target = args.next(),
|
||||||
@@ -98,16 +149,31 @@ fn parse_args(mut args: impl Iterator<Item = String>) -> Result<Args, String> {
|
|||||||
fps = args
|
fps = args
|
||||||
.next()
|
.next()
|
||||||
.and_then(|value| value.parse().ok())
|
.and_then(|value| value.parse().ok())
|
||||||
.filter(|value| (1..=15).contains(value))
|
.filter(|value| (1..=120).contains(value))
|
||||||
.ok_or_else(|| "--fps must be between 1 and 15".to_owned())?;
|
.ok_or_else(|| "--fps must be between 1 and 120".to_owned())?;
|
||||||
}
|
}
|
||||||
"--fullscreen" => fullscreen = true,
|
"--fullscreen" => fullscreen = true,
|
||||||
"--mode" => mode = args.next().ok_or_else(|| "--mode is required".to_owned())?,
|
"--mode" => mode = args.next().ok_or_else(|| "--mode is required".to_owned())?,
|
||||||
"--application" => application = args.next(),
|
"--application" => application = args.next(),
|
||||||
|
"--media-udp" => {
|
||||||
|
let value = args
|
||||||
|
.next()
|
||||||
|
.ok_or_else(|| "--media-udp is required".to_owned())?;
|
||||||
|
let address = value
|
||||||
|
.parse::<SocketAddr>()
|
||||||
|
.map_err(|_| "--media-udp must be an IP socket address".to_owned())?;
|
||||||
|
if !address.ip().is_loopback() {
|
||||||
|
return Err("--media-udp must bind to loopback".to_owned());
|
||||||
|
}
|
||||||
|
media_udp = Some(address);
|
||||||
|
}
|
||||||
_ => return Err(format!("unknown option {option}")),
|
_ => return Err(format!("unknown option {option}")),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let target = target.ok_or_else(|| "--target is required".to_owned())?;
|
let target = target.ok_or_else(|| "--target is required".to_owned())?;
|
||||||
|
if media_udp.is_none() && fps > 15 {
|
||||||
|
return Err("--fps above 15 requires --media-udp".to_owned());
|
||||||
|
}
|
||||||
let session_id = session_id.filter(|value| {
|
let session_id = session_id.filter(|value| {
|
||||||
!value.is_empty()
|
!value.is_empty()
|
||||||
&& value.len() <= 64
|
&& value.len() <= 64
|
||||||
@@ -125,9 +191,112 @@ fn parse_args(mut args: impl Iterator<Item = String>) -> Result<Args, String> {
|
|||||||
_ => return Err("--mode must be desktop or application".to_owned()),
|
_ => return Err("--mode must be desktop or application".to_owned()),
|
||||||
},
|
},
|
||||||
application,
|
application,
|
||||||
|
media_udp,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn receive_encoded_media(
|
||||||
|
bind: SocketAddr,
|
||||||
|
proxy: &winit::event_loop::EventLoopProxy<ViewerEvent>,
|
||||||
|
input: &InputChannel,
|
||||||
|
session_id: &str,
|
||||||
|
audio_sender: mpsc::SyncSender<remotedesk_protocol::AudioPacket>,
|
||||||
|
media_clock: Arc<Mutex<audio::MediaClock>>,
|
||||||
|
) {
|
||||||
|
if let Err(error) =
|
||||||
|
receive_encoded_media_inner(bind, proxy, input, session_id, audio_sender, media_clock)
|
||||||
|
{
|
||||||
|
let _ = proxy.send_event(ViewerEvent::Failed(error));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn receive_encoded_media_inner(
|
||||||
|
bind: SocketAddr,
|
||||||
|
proxy: &winit::event_loop::EventLoopProxy<ViewerEvent>,
|
||||||
|
input: &InputChannel,
|
||||||
|
session_id: &str,
|
||||||
|
audio_sender: mpsc::SyncSender<remotedesk_protocol::AudioPacket>,
|
||||||
|
media_clock: Arc<Mutex<audio::MediaClock>>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let socket =
|
||||||
|
UdpSocket::bind(bind).map_err(|error| format!("media UDP bind failed: {error}"))?;
|
||||||
|
let started = Instant::now();
|
||||||
|
let mut reassembler = VideoReassembler::new();
|
||||||
|
let mut buffer = vec![0_u8; AUDIO_DATAGRAM_MAX_BYTES.max(VIDEO_DATAGRAM_MAX_BYTES) + 1];
|
||||||
|
let mut connected = false;
|
||||||
|
let mut pinned_source = None;
|
||||||
|
let pending = Arc::new(AtomicBool::new(false));
|
||||||
|
let mut last_keyframe_request = None;
|
||||||
|
loop {
|
||||||
|
let (length, source) = socket
|
||||||
|
.recv_from(&mut buffer)
|
||||||
|
.map_err(|error| format!("RDV1 receive failed: {error}"))?;
|
||||||
|
if !source.ip().is_loopback()
|
||||||
|
|| length > buffer.len() - 1
|
||||||
|
|| pinned_source.is_some_and(|pinned| pinned != source)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if length >= 4 && &buffer[..4] == b"RDA1" {
|
||||||
|
if let Ok(packet) = decode_audio_datagram(&buffer[..length]) {
|
||||||
|
pinned_source.get_or_insert(source);
|
||||||
|
let _ = audio_sender.try_send(packet);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if length < 4 || &buffer[..4] != b"RDV1" || length > VIDEO_DATAGRAM_MAX_BYTES {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let access_unit = match reassembler.push_datagram(&buffer[..length], started.elapsed()) {
|
||||||
|
Ok(access_unit) => {
|
||||||
|
pinned_source.get_or_insert(source);
|
||||||
|
access_unit
|
||||||
|
}
|
||||||
|
Err(_) => continue,
|
||||||
|
};
|
||||||
|
if reassembler.keyframe_required()
|
||||||
|
&& last_keyframe_request
|
||||||
|
.is_none_or(|last: Instant| last.elapsed() >= Duration::from_millis(250))
|
||||||
|
{
|
||||||
|
input.send_priority(
|
||||||
|
serde_json::json!({"kind":"request_keyframe","session_id":session_id}),
|
||||||
|
);
|
||||||
|
last_keyframe_request = Some(Instant::now());
|
||||||
|
}
|
||||||
|
let Some(access_unit) = access_unit else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if media_clock
|
||||||
|
.lock()
|
||||||
|
.is_ok_and(|clock| !clock.should_present_video(access_unit.pts))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !connected {
|
||||||
|
proxy
|
||||||
|
.send_event(ViewerEvent::Connected)
|
||||||
|
.map_err(|_| "viewer window closed".to_owned())?;
|
||||||
|
connected = true;
|
||||||
|
}
|
||||||
|
if pending
|
||||||
|
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if proxy
|
||||||
|
.send_event(ViewerEvent::Encoded(EncodedViewerFrame {
|
||||||
|
access_unit,
|
||||||
|
pending: Arc::clone(&pending),
|
||||||
|
}))
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
pending.store(false, Ordering::Release);
|
||||||
|
return Err("viewer window closed".to_owned());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn normalize_target(target: &str) -> Result<String, String> {
|
fn normalize_target(target: &str) -> Result<String, String> {
|
||||||
if target.is_empty() || target.len() > 512 || target.contains(char::is_whitespace) {
|
if target.is_empty() || target.len() > 512 || target.contains(char::is_whitespace) {
|
||||||
return Err("Windows Agent target is invalid".to_owned());
|
return Err("Windows Agent target is invalid".to_owned());
|
||||||
@@ -186,9 +355,9 @@ fn receive_frames_inner(
|
|||||||
}
|
}
|
||||||
let command = if mode == "application" {
|
let command = if mode == "application" {
|
||||||
let id = application.ok_or_else(|| "应用模式未选择应用".to_owned())?;
|
let id = application.ok_or_else(|| "应用模式未选择应用".to_owned())?;
|
||||||
serde_json::json!({ "kind": "open_application", "application": id, "allow_software_fallback": true, "frames_per_second": fps })
|
serde_json::json!({ "kind": "open_application", "application": id, "frames_per_second": fps })
|
||||||
} else {
|
} else {
|
||||||
serde_json::json!({ "kind": "open_desktop", "allow_software_fallback": true, "frames_per_second": fps })
|
serde_json::json!({ "kind": "open_desktop", "frames_per_second": fps })
|
||||||
};
|
};
|
||||||
writeln!(
|
writeln!(
|
||||||
reader.get_mut(),
|
reader.get_mut(),
|
||||||
@@ -354,12 +523,16 @@ struct ViewerApp {
|
|||||||
surface: Option<softbuffer::Surface<OwnedDisplayHandle, Arc<Window>>>,
|
surface: Option<softbuffer::Surface<OwnedDisplayHandle, Arc<Window>>>,
|
||||||
frame: Option<Frame>,
|
frame: Option<Frame>,
|
||||||
fullscreen: bool,
|
fullscreen: bool,
|
||||||
target: String,
|
input: InputChannel,
|
||||||
cursor_position: (i32, i32),
|
cursor_position: (i32, i32),
|
||||||
modifiers: ModifiersState,
|
modifiers: ModifiersState,
|
||||||
control_down: bool,
|
control_down: bool,
|
||||||
alt_down: bool,
|
alt_down: bool,
|
||||||
diagnostics: Diagnostics,
|
diagnostics: Diagnostics,
|
||||||
|
encoded_media: bool,
|
||||||
|
encoded_frame_duration: Duration,
|
||||||
|
#[cfg(windows)]
|
||||||
|
native_renderer: Option<NativeH264Renderer>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ViewerApp {
|
impl ViewerApp {
|
||||||
@@ -367,7 +540,9 @@ impl ViewerApp {
|
|||||||
event_loop: &EventLoop<ViewerEvent>,
|
event_loop: &EventLoop<ViewerEvent>,
|
||||||
fullscreen: bool,
|
fullscreen: bool,
|
||||||
diagnostics: Diagnostics,
|
diagnostics: Diagnostics,
|
||||||
target: String,
|
input: InputChannel,
|
||||||
|
encoded_media: bool,
|
||||||
|
fps: u8,
|
||||||
) -> Result<Self, String> {
|
) -> Result<Self, String> {
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
context: softbuffer::Context::new(event_loop.owned_display_handle())
|
context: softbuffer::Context::new(event_loop.owned_display_handle())
|
||||||
@@ -376,12 +551,16 @@ impl ViewerApp {
|
|||||||
surface: None,
|
surface: None,
|
||||||
frame: None,
|
frame: None,
|
||||||
fullscreen,
|
fullscreen,
|
||||||
target,
|
input,
|
||||||
cursor_position: (0, 0),
|
cursor_position: (0, 0),
|
||||||
modifiers: ModifiersState::empty(),
|
modifiers: ModifiersState::empty(),
|
||||||
control_down: false,
|
control_down: false,
|
||||||
alt_down: false,
|
alt_down: false,
|
||||||
diagnostics,
|
diagnostics,
|
||||||
|
encoded_media,
|
||||||
|
encoded_frame_duration: Duration::from_secs_f64(1.0 / f64::from(fps.max(1))),
|
||||||
|
#[cfg(windows)]
|
||||||
|
native_renderer: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -441,7 +620,28 @@ impl ApplicationHandler<ViewerEvent> for ViewerApp {
|
|||||||
match event_loop.create_window(attributes) {
|
match event_loop.create_window(attributes) {
|
||||||
Ok(window) => {
|
Ok(window) => {
|
||||||
let window = Arc::new(window);
|
let window = Arc::new(window);
|
||||||
self.surface = softbuffer::Surface::new(&self.context, Arc::clone(&window)).ok();
|
if self.encoded_media {
|
||||||
|
#[cfg(windows)]
|
||||||
|
match NativeH264Renderer::new(Arc::clone(&window)) {
|
||||||
|
Ok(renderer) => self.native_renderer = Some(renderer),
|
||||||
|
Err(error) => {
|
||||||
|
self.diagnostics.failed(&error);
|
||||||
|
eprintln!("{error}");
|
||||||
|
event_loop.exit();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[cfg(not(windows))]
|
||||||
|
{
|
||||||
|
self.diagnostics
|
||||||
|
.failed("native H.264 rendering is Windows-only");
|
||||||
|
event_loop.exit();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
self.surface =
|
||||||
|
softbuffer::Surface::new(&self.context, Arc::clone(&window)).ok();
|
||||||
|
}
|
||||||
self.window = Some(window);
|
self.window = Some(window);
|
||||||
}
|
}
|
||||||
Err(_) => event_loop.exit(),
|
Err(_) => event_loop.exit(),
|
||||||
@@ -466,10 +666,14 @@ impl ApplicationHandler<ViewerEvent> for ViewerApp {
|
|||||||
WindowEvent::CursorMoved { position, .. } => {
|
WindowEvent::CursorMoved { position, .. } => {
|
||||||
if let Some(window) = &self.window {
|
if let Some(window) = &self.window {
|
||||||
let size = window.inner_size();
|
let size = window.inner_size();
|
||||||
let x = (position.x.max(0.0).min(size.width as f64) * 65535.0 / size.width.max(1) as f64) as i32;
|
let x = (position.x.max(0.0).min(size.width as f64) * 65535.0
|
||||||
let y = (position.y.max(0.0).min(size.height as f64) * 65535.0 / size.height.max(1) as f64) as i32;
|
/ size.width.max(1) as f64) as i32;
|
||||||
|
let y = (position.y.max(0.0).min(size.height as f64) * 65535.0
|
||||||
|
/ size.height.max(1) as f64) as i32;
|
||||||
self.cursor_position = (x, y);
|
self.cursor_position = (x, y);
|
||||||
send_input(&self.target, serde_json::json!({"kind":"input","input_type":"mouse","x":x,"y":y}));
|
self.input.send_motion(
|
||||||
|
serde_json::json!({"kind":"input","input_type":"mouse","x":x,"y":y}),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
WindowEvent::MouseInput { state, button, .. } => {
|
WindowEvent::MouseInput { state, button, .. } => {
|
||||||
@@ -482,27 +686,43 @@ impl ApplicationHandler<ViewerEvent> for ViewerApp {
|
|||||||
(MouseButton::Middle, ElementState::Released) => "middle_up",
|
(MouseButton::Middle, ElementState::Released) => "middle_up",
|
||||||
_ => return,
|
_ => return,
|
||||||
};
|
};
|
||||||
send_input(&self.target, serde_json::json!({"kind":"input","input_type":"mouse","mouse_action":action,"x":self.cursor_position.0,"y":self.cursor_position.1}));
|
self.input.send_priority(serde_json::json!({"kind":"input","input_type":"mouse","mouse_action":action,"x":self.cursor_position.0,"y":self.cursor_position.1}));
|
||||||
}
|
}
|
||||||
WindowEvent::MouseWheel { delta, .. } => {
|
WindowEvent::MouseWheel { delta, .. } => {
|
||||||
let value = match delta { MouseScrollDelta::LineDelta(_, y) => (y * 120.0) as i32, MouseScrollDelta::PixelDelta(p) => p.y as i32 };
|
let value = match delta {
|
||||||
send_input(&self.target, serde_json::json!({"kind":"input","input_type":"mouse","mouse_action":"wheel","buttons":value,"x":self.cursor_position.0,"y":self.cursor_position.1}));
|
MouseScrollDelta::LineDelta(_, y) => (y * 120.0) as i32,
|
||||||
|
MouseScrollDelta::PixelDelta(p) => p.y as i32,
|
||||||
|
};
|
||||||
|
self.input.send_priority(serde_json::json!({"kind":"input","input_type":"mouse","mouse_action":"wheel","buttons":value,"x":self.cursor_position.0,"y":self.cursor_position.1}));
|
||||||
}
|
}
|
||||||
WindowEvent::ModifiersChanged(modifiers) => self.modifiers = modifiers.state(),
|
WindowEvent::ModifiersChanged(modifiers) => self.modifiers = modifiers.state(),
|
||||||
WindowEvent::KeyboardInput { event, .. } => {
|
WindowEvent::KeyboardInput { event, .. } => {
|
||||||
if matches!(event.logical_key, Key::Named(NamedKey::Super)) {
|
if matches!(event.logical_key, Key::Named(NamedKey::Super)) {
|
||||||
let vk = if matches!(event.physical_key, PhysicalKey::Code(KeyCode::SuperRight)) { 0x5C } else { 0x5B };
|
let vk = if matches!(event.physical_key, PhysicalKey::Code(KeyCode::SuperRight))
|
||||||
send_input(&self.target, serde_json::json!({"kind":"input","input_type":"key","code":vk,"down":event.state == ElementState::Pressed}));
|
{
|
||||||
|
0x5C
|
||||||
|
} else {
|
||||||
|
0x5B
|
||||||
|
};
|
||||||
|
self.input.send_priority(serde_json::json!({"kind":"input","input_type":"key","code":vk,"down":event.state == ElementState::Pressed}));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let control = matches!(event.physical_key, PhysicalKey::Code(KeyCode::ControlLeft | KeyCode::ControlRight));
|
let control = matches!(
|
||||||
let alt = matches!(event.physical_key, PhysicalKey::Code(KeyCode::AltLeft | KeyCode::AltRight));
|
event.physical_key,
|
||||||
|
PhysicalKey::Code(KeyCode::ControlLeft | KeyCode::ControlRight)
|
||||||
|
);
|
||||||
|
let alt = matches!(
|
||||||
|
event.physical_key,
|
||||||
|
PhysicalKey::Code(KeyCode::AltLeft | KeyCode::AltRight)
|
||||||
|
);
|
||||||
if event.state == ElementState::Pressed {
|
if event.state == ElementState::Pressed {
|
||||||
self.control_down |= control;
|
self.control_down |= control;
|
||||||
self.alt_down |= alt;
|
self.alt_down |= alt;
|
||||||
if let PhysicalKey::Code(code) = event.physical_key {
|
if let PhysicalKey::Code(code) = event.physical_key {
|
||||||
let vk = virtual_key(code);
|
let vk = virtual_key(code);
|
||||||
if vk != 0 { send_input(&self.target, serde_json::json!({"kind":"input","input_type":"key","code":vk,"down":true})); }
|
if vk != 0 {
|
||||||
|
self.input.send_priority(serde_json::json!({"kind":"input","input_type":"key","code":vk,"down":true}));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if matches!(event.physical_key, PhysicalKey::Code(KeyCode::KeyF))
|
if matches!(event.physical_key, PhysicalKey::Code(KeyCode::KeyF))
|
||||||
&& (self.control_down || self.modifiers.control_key())
|
&& (self.control_down || self.modifiers.control_key())
|
||||||
@@ -510,16 +730,25 @@ impl ApplicationHandler<ViewerEvent> for ViewerApp {
|
|||||||
{
|
{
|
||||||
self.fullscreen = !self.fullscreen;
|
self.fullscreen = !self.fullscreen;
|
||||||
if let Some(window) = &self.window {
|
if let Some(window) = &self.window {
|
||||||
window.set_fullscreen(self.fullscreen.then(|| Fullscreen::Borderless(window.current_monitor())));
|
window.set_fullscreen(
|
||||||
|
self.fullscreen
|
||||||
|
.then(|| Fullscreen::Borderless(window.current_monitor())),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if let PhysicalKey::Code(code) = event.physical_key {
|
if let PhysicalKey::Code(code) = event.physical_key {
|
||||||
let vk = virtual_key(code);
|
let vk = virtual_key(code);
|
||||||
if vk != 0 { send_input(&self.target, serde_json::json!({"kind":"input","input_type":"key","code":vk,"down":false})); }
|
if vk != 0 {
|
||||||
|
self.input.send_priority(serde_json::json!({"kind":"input","input_type":"key","code":vk,"down":false}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if control {
|
||||||
|
self.control_down = false;
|
||||||
|
}
|
||||||
|
if alt {
|
||||||
|
self.alt_down = false;
|
||||||
}
|
}
|
||||||
if control { self.control_down = false; }
|
|
||||||
if alt { self.alt_down = false; }
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ => {}
|
_ => {}
|
||||||
@@ -536,6 +765,43 @@ impl ApplicationHandler<ViewerEvent> for ViewerApp {
|
|||||||
window.request_redraw();
|
window.request_redraw();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
ViewerEvent::Encoded(encoded) => {
|
||||||
|
encoded.pending.store(false, Ordering::Release);
|
||||||
|
#[cfg(windows)]
|
||||||
|
{
|
||||||
|
let Some(renderer) = self.native_renderer.as_mut() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let input = H264AccessUnit {
|
||||||
|
data: encoded.access_unit.payload,
|
||||||
|
duration: self.encoded_frame_duration,
|
||||||
|
};
|
||||||
|
match renderer.decode_and_present(&input) {
|
||||||
|
Ok(Some(presentation)) => {
|
||||||
|
self.diagnostics.native_frame(
|
||||||
|
presentation.width,
|
||||||
|
presentation.height,
|
||||||
|
presentation.frames,
|
||||||
|
presentation.decode_latency_us,
|
||||||
|
presentation.presentation_latency_us,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(None) => {}
|
||||||
|
Err(error) => {
|
||||||
|
self.diagnostics.failed(&error);
|
||||||
|
eprintln!("{error}");
|
||||||
|
event_loop.exit();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[cfg(not(windows))]
|
||||||
|
{
|
||||||
|
let _ = encoded.access_unit;
|
||||||
|
self.diagnostics
|
||||||
|
.failed("native H.264 rendering is Windows-only");
|
||||||
|
event_loop.exit();
|
||||||
|
}
|
||||||
|
}
|
||||||
ViewerEvent::Failed(error) => {
|
ViewerEvent::Failed(error) => {
|
||||||
eprintln!("{error}");
|
eprintln!("{error}");
|
||||||
self.diagnostics.failed(&error);
|
self.diagnostics.failed(&error);
|
||||||
@@ -551,16 +817,42 @@ impl ApplicationHandler<ViewerEvent> for ViewerApp {
|
|||||||
|
|
||||||
fn virtual_key(code: KeyCode) -> u16 {
|
fn virtual_key(code: KeyCode) -> u16 {
|
||||||
match code {
|
match code {
|
||||||
KeyCode::KeyA => 0x41, KeyCode::KeyB => 0x42, KeyCode::KeyC => 0x43, KeyCode::KeyD => 0x44,
|
KeyCode::KeyA => 0x41,
|
||||||
KeyCode::KeyE => 0x45, KeyCode::KeyF => 0x46, KeyCode::KeyG => 0x47, KeyCode::KeyH => 0x48,
|
KeyCode::KeyB => 0x42,
|
||||||
KeyCode::KeyI => 0x49, KeyCode::KeyJ => 0x4A, KeyCode::KeyK => 0x4B, KeyCode::KeyL => 0x4C,
|
KeyCode::KeyC => 0x43,
|
||||||
KeyCode::KeyM => 0x4D, KeyCode::KeyN => 0x4E, KeyCode::KeyO => 0x4F, KeyCode::KeyP => 0x50,
|
KeyCode::KeyD => 0x44,
|
||||||
KeyCode::KeyQ => 0x51, KeyCode::KeyR => 0x52, KeyCode::KeyS => 0x53, KeyCode::KeyT => 0x54,
|
KeyCode::KeyE => 0x45,
|
||||||
KeyCode::KeyU => 0x55, KeyCode::KeyV => 0x56, KeyCode::KeyW => 0x57, KeyCode::KeyX => 0x58,
|
KeyCode::KeyF => 0x46,
|
||||||
KeyCode::KeyY => 0x59, KeyCode::KeyZ => 0x5A,
|
KeyCode::KeyG => 0x47,
|
||||||
KeyCode::Digit0 => 0x30, KeyCode::Digit1 => 0x31, KeyCode::Digit2 => 0x32, KeyCode::Digit3 => 0x33,
|
KeyCode::KeyH => 0x48,
|
||||||
KeyCode::Digit4 => 0x34, KeyCode::Digit5 => 0x35, KeyCode::Digit6 => 0x36, KeyCode::Digit7 => 0x37,
|
KeyCode::KeyI => 0x49,
|
||||||
KeyCode::Digit8 => 0x38, KeyCode::Digit9 => 0x39,
|
KeyCode::KeyJ => 0x4A,
|
||||||
|
KeyCode::KeyK => 0x4B,
|
||||||
|
KeyCode::KeyL => 0x4C,
|
||||||
|
KeyCode::KeyM => 0x4D,
|
||||||
|
KeyCode::KeyN => 0x4E,
|
||||||
|
KeyCode::KeyO => 0x4F,
|
||||||
|
KeyCode::KeyP => 0x50,
|
||||||
|
KeyCode::KeyQ => 0x51,
|
||||||
|
KeyCode::KeyR => 0x52,
|
||||||
|
KeyCode::KeyS => 0x53,
|
||||||
|
KeyCode::KeyT => 0x54,
|
||||||
|
KeyCode::KeyU => 0x55,
|
||||||
|
KeyCode::KeyV => 0x56,
|
||||||
|
KeyCode::KeyW => 0x57,
|
||||||
|
KeyCode::KeyX => 0x58,
|
||||||
|
KeyCode::KeyY => 0x59,
|
||||||
|
KeyCode::KeyZ => 0x5A,
|
||||||
|
KeyCode::Digit0 => 0x30,
|
||||||
|
KeyCode::Digit1 => 0x31,
|
||||||
|
KeyCode::Digit2 => 0x32,
|
||||||
|
KeyCode::Digit3 => 0x33,
|
||||||
|
KeyCode::Digit4 => 0x34,
|
||||||
|
KeyCode::Digit5 => 0x35,
|
||||||
|
KeyCode::Digit6 => 0x36,
|
||||||
|
KeyCode::Digit7 => 0x37,
|
||||||
|
KeyCode::Digit8 => 0x38,
|
||||||
|
KeyCode::Digit9 => 0x39,
|
||||||
KeyCode::ControlLeft | KeyCode::ControlRight => 0x11,
|
KeyCode::ControlLeft | KeyCode::ControlRight => 0x11,
|
||||||
KeyCode::AltLeft | KeyCode::AltRight => 0x12,
|
KeyCode::AltLeft | KeyCode::AltRight => 0x12,
|
||||||
KeyCode::SuperLeft => 0x5B,
|
KeyCode::SuperLeft => 0x5B,
|
||||||
@@ -572,26 +864,106 @@ fn virtual_key(code: KeyCode) -> u16 {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn send_input(target: &str, command: serde_json::Value) {
|
#[derive(Clone)]
|
||||||
let Ok(mut stream) = TcpStream::connect_timeout(
|
struct InputChannel {
|
||||||
&match target.to_socket_addrs().ok().and_then(|mut addrs| addrs.next()) {
|
priority: mpsc::SyncSender<serde_json::Value>,
|
||||||
Some(address) => address,
|
latest_motion: Arc<Mutex<Option<serde_json::Value>>>,
|
||||||
None => return,
|
}
|
||||||
},
|
|
||||||
Duration::from_secs(2),
|
impl InputChannel {
|
||||||
) else { return };
|
fn start(target: String) -> Self {
|
||||||
let _ = stream.set_nodelay(true);
|
let (priority, receiver) = mpsc::sync_channel(INPUT_QUEUE_CAPACITY);
|
||||||
let _ = stream.set_read_timeout(Some(Duration::from_secs(2)));
|
let latest_motion = Arc::new(Mutex::new(None));
|
||||||
let Ok(clone) = stream.try_clone() else { return };
|
let worker_motion = Arc::clone(&latest_motion);
|
||||||
let mut reader = BufReader::with_capacity(1, clone);
|
let _ = std::thread::Builder::new()
|
||||||
let Ok(hello) = read_json_line(&mut reader) else { return };
|
.name("windows-agent-input".to_owned())
|
||||||
if hello.get("kind").and_then(serde_json::Value::as_str) != Some("windows_agent_hello") { return; }
|
.spawn(move || input_worker(&target, &receiver, &worker_motion));
|
||||||
if writeln!(stream, "{}", command).is_err() { return; }
|
Self {
|
||||||
if let Ok(response) = read_json_line(&mut reader) {
|
priority,
|
||||||
if response.get("kind").and_then(serde_json::Value::as_str) == Some("input_failed") {
|
latest_motion,
|
||||||
eprintln!("Windows Agent input failed: {}", response.get("error").and_then(serde_json::Value::as_str).unwrap_or("unknown"));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn send_priority(&self, command: serde_json::Value) {
|
||||||
|
if let Err(mpsc::TrySendError::Full(_)) = self.priority.try_send(command) {
|
||||||
|
eprintln!("Windows Agent input queue is full");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn send_motion(&self, command: serde_json::Value) {
|
||||||
|
if let Ok(mut latest) = self.latest_motion.lock() {
|
||||||
|
*latest = Some(command);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn input_worker(
|
||||||
|
target: &str,
|
||||||
|
priority: &mpsc::Receiver<serde_json::Value>,
|
||||||
|
latest_motion: &Mutex<Option<serde_json::Value>>,
|
||||||
|
) {
|
||||||
|
let mut stream = None;
|
||||||
|
loop {
|
||||||
|
let command = match priority.recv_timeout(Duration::from_millis(4)) {
|
||||||
|
Ok(command) => Some(command),
|
||||||
|
Err(mpsc::RecvTimeoutError::Timeout) => latest_motion
|
||||||
|
.lock()
|
||||||
|
.ok()
|
||||||
|
.and_then(|mut latest| latest.take()),
|
||||||
|
Err(mpsc::RecvTimeoutError::Disconnected) => return,
|
||||||
|
};
|
||||||
|
let Some(command) = command else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if stream.is_none() {
|
||||||
|
stream = connect_input_channel(target).ok();
|
||||||
|
}
|
||||||
|
let Some(active) = stream.as_mut() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if writeln!(active, "{command}").is_err() {
|
||||||
|
stream = None;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn connect_input_channel(target: &str) -> Result<TcpStream, String> {
|
||||||
|
let address = target
|
||||||
|
.to_socket_addrs()
|
||||||
|
.map_err(|error| error.to_string())?
|
||||||
|
.next()
|
||||||
|
.ok_or_else(|| "Windows Agent input target did not resolve".to_owned())?;
|
||||||
|
let stream = TcpStream::connect_timeout(&address, Duration::from_secs(2))
|
||||||
|
.map_err(|error| error.to_string())?;
|
||||||
|
stream
|
||||||
|
.set_nodelay(true)
|
||||||
|
.map_err(|error| error.to_string())?;
|
||||||
|
stream
|
||||||
|
.set_write_timeout(Some(Duration::from_secs(2)))
|
||||||
|
.map_err(|error| error.to_string())?;
|
||||||
|
let reader_stream = stream.try_clone().map_err(|error| error.to_string())?;
|
||||||
|
let mut reader = BufReader::with_capacity(1024, reader_stream);
|
||||||
|
let hello = read_json_line(&mut reader)?;
|
||||||
|
if hello.get("kind").and_then(serde_json::Value::as_str) != Some("windows_agent_hello") {
|
||||||
|
return Err("input endpoint is not a Windows Agent".to_owned());
|
||||||
|
}
|
||||||
|
let _ = std::thread::Builder::new()
|
||||||
|
.name("windows-agent-input-ack".to_owned())
|
||||||
|
.spawn(move || {
|
||||||
|
while let Ok(response) = read_json_line(&mut reader) {
|
||||||
|
if response.get("kind").and_then(serde_json::Value::as_str) == Some("input_failed")
|
||||||
|
{
|
||||||
|
eprintln!(
|
||||||
|
"Windows Agent input failed: {}",
|
||||||
|
response
|
||||||
|
.get("error")
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
.unwrap_or("unknown")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
Ok(stream)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
@@ -628,6 +1000,7 @@ struct Diagnostics {
|
|||||||
decode: Option<f64>,
|
decode: Option<f64>,
|
||||||
presentation: Option<f64>,
|
presentation: Option<f64>,
|
||||||
error: Option<String>,
|
error: Option<String>,
|
||||||
|
renderer: &'static str,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Diagnostics {
|
impl Diagnostics {
|
||||||
@@ -651,6 +1024,7 @@ impl Diagnostics {
|
|||||||
decode: None,
|
decode: None,
|
||||||
presentation: None,
|
presentation: None,
|
||||||
error: None,
|
error: None,
|
||||||
|
renderer: "softbuffer-bgra",
|
||||||
};
|
};
|
||||||
value.write()?;
|
value.write()?;
|
||||||
Ok(value)
|
Ok(value)
|
||||||
@@ -680,6 +1054,32 @@ impl Diagnostics {
|
|||||||
self.presentation = Some(elapsed.as_secs_f64() * 1_000.0);
|
self.presentation = Some(elapsed.as_secs_f64() * 1_000.0);
|
||||||
let _ = self.write();
|
let _ = self.write();
|
||||||
}
|
}
|
||||||
|
fn native_frame(
|
||||||
|
&mut self,
|
||||||
|
width: u32,
|
||||||
|
height: u32,
|
||||||
|
frames: usize,
|
||||||
|
decode_latency_us: u64,
|
||||||
|
presentation_latency_us: u64,
|
||||||
|
) {
|
||||||
|
self.state = "connected";
|
||||||
|
let frames = u64::try_from(frames).unwrap_or(u64::MAX);
|
||||||
|
self.frame_count = self.frame_count.saturating_add(frames);
|
||||||
|
self.fps_frames = self
|
||||||
|
.fps_frames
|
||||||
|
.saturating_add(u32::try_from(frames).unwrap_or(u32::MAX));
|
||||||
|
self.size = (width, height);
|
||||||
|
self.decode = Some(micros_ms(decode_latency_us));
|
||||||
|
self.presentation = Some(micros_ms(presentation_latency_us));
|
||||||
|
self.renderer = "media-foundation-d3d11";
|
||||||
|
let elapsed = self.fps_started.elapsed();
|
||||||
|
if elapsed >= Duration::from_secs(1) {
|
||||||
|
self.fps = Some(f64::from(self.fps_frames) / elapsed.as_secs_f64());
|
||||||
|
self.fps_frames = 0;
|
||||||
|
self.fps_started = Instant::now();
|
||||||
|
}
|
||||||
|
let _ = self.write();
|
||||||
|
}
|
||||||
fn failed(&mut self, error: &str) {
|
fn failed(&mut self, error: &str) {
|
||||||
self.state = "failed";
|
self.state = "failed";
|
||||||
self.error = Some(error.chars().take(128).collect());
|
self.error = Some(error.chars().take(128).collect());
|
||||||
@@ -709,7 +1109,7 @@ impl Diagnostics {
|
|||||||
frame_processing_latency_ms: self.processing,
|
frame_processing_latency_ms: self.processing,
|
||||||
decode_latency_ms: self.decode,
|
decode_latency_ms: self.decode,
|
||||||
presentation_latency_ms: self.presentation,
|
presentation_latency_ms: self.presentation,
|
||||||
renderer: "softbuffer-bgra",
|
renderer: self.renderer,
|
||||||
error_code: self.error.as_deref(),
|
error_code: self.error.as_deref(),
|
||||||
};
|
};
|
||||||
let data = serde_json::to_vec(&snapshot).map_err(|error| error.to_string())?;
|
let data = serde_json::to_vec(&snapshot).map_err(|error| error.to_string())?;
|
||||||
@@ -732,4 +1132,59 @@ mod tests {
|
|||||||
assert_eq!(normalize_target("10.0.0.2").unwrap(), "10.0.0.2:39501");
|
assert_eq!(normalize_target("10.0.0.2").unwrap(), "10.0.0.2:39501");
|
||||||
assert_eq!(normalize_target("10.0.0.2:4000").unwrap(), "10.0.0.2:4000");
|
assert_eq!(normalize_target("10.0.0.2:4000").unwrap(), "10.0.0.2:4000");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn encoded_media_requires_loopback_and_allows_120_fps() {
|
||||||
|
let args = parse_args(
|
||||||
|
[
|
||||||
|
"--target",
|
||||||
|
"10.0.0.2",
|
||||||
|
"--session-id",
|
||||||
|
"session-1",
|
||||||
|
"--fps",
|
||||||
|
"120",
|
||||||
|
"--media-udp",
|
||||||
|
"127.0.0.1:39502",
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.map(str::to_owned),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(args.fps, 120);
|
||||||
|
assert_eq!(args.media_udp.unwrap().to_string(), "127.0.0.1:39502");
|
||||||
|
assert!(
|
||||||
|
parse_args(
|
||||||
|
[
|
||||||
|
"--target",
|
||||||
|
"10.0.0.2",
|
||||||
|
"--session-id",
|
||||||
|
"session-1",
|
||||||
|
"--media-udp",
|
||||||
|
"0.0.0.0:39502",
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.map(str::to_owned),
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rdwf_mode_keeps_fifteen_fps_limit() {
|
||||||
|
assert!(
|
||||||
|
parse_args(
|
||||||
|
[
|
||||||
|
"--target",
|
||||||
|
"10.0.0.2",
|
||||||
|
"--session-id",
|
||||||
|
"session-1",
|
||||||
|
"--fps",
|
||||||
|
"16",
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.map(str::to_owned),
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,20 +1,26 @@
|
|||||||
#![cfg(windows)]
|
#![cfg(windows)]
|
||||||
|
|
||||||
use std::net::{TcpStream, ToSocketAddrs};
|
use crate::InputChannel;
|
||||||
|
use serde_json::json;
|
||||||
use std::sync::OnceLock;
|
use std::sync::OnceLock;
|
||||||
use std::thread;
|
use std::thread;
|
||||||
use std::time::Duration;
|
|
||||||
use windows::Win32::Foundation::{LPARAM, LRESULT, WPARAM};
|
use windows::Win32::Foundation::{LPARAM, LRESULT, WPARAM};
|
||||||
use windows::Win32::UI::WindowsAndMessaging::{CallNextHookEx, DispatchMessageW, GetForegroundWindow, GetMessageW, GetWindowThreadProcessId, KBDLLHOOKSTRUCT, MSG, SetWindowsHookExW, TranslateMessage, UnhookWindowsHookEx, WH_KEYBOARD_LL, WM_KEYDOWN, WM_KEYUP, WM_SYSKEYDOWN, WM_SYSKEYUP};
|
use windows::Win32::UI::WindowsAndMessaging::{
|
||||||
|
CallNextHookEx, DispatchMessageW, GetForegroundWindow, GetMessageW, GetWindowThreadProcessId,
|
||||||
|
KBDLLHOOKSTRUCT, MSG, SetWindowsHookExW, TranslateMessage, UnhookWindowsHookEx, WH_KEYBOARD_LL,
|
||||||
|
WM_KEYDOWN, WM_KEYUP, WM_SYSKEYDOWN, WM_SYSKEYUP,
|
||||||
|
};
|
||||||
|
|
||||||
static PROCESS_ID: OnceLock<u32> = OnceLock::new();
|
static PROCESS_ID: OnceLock<u32> = OnceLock::new();
|
||||||
static TARGET: OnceLock<String> = OnceLock::new();
|
static INPUT: OnceLock<InputChannel> = OnceLock::new();
|
||||||
|
|
||||||
pub fn install(target: String) {
|
pub fn install(input: InputChannel) {
|
||||||
let _ = PROCESS_ID.set(std::process::id());
|
let _ = PROCESS_ID.set(std::process::id());
|
||||||
let _ = TARGET.set(target);
|
let _ = INPUT.set(input);
|
||||||
thread::spawn(move || unsafe {
|
thread::spawn(move || unsafe {
|
||||||
let Ok(hook) = SetWindowsHookExW(WH_KEYBOARD_LL, Some(callback), None, 0) else { return };
|
let Ok(hook) = SetWindowsHookExW(WH_KEYBOARD_LL, Some(callback), None, 0) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
let mut message = MSG::default();
|
let mut message = MSG::default();
|
||||||
while GetMessageW(&mut message, None, 0, 0).as_bool() {
|
while GetMessageW(&mut message, None, 0, 0).as_bool() {
|
||||||
let _ = TranslateMessage(&message);
|
let _ = TranslateMessage(&message);
|
||||||
@@ -32,7 +38,14 @@ unsafe extern "system" fn callback(code: i32, wparam: WPARAM, lparam: LPARAM) ->
|
|||||||
let pressed = matches!(message, WM_KEYDOWN | WM_SYSKEYDOWN);
|
let pressed = matches!(message, WM_KEYDOWN | WM_SYSKEYDOWN);
|
||||||
let released = matches!(message, WM_KEYUP | WM_SYSKEYUP);
|
let released = matches!(message, WM_KEYUP | WM_SYSKEYUP);
|
||||||
if pressed || released {
|
if pressed || released {
|
||||||
send_remote(data.vkCode as u16, pressed);
|
if let Some(input) = INPUT.get() {
|
||||||
|
input.send_priority(json!({
|
||||||
|
"kind": "input",
|
||||||
|
"input_type": "key",
|
||||||
|
"code": data.vkCode,
|
||||||
|
"down": pressed
|
||||||
|
}));
|
||||||
|
}
|
||||||
return LRESULT(1);
|
return LRESULT(1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -41,18 +54,14 @@ unsafe extern "system" fn callback(code: i32, wparam: WPARAM, lparam: LPARAM) ->
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn foreground_is_viewer() -> bool {
|
fn foreground_is_viewer() -> bool {
|
||||||
let Some(pid) = PROCESS_ID.get().copied() else { return false };
|
let Some(pid) = PROCESS_ID.get().copied() else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
let window = unsafe { GetForegroundWindow() };
|
let window = unsafe { GetForegroundWindow() };
|
||||||
if window.0.is_null() { return false }
|
if window.0.is_null() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
let mut foreground_pid = 0;
|
let mut foreground_pid = 0;
|
||||||
unsafe { GetWindowThreadProcessId(window, Some(&mut foreground_pid)) };
|
unsafe { GetWindowThreadProcessId(window, Some(&mut foreground_pid)) };
|
||||||
foreground_pid == pid
|
foreground_pid == pid
|
||||||
}
|
}
|
||||||
|
|
||||||
fn send_remote(vk: u16, pressed: bool) {
|
|
||||||
let Some(target) = TARGET.get() else { return };
|
|
||||||
let Some(address) = target.to_socket_addrs().ok().and_then(|mut values| values.next()) else { return };
|
|
||||||
let Ok(mut stream) = TcpStream::connect_timeout(&address, Duration::from_millis(250)) else { return };
|
|
||||||
let _ = stream.set_nodelay(true);
|
|
||||||
let _ = std::io::Write::write_all(&mut stream, format!("{{\"kind\":\"input\",\"input_type\":\"key\",\"code\":{vk},\"down\":{pressed}}}\n").as_bytes());
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
[package]
|
||||||
|
name = "remotedesk-native-gui"
|
||||||
|
version.workspace = true
|
||||||
|
edition.workspace = true
|
||||||
|
license.workspace = true
|
||||||
|
rust-version.workspace = true
|
||||||
|
|
||||||
|
[[bin]]
|
||||||
|
name = "remotedesk-native"
|
||||||
|
path = "src/main.rs"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
egui = "0.31"
|
||||||
|
egui-wgpu = "0.31"
|
||||||
|
egui-winit = "0.31"
|
||||||
|
pollster = "0.4"
|
||||||
|
wgpu = "24"
|
||||||
|
winit = "0.30"
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
@@ -0,0 +1,215 @@
|
|||||||
|
#![forbid(unsafe_code)]
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use egui_wgpu::Renderer;
|
||||||
|
use egui_winit::State;
|
||||||
|
use wgpu::SurfaceError;
|
||||||
|
use winit::application::ApplicationHandler;
|
||||||
|
use winit::dpi::PhysicalSize;
|
||||||
|
use winit::event::WindowEvent;
|
||||||
|
use winit::event_loop::{ActiveEventLoop, EventLoop};
|
||||||
|
use winit::window::{Window, WindowId};
|
||||||
|
|
||||||
|
struct App {
|
||||||
|
window: Option<Arc<Window>>,
|
||||||
|
surface: Option<wgpu::Surface<'static>>,
|
||||||
|
device: Option<wgpu::Device>,
|
||||||
|
queue: Option<wgpu::Queue>,
|
||||||
|
config: Option<wgpu::SurfaceConfiguration>,
|
||||||
|
renderer: Option<Renderer>,
|
||||||
|
egui_state: Option<State>,
|
||||||
|
started: Instant,
|
||||||
|
endpoint: String,
|
||||||
|
status: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for App {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
window: None,
|
||||||
|
surface: None,
|
||||||
|
device: None,
|
||||||
|
queue: None,
|
||||||
|
config: None,
|
||||||
|
renderer: None,
|
||||||
|
egui_state: None,
|
||||||
|
started: Instant::now(),
|
||||||
|
endpoint: String::new(),
|
||||||
|
status: "Disconnected".to_owned(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ApplicationHandler for App {
|
||||||
|
fn resumed(&mut self, event_loop: &ActiveEventLoop) {
|
||||||
|
if self.window.is_some() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let window = Arc::new(
|
||||||
|
event_loop
|
||||||
|
.create_window(
|
||||||
|
Window::default_attributes()
|
||||||
|
.with_title("RemoteDesk")
|
||||||
|
.with_inner_size(PhysicalSize::new(1280, 800)),
|
||||||
|
)
|
||||||
|
.expect("create RemoteDesk window"),
|
||||||
|
);
|
||||||
|
let instance = wgpu::Instance::default();
|
||||||
|
let surface = instance
|
||||||
|
.create_surface(window.clone())
|
||||||
|
.expect("create GPU surface");
|
||||||
|
let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions {
|
||||||
|
power_preference: wgpu::PowerPreference::HighPerformance,
|
||||||
|
compatible_surface: Some(&surface),
|
||||||
|
force_fallback_adapter: false,
|
||||||
|
}))
|
||||||
|
.expect("find a compatible GPU adapter");
|
||||||
|
let (device, queue) =
|
||||||
|
pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default()))
|
||||||
|
.expect("create GPU device");
|
||||||
|
let size = window.inner_size();
|
||||||
|
let capabilities = surface.get_capabilities(&adapter);
|
||||||
|
let format = capabilities.formats[0];
|
||||||
|
let config = wgpu::SurfaceConfiguration {
|
||||||
|
usage: wgpu::TextureUsages::RENDER_ATTACHMENT,
|
||||||
|
format,
|
||||||
|
width: size.width.max(1),
|
||||||
|
height: size.height.max(1),
|
||||||
|
present_mode: capabilities.present_modes[0],
|
||||||
|
alpha_mode: capabilities.alpha_modes[0],
|
||||||
|
view_formats: vec![],
|
||||||
|
desired_maximum_frame_latency: 2,
|
||||||
|
};
|
||||||
|
surface.configure(&device, &config);
|
||||||
|
self.egui_state = Some(State::new(
|
||||||
|
egui::ViewportId::ROOT,
|
||||||
|
event_loop,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
));
|
||||||
|
self.renderer = Some(Renderer::new(&device, format, None, 1, false));
|
||||||
|
self.window = Some(window);
|
||||||
|
self.surface = Some(surface);
|
||||||
|
self.device = Some(device);
|
||||||
|
self.queue = Some(queue);
|
||||||
|
self.config = Some(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn window_event(
|
||||||
|
&mut self,
|
||||||
|
event_loop: &ActiveEventLoop,
|
||||||
|
window_id: WindowId,
|
||||||
|
event: WindowEvent,
|
||||||
|
) {
|
||||||
|
let Some(window) = self.window.clone() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if window.id() != window_id {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if let WindowEvent::CloseRequested = event {
|
||||||
|
event_loop.exit();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let Some(state) = self.egui_state.as_mut() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let response = state.on_window_event(&window, &event);
|
||||||
|
if response.repaint {
|
||||||
|
window.request_redraw();
|
||||||
|
}
|
||||||
|
if let WindowEvent::Resized(size) = event {
|
||||||
|
if let (Some(surface), Some(device), Some(config)) =
|
||||||
|
(&self.surface, &self.device, &mut self.config)
|
||||||
|
{
|
||||||
|
config.width = size.width.max(1);
|
||||||
|
config.height = size.height.max(1);
|
||||||
|
surface.configure(device, config);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let WindowEvent::RedrawRequested = event {
|
||||||
|
self.redraw(&window);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl App {
|
||||||
|
fn redraw(&mut self, window: &Window) {
|
||||||
|
let (Some(surface), Some(device), Some(queue), Some(config), Some(renderer), Some(state)) = (
|
||||||
|
&self.surface,
|
||||||
|
&self.device,
|
||||||
|
&self.queue,
|
||||||
|
&self.config,
|
||||||
|
&mut self.renderer,
|
||||||
|
&mut self.egui_state,
|
||||||
|
) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let raw_input = state.take_egui_input(window);
|
||||||
|
let full_output = egui::Context::default().run(raw_input, |ctx| {
|
||||||
|
egui::CentralPanel::default().show(ctx, |ui| {
|
||||||
|
ui.heading("RemoteDesk");
|
||||||
|
ui.label("Native cross-platform client");
|
||||||
|
ui.separator();
|
||||||
|
ui.horizontal(|ui| {
|
||||||
|
ui.label("Server");
|
||||||
|
ui.text_edit_singleline(&mut self.endpoint);
|
||||||
|
if ui.button("Connect").clicked() {
|
||||||
|
self.status = "Connecting".to_owned();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
ui.label(format!("Status: {}", self.status));
|
||||||
|
ui.label(format!(
|
||||||
|
"Uptime: {:.1}s",
|
||||||
|
self.started.elapsed().as_secs_f32()
|
||||||
|
));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
state.handle_platform_output(window, full_output.platform_output);
|
||||||
|
let output = match surface.get_current_texture() {
|
||||||
|
Ok(output) => output,
|
||||||
|
Err(SurfaceError::Lost) => {
|
||||||
|
surface.configure(device, config);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Err(_) => return,
|
||||||
|
};
|
||||||
|
let view = output
|
||||||
|
.texture
|
||||||
|
.create_view(&wgpu::TextureViewDescriptor::default());
|
||||||
|
let mut encoder = device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
|
||||||
|
label: Some("egui"),
|
||||||
|
});
|
||||||
|
let clipped =
|
||||||
|
egui::Context::default().tessellate(full_output.shapes, full_output.pixels_per_point);
|
||||||
|
renderer.update_buffers(
|
||||||
|
device,
|
||||||
|
queue,
|
||||||
|
&mut encoder,
|
||||||
|
&clipped,
|
||||||
|
&egui_wgpu::ScreenDescriptor {
|
||||||
|
size_in_pixels: [config.width, config.height],
|
||||||
|
pixels_per_point: full_output.pixels_per_point,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
renderer.render(
|
||||||
|
&mut encoder,
|
||||||
|
&view,
|
||||||
|
&clipped,
|
||||||
|
&egui_wgpu::ScreenDescriptor {
|
||||||
|
size_in_pixels: [config.width, config.height],
|
||||||
|
pixels_per_point: full_output.pixels_per_point,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
queue.submit([encoder.finish()]);
|
||||||
|
output.present();
|
||||||
|
window.request_redraw();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() -> Result<(), winit::error::EventLoopError> {
|
||||||
|
EventLoop::new()?.run_app(&mut App::default())
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
# ADR 0001: All-Rust RemoteDesk Runtime
|
||||||
|
|
||||||
|
- Status: Accepted
|
||||||
|
- Date: 2026-08-14
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
RemoteDesk uses Rust for every shipped runtime component. Go and Hysteria2 are
|
||||||
|
not part of the target architecture.
|
||||||
|
|
||||||
|
The product consists of a Rust Agent server, a Rust native Client, shared Rust
|
||||||
|
protocol crates, and optional Rust signaling and relay services required by
|
||||||
|
WebRTC.
|
||||||
|
|
||||||
|
The native Client uses winit for cross-platform windows and input events, wgpu
|
||||||
|
for GPU rendering, egui for the control interface, and str0m for WebRTC and
|
||||||
|
DataChannel support. Platform hardware codecs provide video decode and platform
|
||||||
|
audio APIs are exposed through Rust abstractions.
|
||||||
|
|
||||||
|
The Agent uses str0m WebRTC, platform capture and input APIs, platform hardware
|
||||||
|
video encoders, Opus audio, and Rust HTTPS/WebSocket signaling where required.
|
||||||
|
|
||||||
|
## Transport
|
||||||
|
|
||||||
|
WebRTC is the primary real-time transport:
|
||||||
|
|
||||||
|
- RTP/RTCP carries video and audio.
|
||||||
|
- A reliable ordered DataChannel carries authentication-bound control,
|
||||||
|
keyboard, clipboard, and session state.
|
||||||
|
- An unordered limited-retransmission DataChannel carries pointer movement and
|
||||||
|
disposable telemetry.
|
||||||
|
- ICE/STUN provides direct connectivity and TURN provides relay fallback.
|
||||||
|
|
||||||
|
Large file transfer and software updates use a separate reliable Rust
|
||||||
|
HTTPS/QUIC service so they cannot block real-time media or input.
|
||||||
|
|
||||||
|
## Media and rendering
|
||||||
|
|
||||||
|
Compressed video may pass through ordinary Rust byte buffers. Decoded raw
|
||||||
|
frames must remain on platform GPU surfaces and be rendered by the native
|
||||||
|
client without JavaScript, WebView, Tauri IPC, or CPU bitmap transport.
|
||||||
|
|
||||||
|
Target platform paths are Windows Graphics Capture or DXGI with Media
|
||||||
|
Foundation and D3D11/D3D12; Linux PipeWire, DMA-BUF, VA-API, and Vulkan; and
|
||||||
|
macOS ScreenCaptureKit, VideoToolbox, and Metal.
|
||||||
|
|
||||||
|
wgpu is the cross-platform presentation abstraction. Platform-specific interop
|
||||||
|
remains isolated behind Rust modules where wgpu cannot directly import a codec
|
||||||
|
surface.
|
||||||
|
|
||||||
|
## Removed architecture
|
||||||
|
|
||||||
|
The following are not part of the target architecture:
|
||||||
|
|
||||||
|
- Go runtime components and the Go Hysteria2 adapter.
|
||||||
|
- Hysteria2 as a default or optional transport.
|
||||||
|
- Tauri, WebView, React, or JavaScript as the shipped desktop GUI.
|
||||||
|
- Software video fallback and compatibility modes.
|
||||||
|
- A separately shipped Edge application unrelated to the Agent/Client model.
|
||||||
|
|
||||||
|
Existing source and historical documentation may temporarily describe these
|
||||||
|
components during migration. This ADR takes precedence wherever descriptions
|
||||||
|
conflict.
|
||||||
|
|
||||||
|
## Repository target
|
||||||
|
|
||||||
|
```text
|
||||||
|
agent/ Rust server, capture, encode, input, audio, and WebRTC
|
||||||
|
client/ Rust native client, GUI, decode, render, input, and audio
|
||||||
|
protocol/ Shared Rust protocol and message contracts
|
||||||
|
packaging/ Windows, Linux, and macOS packages for Agent and Client
|
||||||
|
docs/ Architecture decisions and operational documentation
|
||||||
|
```
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
- One language, toolchain, lockfile, and security review boundary covers the
|
||||||
|
shipped application.
|
||||||
|
- The client avoids WebView frame copies and browser-process overhead.
|
||||||
|
- WebRTC supplies NAT traversal and real-time media feedback instead of a
|
||||||
|
custom QUIC media protocol.
|
||||||
|
- str0m does not provide capture, codecs, rendering, or TURN; these remain
|
||||||
|
explicit platform and deployment responsibilities.
|
||||||
|
- Migration is complete only after Go/Hysteria2 and Tauri/Web assets are
|
||||||
|
removed from builds, packages, CI, and current documentation.
|
||||||
|
|
||||||
|
The ordered implementation and acceptance plan is maintained in
|
||||||
|
[All-Rust Migration Plan](../all-rust-migration.md).
|
||||||
@@ -0,0 +1,160 @@
|
|||||||
|
# 全 Rust 架构迁移计划
|
||||||
|
|
||||||
|
本文是 [ADR 0001](adr/0001-all-rust-runtime.md) 的实施计划。它描述目标和验收门槛,
|
||||||
|
不代表对应能力已经实现。实际完成度以 [实现状态](implementation-status.md) 为准。
|
||||||
|
|
||||||
|
## 迁移原则
|
||||||
|
|
||||||
|
- Agent 是服务端,Client 是原生客户端,发布运行时全部使用 Rust。
|
||||||
|
- 实时会话使用 str0m WebRTC;不自研替代 WebRTC 的媒体传输协议。
|
||||||
|
- 原始视频帧不得经过 JavaScript、WebView、Tauri IPC 或跨进程 CPU bitmap。
|
||||||
|
- 每一阶段先建立可重复测试和性能基线,再删除被替代实现。
|
||||||
|
- 迁移期间旧代码不得继续进入正式安装包或默认 CI 发布产物。
|
||||||
|
|
||||||
|
## 阶段 1:删除 Go/Hysteria2 构建链
|
||||||
|
|
||||||
|
状态:未开始。
|
||||||
|
|
||||||
|
工作:
|
||||||
|
|
||||||
|
- 从 Windows、Linux 和聚合打包脚本中删除 Hysteria2 构建步骤。
|
||||||
|
- 从 CI、发布清单、安装器、systemd 和环境模板中删除 Go/Hysteria2。
|
||||||
|
- 删除 transport/hysteria2-agent、go.mod、go.sum 及生成物引用。
|
||||||
|
- 移除 Go 工具链、缓存和供应链审计要求。
|
||||||
|
|
||||||
|
验收:Cargo workspace、CI 和所有安装包不调用 Go;仓库不存在被引用的 Hysteria2
|
||||||
|
二进制、服务或配置;Rust Agent 与 Client 的现有非 Hysteria2 构建仍通过。
|
||||||
|
|
||||||
|
## 阶段 2:新增共享 WebRTC Rust crate
|
||||||
|
|
||||||
|
状态:未开始。
|
||||||
|
|
||||||
|
工作:
|
||||||
|
|
||||||
|
- 新增共享 webrtc crate 并锁定 str0m 版本。
|
||||||
|
- 封装 ICE、SDP、DTLS、SRTP、RTP/RTCP 和 DataChannel 事件状态机。
|
||||||
|
- 定义 socket 驱动、时钟、超时、证书指纹和会话身份接口。
|
||||||
|
- 对所有消息、候选地址、SDP 和队列设置上限。
|
||||||
|
|
||||||
|
验收:crate 在 Windows、Linux 和 macOS 目标上编译;状态机具备确定性单元测试、
|
||||||
|
畸形输入测试和超时测试;上层代码不直接依赖 str0m 内部类型。
|
||||||
|
|
||||||
|
## 阶段 3:Agent/Client Loopback
|
||||||
|
|
||||||
|
状态:未开始。
|
||||||
|
|
||||||
|
工作:
|
||||||
|
|
||||||
|
- 在同机建立 Rust Agent 与 Rust Client 的 SDP offer/answer 交换。
|
||||||
|
- 支持 trickle ICE、候选结束、DTLS 指纹校验和 ICE restart。
|
||||||
|
- 建立可靠有序 control DataChannel。
|
||||||
|
- 建立不可靠无序 pointer DataChannel。
|
||||||
|
- 定义协议版本、消息大小、速率和权限边界。
|
||||||
|
|
||||||
|
验收:自动化测试完成 SDP、ICE、DTLS 和 DataChannel ping/pong;断开、重连、超时、
|
||||||
|
伪造指纹、乱序和超限消息均按预期失败;输入通道不被媒体测试流阻塞。
|
||||||
|
|
||||||
|
## 阶段 4:接入 H.264 RTP
|
||||||
|
|
||||||
|
状态:未开始。
|
||||||
|
|
||||||
|
工作:
|
||||||
|
|
||||||
|
- 固定 H.264 SDP profile、packetization-mode 和时钟频率。
|
||||||
|
- 实现 RFC 6184 单 NAL、STAP-A 和 FU-A 打包与重组。
|
||||||
|
- 接入 RTP 序列号、时间戳、帧边界、NACK、PLI、RTX 和关键帧恢复。
|
||||||
|
- 使用一到两帧有界队列并丢弃过期帧。
|
||||||
|
|
||||||
|
验收:录制码流可在 Agent 与 Client 间连续传输;随机和突发丢包测试可恢复;
|
||||||
|
缺片帧不呈现;关键帧丢失会请求新 IDR;媒体队列不会无界增长。
|
||||||
|
|
||||||
|
## 阶段 5:平台硬件编码与解码
|
||||||
|
|
||||||
|
状态:未开始。
|
||||||
|
|
||||||
|
工作:
|
||||||
|
|
||||||
|
- Windows 接入 Windows Graphics Capture 或 DXGI、Media Foundation 和 D3D11/D3D12。
|
||||||
|
- Linux 接入 PipeWire、DMA-BUF 和 VA-API。
|
||||||
|
- macOS 接入 ScreenCaptureKit、VideoToolbox 和 Metal。
|
||||||
|
- 将编码器、解码器和 GPU surface 约束到可验证的 Adapter/device。
|
||||||
|
- 不提供软件编码、软件解码或 CPU bitmap 兼容回退。
|
||||||
|
|
||||||
|
验收:每个平台至少一个硬件路径完成端到端测试;运行时可证明实际硬件后端、
|
||||||
|
surface 类型和 Adapter;硬件能力不足时明确拒绝会话,不静默降级。
|
||||||
|
|
||||||
|
## 阶段 6:接入 wgpu 呈现
|
||||||
|
|
||||||
|
状态:未开始。
|
||||||
|
|
||||||
|
工作:
|
||||||
|
|
||||||
|
- 使用 winit 管理窗口、显示器、DPI、全屏和输入生命周期。
|
||||||
|
- 使用 wgpu 创建高性能 Adapter、surface 和呈现管线。
|
||||||
|
- 为平台解码 surface 建立零拷贝或有证据约束的 GPU interop。
|
||||||
|
- egui 仅负责控制界面,不读取视频像素。
|
||||||
|
|
||||||
|
验收:Windows、Linux 和 macOS 均可呈现测试视频;resize、DPI、全屏、设备丢失和
|
||||||
|
显示器切换可恢复;不存在通过 JavaScript、WebView 或 CPU bitmap 的帧路径。
|
||||||
|
|
||||||
|
## 阶段 7:接入 Opus
|
||||||
|
|
||||||
|
状态:未开始。
|
||||||
|
|
||||||
|
工作:
|
||||||
|
|
||||||
|
- Agent 采集系统输出并编码 48 kHz Opus。
|
||||||
|
- 通过独立 RTP 音频 Track 发送并维护统一单调时钟。
|
||||||
|
- Client 使用有界 jitter buffer 解码和播放。
|
||||||
|
- 音频错误与视频、输入生命周期隔离。
|
||||||
|
|
||||||
|
验收:音频连续播放且无无界积压;丢包时使用 Opus PLC;音视频漂移受控;
|
||||||
|
音频设备切换或失败不会阻塞视频和输入。
|
||||||
|
|
||||||
|
## 阶段 8:接入 STUN/TURN
|
||||||
|
|
||||||
|
状态:未开始。
|
||||||
|
|
||||||
|
工作:
|
||||||
|
|
||||||
|
- 支持 host、server-reflexive 和 relay candidates。
|
||||||
|
- 接入 STUN、TURN/UDP、TURN/TCP 和 TURN/TLS 443。
|
||||||
|
- 使用短期、会话绑定的 TURN 凭据。
|
||||||
|
- 支持 ICE restart、网络切换、候选优先级和路径诊断。
|
||||||
|
|
||||||
|
验收:局域网直连、不同 NAT、公网 TURN/UDP 和受限网络 TURN/TLS 均完成测试;
|
||||||
|
客户端显示实际路径、RTT 和 relay;凭据过期、重放和跨会话使用均失败。
|
||||||
|
|
||||||
|
## 阶段 9:更新安装包和 CI
|
||||||
|
|
||||||
|
状态:未开始。
|
||||||
|
|
||||||
|
工作:
|
||||||
|
|
||||||
|
- Windows、Linux 和 macOS 只打包 Rust Agent、Rust Client 和必要资源。
|
||||||
|
- CI 覆盖格式化、Clippy、测试、跨平台编译、SBOM、签名和安装验证。
|
||||||
|
- 增加 WebRTC loopback、RTP 丢包、DataChannel 和包内容测试。
|
||||||
|
- 删除 npm、Tauri、WebView2 和 Go 的发布依赖。
|
||||||
|
|
||||||
|
验收:三平台产物可安装、升级和卸载;包内容白名单通过;CI 不下载或执行 Go、
|
||||||
|
Node、Tauri CLI 或 WebView 构建工具;发布产物具备签名、校验和和 SBOM。
|
||||||
|
|
||||||
|
## 阶段 10:删除旧 Tauri/WebView 与兼容路径
|
||||||
|
|
||||||
|
状态:未开始。
|
||||||
|
|
||||||
|
工作:
|
||||||
|
|
||||||
|
- 删除 client/web、Tauri app shell、React、Vite、npm lockfile 和 WebView 配置。
|
||||||
|
- 删除软件视频回退、zlib framebuffer、MSTSC fallback 和 Compatibility 策略。
|
||||||
|
- 删除旧 helper、旧协议字段、旧测试、旧打包入口和失效文档。
|
||||||
|
- 更新安全模型、用户指南、实现状态和架构图。
|
||||||
|
|
||||||
|
验收:仓库搜索不再出现生产 Tauri/WebView/Hysteria2/Go/Compatibility 入口;
|
||||||
|
所有正式功能通过 Rust 原生 Client 和 Rust Agent 完成;完整 workspace、安装包和
|
||||||
|
端到端测试通过。
|
||||||
|
|
||||||
|
## 完成定义
|
||||||
|
|
||||||
|
只有十个阶段全部达到验收条件,并且旧构建链不再产生发布产物,才能将全 Rust
|
||||||
|
迁移标记为完成。存在源码骨架、未运行的平台代码或仅通过 cargo check 均不算完成。
|
||||||
@@ -1,5 +1,10 @@
|
|||||||
# RemoteDesk 总体架构
|
# RemoteDesk 总体架构
|
||||||
|
|
||||||
|
> 当前生效的目标架构以 [ADR 0001](adr/0001-all-rust-runtime.md) 为准:
|
||||||
|
> Agent、Client、协议和实时通信运行时全部使用 Rust,客户端采用
|
||||||
|
> winit、wgpu、egui,实时传输采用 str0m WebRTC。下文若出现 Tauri、
|
||||||
|
> WebView、Go、Hysteria2、兼容模式或独立 Edge,均视为迁移历史。
|
||||||
|
|
||||||
## 1. 目标和边界
|
## 1. 目标和边界
|
||||||
|
|
||||||
RemoteDesk 解决两个问题:
|
RemoteDesk 解决两个问题:
|
||||||
@@ -105,6 +110,36 @@ Agent 通过出站 TLS WebSocket 连接最近的 Rendezvous Gateway,客户端
|
|||||||
|
|
||||||
Windows RDP 不使用 ICE/TURN。其 CDN 加速需要目标网络中的出站 L4 Edge Connector,在两端 POP 间透明转发 RDP TCP/UDP;RDP TLS/NLA 仍由 IronRDP 与目标 Windows 端到端完成。
|
Windows RDP 不使用 ICE/TURN。其 CDN 加速需要目标网络中的出站 L4 Edge Connector,在两端 POP 间透明转发 RDP TCP/UDP;RDP TLS/NLA 仍由 IronRDP 与目标 Windows 端到端完成。
|
||||||
|
|
||||||
|
### 3.5 Windows Headless Native Endpoint
|
||||||
|
|
||||||
|
Windows 被控端的高性能桌面会话使用 IDD/IddCx 创建 Headless SDR 虚拟显示器。IDD 只负责虚拟显示器、显示模式和 swap-chain 生命周期,不负责编码、网络或业务授权。当前产品约束为 SDR 8-bit;IDD 不作为 HDR10/10-bit 采集源。
|
||||||
|
|
||||||
|
正式组件边界如下:
|
||||||
|
|
||||||
|
Windows Service / Go Backend
|
||||||
|
- session、认证、配置和生命周期
|
||||||
|
- 输入路由(独立高优先级控制面)
|
||||||
|
- Hysteria2 路径和媒体策略
|
||||||
|
|
||||||
|
Windows Capture Worker(Rust/C++ 原生 helper)
|
||||||
|
- IDD frame 或 Desktop Duplication 兼容捕获
|
||||||
|
- BGRA8 GPU texture -> NV12 GPU conversion
|
||||||
|
- NVENC / AMF(VCE/VCN) / Quick Sync
|
||||||
|
- x264 / x265 / SVT-AV1 软件兜底
|
||||||
|
- 独立视频和音频队列
|
||||||
|
- 有界媒体 datagram / 控制流
|
||||||
|
|
||||||
|
IDD Driver
|
||||||
|
- Headless SDR 虚拟显示器和 4K/120 模式
|
||||||
|
|
||||||
|
Windows Headless 主路径优先直接消费 IDD GPU frame;第一阶段允许使用 Desktop Duplication 复制已存在的虚拟 output 作为兼容实现。两种路径都必须在同一 DXGI Adapter 上完成 GPU 处理,原始 BGRA8 不得进入 Go、JSON、Tauri IPC 或网络队列。
|
||||||
|
|
||||||
|
Windows Headless 视频默认支持 H.264/AVC、H.265/HEVC 和 AV1。编码器先尝试同 Adapter 的硬件后端:NVIDIA 使用 NVENC,AMD 使用 AMF/VCE/VCN,Intel 使用 Quick Sync;硬件能力验证失败时按协商结果切换 x264、x265 或 SVT-AV1。软件编码只能作为明确的兼容性降级,并自动降低分辨率、帧率或并发上限。
|
||||||
|
|
||||||
|
Windows Headless 音频独立使用 WASAPI loopback 或配置的虚拟音频 endpoint 采集,编码为 Opus,通过独立音频流发送。音频和视频使用同一会话单调时钟;输入不等待任何音频、视频帧、编码器或呈现确认。
|
||||||
|
|
||||||
|
无硬件 GPU 时启用 compatibility 模式。启动阶段依次探测硬件 DXGI Adapter、IDD/DDA output、WARP/Basic Display Adapter、硬件编码器和软件编码器;如果只有软件路径可用,允许 DDA/IDD 继续采集,但把像素路径标记为 software,并按 CPU 实测上限自动降低分辨率、帧率和并发数。compatibility 模式不得继续宣称 zero-copy、hardware_encode 或多路 4K/120 保证;strict 性能模式则在硬件编码器不可用时拒绝请求。
|
||||||
|
|
||||||
## 4. Linux 会话模式
|
## 4. Linux 会话模式
|
||||||
|
|
||||||
Wayland 不允许普通进程静默截屏和任意注入输入,这是安全边界,不应通过默认 root 运行 Agent 来绕过。
|
Wayland 不允许普通进程静默截屏和任意注入输入,这是安全边界,不应通过默认 root 运行 Agent 来绕过。
|
||||||
|
|||||||
@@ -7,6 +7,12 @@
|
|||||||
- Windows RDP viewer 的安全 Named Pipe 和 Tauri 2 桌面壳已完成源码接线;Linux 远程 H.264 的 RFC 6184 接收、Media Foundation NV12 解码和 D3D11 VideoProcessor 窗口呈现也已接线,但尚未编译、实机验证或通过 ETW/GPUView 证明硬件解码与隐藏复制边界。Tauri 和媒体新依赖按本轮要求尚未解析或编译。
|
- Windows RDP viewer 的安全 Named Pipe 和 Tauri 2 桌面壳已完成源码接线;Linux 远程 H.264 的 RFC 6184 接收、Media Foundation NV12 解码和 D3D11 VideoProcessor 窗口呈现也已接线,但尚未编译、实机验证或通过 ETW/GPUView 证明硬件解码与隐藏复制边界。Tauri 和媒体新依赖按本轮要求尚未解析或编译。
|
||||||
- Linux PipeWire、Portal、DRI3 与硬件编码器必须在对应 Linux 测试机执行 M0 Spike。
|
- Linux PipeWire、Portal、DRI3 与硬件编码器必须在对应 Linux 测试机执行 M0 Spike。
|
||||||
|
|
||||||
|
## 1.1 Windows Headless compatibility 验收
|
||||||
|
|
||||||
|
无 GPU 兼容模式必须覆盖仅 WARP、Microsoft Basic Display Adapter、Intel 核显和虚拟 GPU 环境;验证 DDA 的 DuplicateOutput/AcquireNextFrame 成功与失败、硬件编码器不可用时的软件编码选择、自动降到 1080p30/60 或更低档位、输入延迟不受软件编码阻塞、视频丢帧和关键帧恢复。诊断必须明确显示 software、cpu_upload、degraded 和不支持 4K/120。
|
||||||
|
|
||||||
|
兼容模式只在用户显式选择时启用。没有可复制的 DXGI Output 时应返回结构化 capture unavailable;WARP 或 Basic Display Adapter 不得被报告为硬件 GPU 或硬件编码。
|
||||||
|
|
||||||
## 2. Rust workspace
|
## 2. Rust workspace
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
|
|||||||
@@ -13,6 +13,22 @@ Windows 客户端的 UI、远程视频、RDP 图形和终端均支持 GPU 加速
|
|||||||
|
|
||||||
这里的“零拷贝”特指未压缩视频像素从捕获到编码、或从解码到呈现不经过 CPU map/copy。压缩码流和 RTP 包仍可经过普通内存。跨 Adapter GPU copy 不算零拷贝。
|
这里的“零拷贝”特指未压缩视频像素从捕获到编码、或从解码到呈现不经过 CPU map/copy。压缩码流和 RTP 包仍可经过普通内存。跨 Adapter GPU copy 不算零拷贝。
|
||||||
|
|
||||||
|
### Windows Headless compatibility 模式
|
||||||
|
|
||||||
|
没有独立 GPU 不等于一定没有 DXGI 输出:机器可能存在 Intel 核显、虚拟 GPU、Microsoft Basic Display Adapter 或 WARP。采集端必须以运行时结果区分这些情况,不得因为成功创建 ID3D11Texture2D 就报告硬件加速。
|
||||||
|
|
||||||
|
探测顺序:
|
||||||
|
|
||||||
|
1. 枚举 DXGI Adapter 和 Output。
|
||||||
|
2. 尝试创建 D3D11 device 和 Desktop Duplication。
|
||||||
|
3. 尝试 AcquireNextFrame,并验证连续帧。
|
||||||
|
4. 探测 NVENC、AMF/VCE/VCN、Quick Sync 的实际编码能力。
|
||||||
|
5. 硬件路径不可用时,若用户显式选择 compatibility,则允许 WARP/CPU surface 和 x264、x265、SVT-AV1。
|
||||||
|
|
||||||
|
compatibility 路径必须报告 adapter_type、surface_type、cpu_map_count、cpu_upload_bytes、encoder_backend、software_encode_latency_us、encode_fps 和 degraded。WARP 是软件 D3D 实现,不能报告为硬件 GPU;Microsoft Basic Display Adapter 也不能视为硬件编码器。
|
||||||
|
|
||||||
|
无硬件编码器时,系统自动应用软件上限。默认先尝试 1080p30/60 或实测可承受的更低档位;请求多路 4K/120 时,strict 性能模式返回 HARDWARE_ENCODER_UNAVAILABLE,compatibility 模式降档并明确显示 software/degraded。软件视频队列有界、允许丢帧,且不阻塞输入和音频线程。
|
||||||
|
|
||||||
## 2. 渲染架构
|
## 2. 渲染架构
|
||||||
|
|
||||||
```text
|
```text
|
||||||
|
|||||||
@@ -39,6 +39,10 @@ IronRDP 单屏 resize 源码现已补齐单调 generation、调整后尺寸回
|
|||||||
|
|
||||||
Linux 用户会话现已将 X11 与 Wayland 生命周期收敛到统一 Agent 后端。Wayland 默认编译 `wayland-eis`,通过用户 D-Bus 实际查询 RemoteDesktop/ScreenCast Portal,按 `CreateSession -> SelectDevices -> SelectSources -> Start` 顺序授权,持有 PipeWire remote 与 EIS fd,并以 DMABUF 严格硬件 H.264 管线发送;Portal/EIS/VA 能力不足时返回明确错误,不伪装成 X11 或 CPU 回退。控制服务的 `linux.wayland_desktop` 现在表示 Linux Agent helper 已包含该能力,具体桌面是否可用仍由 Agent status 的 Portal probe 和会话启动结果决定。
|
Linux 用户会话现已将 X11 与 Wayland 生命周期收敛到统一 Agent 后端。Wayland 默认编译 `wayland-eis`,通过用户 D-Bus 实际查询 RemoteDesktop/ScreenCast Portal,按 `CreateSession -> SelectDevices -> SelectSources -> Start` 顺序授权,持有 PipeWire remote 与 EIS fd,并以 DMABUF 严格硬件 H.264 管线发送;Portal/EIS/VA 能力不足时返回明确错误,不伪装成 X11 或 CPU 回退。控制服务的 `linux.wayland_desktop` 现在表示 Linux Agent helper 已包含该能力,具体桌面是否可用仍由 Agent status 的 Portal probe 和会话启动结果决定。
|
||||||
|
|
||||||
|
Windows Headless 方案状态:架构已确定,实现在尚未实现表中跟踪。目标是 IDD/IddCx SDR 8-bit 虚拟显示器、Desktop Duplication 首阶段兼容捕获或后续 IDD GPU frame、NVENC/AMF/VCE/VCN/Quick Sync 硬件优先、x264/x265/SVT-AV1 软件兜底、独立 Opus 音频、输入优先控制流和 Hysteria2 视频 datagram。IDD 不提供 HDR10/10-bit 能力;本方案不承诺 HDR10。
|
||||||
|
|
||||||
|
Windows Headless compatibility 模式允许无硬件 GPU 主机使用可用的 Desktop Duplication/IDD 输出、WARP 或 CPU 编码,但只报告 software/degraded,并根据实测 CPU 能力降低分辨率、帧率和并发上限。该模式不保证多路 4K/120;strict 性能模式在硬件编码器不可用时拒绝请求。当前仍未完成真实无 GPU 主机验收。
|
||||||
|
|
||||||
## 尚未实现
|
## 尚未实现
|
||||||
|
|
||||||
| 路线 | 未实现的主要能力 |
|
| 路线 | 未实现的主要能力 |
|
||||||
@@ -47,6 +51,8 @@ Linux 用户会话现已将 X11 与 Wayland 生命周期收敛到统一 Agent
|
|||||||
| M1 Windows RDP MVP | 原生客户端全部本地显示器布局、自定义显示器子集、D3D11 呈现路径已编译并通过自动化测试;仍缺真实主机验收、硬件解码/零拷贝、断线恢复实机验收、Credential Manager 远端 NLA 验收和正式签名安装包 |
|
| M1 Windows RDP MVP | 原生客户端全部本地显示器布局、自定义显示器子集、D3D11 呈现路径已编译并通过自动化测试;仍缺真实主机验收、硬件解码/零拷贝、断线恢复实机验收、Credential Manager 远端 NLA 验收和正式签名安装包 |
|
||||||
| M2-M4 Linux Agent | X11 `GetImage`/XTEST、Windows 软件桌面窗口、权限路由、有界帧协议、三次有界桌面重连、用户会话进程内 15 秒绑定令牌续接、GStreamer H.264 AU 编码/WebRTC sender、Linux 输出 monitor Opus 编码/有界 IPC、Windows 远端 H.264 RTP 消费/原生呈现、Opus 解码/WASAPI 播放、认证 WSS 直连 WebRTC 信令、X11/Windows 双向 UTF-8 文本剪贴板,以及 Windows Credential Manager 支持的 helper 崩溃后短时续接已完成源码接线并通过相关 Rust 测试;真实 Linux/Windows 主机验收仍未完成。Wayland Portal 已完成能力探测、认证后授权、PipeWire/EIS fd 所有权、严格 DMABUF-to-VA-H.264 会话循环、呈现 ACK、resize 重建、`reis` EIS 输入和绑定 Portal/EIS 的 15 秒会话恢复源码;仍缺 GNOME/KDE/Xorg/纯终端端到端验证和 Wayland 文本剪贴板实现 |
|
| M2-M4 Linux Agent | X11 `GetImage`/XTEST、Windows 软件桌面窗口、权限路由、有界帧协议、三次有界桌面重连、用户会话进程内 15 秒绑定令牌续接、GStreamer H.264 AU 编码/WebRTC sender、Linux 输出 monitor Opus 编码/有界 IPC、Windows 远端 H.264 RTP 消费/原生呈现、Opus 解码/WASAPI 播放、认证 WSS 直连 WebRTC 信令、X11/Windows 双向 UTF-8 文本剪贴板,以及 Windows Credential Manager 支持的 helper 崩溃后短时续接已完成源码接线并通过相关 Rust 测试;真实 Linux/Windows 主机验收仍未完成。Wayland Portal 已完成能力探测、认证后授权、PipeWire/EIS fd 所有权、严格 DMABUF-to-VA-H.264 会话循环、呈现 ACK、resize 重建、`reis` EIS 输入和绑定 Portal/EIS 的 15 秒会话恢复源码;仍缺 GNOME/KDE/Xorg/纯终端端到端验证和 Wayland 文本剪贴板实现 |
|
||||||
| M5 体验和性能 | Linux X11 兼容帧链路已完成采集/编码/解码/呈现耗时源码遥测,并已接入基于呈现 ACK 的有界帧率和 zlib level 1..6 压缩策略自适应;IronRDP 已完成网络图形 PDU 本地处理与像素转换耗时源码遥测;相关 Rust/Web 自动化已通过。正式更新源发布自动化和 Authenticode 签名/验签已完成源码接线但尚未执行生产发布;仍缺 RDP 服务端编码遥测、H.264/Opus 码率与编解码策略自适应、HEVC/AV1、通用会话崩溃恢复,以及真实证书/时间戳/下载/原位升级验收;文件上传/下载断点续传已实现 |
|
| M5 体验和性能 | Linux X11 兼容帧链路已完成采集/编码/解码/呈现耗时源码遥测,并已接入基于呈现 ACK 的有界帧率和 zlib level 1..6 压缩策略自适应;IronRDP 已完成网络图形 PDU 本地处理与像素转换耗时源码遥测;相关 Rust/Web 自动化已通过。正式更新源发布自动化和 Authenticode 签名/验签已完成源码接线但尚未执行生产发布;仍缺 RDP 服务端编码遥测、H.264/Opus 码率与编解码策略自适应、HEVC/AV1、通用会话崩溃恢复,以及真实证书/时间戳/下载/原位升级验收;文件上传/下载断点续传已实现 |
|
||||||
|
| Windows Headless Native | 部分 | IDD/IddCx 虚拟显示器、IDD GPU frame、DDA 到 GPU 编码器 zero-copy、NVENC/AMF/VCE/VCN/Quick Sync、HEVC/AV1 和认证 session lifecycle 仍未完成;独立 Opus/WASAPI 源码链已接入 compatibility runtime,但统一 QPC 精确同步和实机验收未完成;SDR 8-bit H.264 compatibility 已接通到 RDV1 GUI;HDR10 明确不在目标范围内 |
|
||||||
|
| Windows Headless compatibility | 部分 | DDA staging/readback、CPU BGRA->NV12、Media Foundation H.264、Annex-B、encoded mmap ring、Go RDV1/RDA1/Hysteria2、Rust RDV1 重组、IDR 恢复、GUI MFT/D3D11 呈现,以及独立 WASAPI loopback/Opus/audio ring/jitter/PLC/FEC/WASAPI playback 源码已接入;无 GPU、4K/120、多路压力、统一 QPC 精确音画同步和真实 Windows 端到端验收仍未完成 |
|
||||||
| M6 CDN 边缘中继 | 已有单节点短 TTL Presence、Linux Agent 心跳、Windows Client 签名 Session Intent、Agent 本地授权、角色票据、远程首次配对、已知设备 outbound-only rendezvous、Linux 终端/文件 TLS-over-relay transport、设备签名的有界双向 SDP/ICE mailbox、授权响应内的 session-bound TURN 短期凭据、Agent/Client 共用的真实 PeerConnection/H.264/RTX/Opus/DataChannel 传输内核、Linux X11 捕获/GStreamer H.264 与 Opus track 发送,以及 Windows H.264 RTP 解码/呈现和 Opus/WASAPI 播放源码;Edge、Agent runtime、Linux terminal 和 native video 的自动化测试已通过。仍缺实际 coturn POP 部署与 UDP/TLS 验收、真实双 POP 骨干指标/亲和/故障切换、RDP TCP/UDP L4 Edge Connector、静态发布 CDN 与跨网络端到端验证 |
|
| M6 CDN 边缘中继 | 已有单节点短 TTL Presence、Linux Agent 心跳、Windows Client 签名 Session Intent、Agent 本地授权、角色票据、远程首次配对、已知设备 outbound-only rendezvous、Linux 终端/文件 TLS-over-relay transport、设备签名的有界双向 SDP/ICE mailbox、授权响应内的 session-bound TURN 短期凭据、Agent/Client 共用的真实 PeerConnection/H.264/RTX/Opus/DataChannel 传输内核、Linux X11 捕获/GStreamer H.264 与 Opus track 发送,以及 Windows H.264 RTP 解码/呈现和 Opus/WASAPI 播放源码;Edge、Agent runtime、Linux terminal 和 native video 的自动化测试已通过。仍缺实际 coturn POP 部署与 UDP/TLS 验收、真实双 POP 骨干指标/亲和/故障切换、RDP TCP/UDP L4 Edge Connector、静态发布 CDN 与跨网络端到端验证 |
|
||||||
|
|
||||||
## 运行时能力接口
|
## 运行时能力接口
|
||||||
@@ -176,3 +182,26 @@ Linux 用户会话现已将 X11 与 Wayland 生命周期收敛到统一 Agent
|
|||||||
- Windows 原生 RDP 自定义显示器子集现已完成 secure-pipe JSON、控制端请求、Web UI 选择、viewer 参数和 Win32 monitor layout 筛选:索引必须唯一、范围有效且包含主显示器,筛选后重新计算组合窗口边界;viewer 单元测试和 Web build 已通过。仍需真实多显示器 Windows RDP 服务端验收。
|
- Windows 原生 RDP 自定义显示器子集现已完成 secure-pipe JSON、控制端请求、Web UI 选择、viewer 参数和 Win32 monitor layout 筛选:索引必须唯一、范围有效且包含主显示器,筛选后重新计算组合窗口边界;viewer 单元测试和 Web build 已通过。仍需真实多显示器 Windows RDP 服务端验收。
|
||||||
- 自定义显示器子集接入后的控制端 MSI 已重新生成,最新 SHA-256 为 `66b34ea4a0292e6a4ddb5b8feab56ead19d6660135785f1893405e63cec4c7f0`(未签名开发包)。
|
- 自定义显示器子集接入后的控制端 MSI 已重新生成,最新 SHA-256 为 `66b34ea4a0292e6a4ddb5b8feab56ead19d6660135785f1893405e63cec4c7f0`(未签名开发包)。
|
||||||
- 上述结果证明可编译和自动化测试覆盖,不等同于 Windows RDP 服务端、GNOME/KDE/Xorg、PipeWire/VA 驱动、coturn 跨网络或硬件解码的实机端到端验收;这些项目仍保持原有“部分”状态。
|
- 上述结果证明可编译和自动化测试覆盖,不等同于 Windows RDP 服务端、GNOME/KDE/Xorg、PipeWire/VA 驱动、coturn 跨网络或硬件解码的实机端到端验收;这些项目仍保持原有“部分”状态。
|
||||||
|
|
||||||
|
## Windows Headless 媒体实现状态
|
||||||
|
|
||||||
|
本轮已完成以下可运行基础:
|
||||||
|
|
||||||
|
- protocol/src/windows_media.rs 提供 SDR 8-bit/NV12、H.264/HEVC/AV1、Opus、硬件优先和 compatibility 降级的校验模型。
|
||||||
|
- transport/hysteria2-agent 已接入 RDV1 视频分片大小校验和 Hysteria2 UDP datagram 转发;视频重组超时丢弃,不重传过期视频。Go client 还提供可选的 encoded-ring consumer,将 Rust 的 H.264 Annex-B access unit 复制、分片并发送到同一认证 Hysteria2 UDP session。
|
||||||
|
- agent/windows-agent/src/media_ring.rs 提供跨进程 memory-mapped SPSC ring 和 session-bound encoded-only descriptor,承载已编码码流,不承载原始 BGRA/NV12。满队列时非阻塞返回,调用方可丢弃视频包;Windows H.264 compatibility worker 已写入该 ring,Go Hysteria2 client 可通过 `--encoded-ring` 打开消费者并转成 RDV1 datagram。Windows 创建 ring 时使用受保护 DACL,只授权当前用户 SID,并允许同一会话消费者映射及关闭清理。
|
||||||
|
- remotedesk-windows-agent pipe 已使用当前用户 SID ACL、拒绝远程 Pipe 客户端和首实例保护;默认要求 `REMOTEDESK_AGENT_PIPE_TOKEN` 提供随机 32 字节 base64url bootstrap,每个连接以随机 nonce 执行 HMAC-SHA256 challenge,认证总时限 10 秒,认证后才把 `authenticated: true` hello 和 ring descriptor 交给 Go bridge。`--allow-unauthenticated` 与 loopback TCP 自动编排只保留本地开发兼容。
|
||||||
|
- Windows agent 的桌面请求支持 capture_mode: strict_hardware|compatibility 和 video_codec: h264|hevc|av1。Windows 兼容捕获已实现 Desktop Duplication 的 D3D11 staging/readback 路径,并在 DDA 不可用时回退 GDI;响应会报告实际后端、cpu_readback、cpu_map_count 和 degraded。H.264 compatibility worker 已完成 DDA BGRA -> CPU NV12 -> Media Foundation MFT -> Annex-B AU -> encoded ring 的短样本运行链;`hardware_encoder_verified` 只表示 MFT 短样本成功,`hardware_pipeline_verified` 仍为 false,因此 strict 请求仍返回结构化不可用错误。
|
||||||
|
- protocol 的 RDV1 decoder/reassembler 严格校验 58 字节 header、未知 flags、分片位置、元数据一致性、重复分片和 16 MiB 上限;首帧、序号中断、分片过期或 discontinuity 后丢弃 P-frame,直到收到 IDR。viewer 以一槽待处理门控接收 loopback RDV1,积压时丢视频而不阻塞输入;H.264 Annex-B 进入 D3D11-aware Media Foundation decoder,只接受同 device NV12 DXGI surface,再由 D3D11 VideoProcessor 呈现。
|
||||||
|
- protocol 与 Go transport 的 RDA1 decoder 严格校验 40 字节 header、version、reserved byte、已知 flags、stream/generation/sequence、10/20/40/60 ms duration 和 4 KiB Opus payload 上限;音频 datagram 不与视频帧重组,也不等待视频。Windows Agent 使用独立线程执行 WASAPI loopback、48 kHz 双声道归一化、20 ms Opus 编码并写入独立 audio mmap ring;Go bridge 独立轮询并发送 RDA1。audio ring 或 callback 积压时显式标记 discontinuity,不阻塞视频或输入。
|
||||||
|
- windows-agent-viewer 在 --media-udp 模式严格区分 RDV1/RDA1,使用容量 64 的非阻塞接收队列、三包启动缓冲、最多 64 包的有界 jitter map、Opus in-band FEC/PLC、按实际输出设备采样率重采样和 CPAL/WASAPI playback。音频设备或解码失败只关闭音频;音频播放时钟建立后,viewer 丢弃明显落后的旧视频帧,不等待未来视频帧。播放端现在以连续按序音频 PTS 估计源时钟偏差,并结合 60 ms 目标 PCM 占用以原子 ppm(±0.5%)调节重采样步长;stream/generation/discontinuity 重置时清空旧 PCM、清零校正并重新预缓冲。
|
||||||
|
- viewer 最多每 250 ms 合并一次 `request_keyframe`,经独立可靠输入连接按 session ID 路由到 Agent 的一槽请求队列;编码 worker 使用 `ICodecAPI/CODECAPI_AVEncVideoForceKeyFrame` 请求 IDR。Go Hysteria2 client 可通过 `--windows-agent-pipe` 完成 challenge 后自动打开编码会话并严格校验 ring descriptor,也保留手工 `--encoded-ring` 与开发用 loopback `--windows-agent-control` 模式。
|
||||||
|
|
||||||
|
以下仍未完成,不能作为已支持能力发布:
|
||||||
|
|
||||||
|
- IDD/IddCx 虚拟显示器驱动、Desktop Duplication 到 GPU 编码器的 zero-copy capture、BGRA 到 NV12 的 D3D11 GPU conversion。
|
||||||
|
- NVENC、AMF/VCE/VCN、Quick Sync 真实编码器探测与编码 worker,以及 x264/x265/SVT-AV1 真实 fallback。
|
||||||
|
- 音视频生产端统一 session-start QPC/PTS 映射已接入 Windows Agent compatibility 链;viewer 接收端已完成连续音频 PTS 漂移估计、60 ms PCM 占用反馈和 ±0.5% 有界动态重采样。仍需 Windows 实机长时间稳定性、默认设备切换/拔出和跨设备音画端到端验收,不能据此宣称已完成生产级硬件媒体链。
|
||||||
|
- WASAPI 默认设备切换/拔出、无输出设备、callback error、长时间稳定性、内置 libopus 产物体积/许可证和真实 Windows 音画端到端验收。
|
||||||
|
- 认证 Named Pipe 和 ring 当前按 Windows 当前用户 SID 隔离;生产安装器/控制服务仍需负责生成每次 Agent 生命周期唯一的 bootstrap、仅注入目标 Agent 与 Go bridge、启动后清除父进程环境,并把两进程纳入统一 Job/service 生命周期。viewer 的输入事件使用独立持久连接和有界优先队列,Win32 全局键盘 hook 复用该队列。
|
||||||
|
- Windows 实机下的多路 4K/120 性能、实际丢包/IDR 恢复和 NVENC/AMF/QSV 多路 session 压力测试。
|
||||||
|
|||||||
@@ -24,6 +24,26 @@ RDP 的网络恢复能力由 IronRDP 协议实现决定。M0 必须验证:
|
|||||||
|
|
||||||
- RDP UDP multitransport。
|
- RDP UDP multitransport。
|
||||||
- 图形管线在丢包和重连后的恢复行为。
|
- 图形管线在丢包和重连后的恢复行为。
|
||||||
|
|
||||||
|
### 2.3 Windows Headless Hysteria2
|
||||||
|
|
||||||
|
Windows Headless 使用 Hysteria2 作为网络承载,但应用层必须保持输入、音频和视频分流:
|
||||||
|
|
||||||
|
control stream:输入、心跳、配置和 KeyframeRequest,可靠且最高优先级。
|
||||||
|
|
||||||
|
audio path:Opus,独立 jitter buffer,允许 PLC/FEC。
|
||||||
|
|
||||||
|
video datagram:H.264/HEVC/AV1,允许丢包和丢帧。
|
||||||
|
|
||||||
|
file stream:文件和剪贴板,可靠、最低优先级、限速。
|
||||||
|
|
||||||
|
视频不能放入会因丢包而阻塞后续数据的可靠流。Hysteria2 的 datagram 能力必须由具体 adapter 验证;不能仅因为底层使用 UDP 就假设应用已经具备不可靠语义。若某部署只能提供可靠 QUIC stream,Windows Headless 视频通道不得宣称实时丢包模式。
|
||||||
|
|
||||||
|
视频分片带有 stream_id、generation、frame_id、fragment_id、fragment_count、PTS 和关键帧标记。缺少任意分片时丢弃整个视频帧;重组超时或帧已过期时立即丢弃;不重传过期视频包。关键帧或参考链损坏后通过控制流请求 IDR。
|
||||||
|
|
||||||
|
无硬件 GPU 时 compatibility 模式会进一步受 CPU 编码预算限制。编码器报告 software/degraded 后,质量控制器必须优先降低视频帧率和分辨率,再降低码率;视频队列不得因 CPU 编码变慢而无限增长。输入控制和 Opus 音频保持独立调度,软件编码过载时可以丢弃过期视频帧,但不能阻塞输入或音频。
|
||||||
|
|
||||||
|
Hysteria2 的带宽和队列不能让视频占满控制面。输入预留最高优先级,音频次之,视频使用有界队列,文件只使用剩余带宽。必须监控路径 RTT、抖动、队列延迟、包丢失、输入延迟和音画偏差,而不是只观察总吞吐。
|
||||||
- 动态调整视觉效果、色深或编码模式的能力。
|
- 动态调整视觉效果、色深或编码模式的能力。
|
||||||
- UDP 不可用时 TCP 模式的延迟和队头阻塞表现。
|
- UDP 不可用时 TCP 模式的延迟和队头阻塞表现。
|
||||||
|
|
||||||
|
|||||||
@@ -217,6 +217,24 @@ DisplaySelection:
|
|||||||
|
|
||||||
Windows RDP helper 把验证后的本地布局映射为 Display Control Dynamic Monitor Layout。Linux Agent 把同一选择模型映射到 Portal/PipeWire 或 XRandR 捕获源,但不会修改 Linux 实体显示模式。
|
Windows RDP helper 把验证后的本地布局映射为 Display Control Dynamic Monitor Layout。Linux Agent 把同一选择模型映射到 Portal/PipeWire 或 XRandR 捕获源,但不会修改 Linux 实体显示模式。
|
||||||
|
|
||||||
|
### 8.5 Windows Headless Compatibility Contract
|
||||||
|
|
||||||
|
Windows Headless compatibility 模式允许无硬件 GPU 主机使用可用的 Desktop Duplication 或 IDD 输出、WARP 或 CPU 编码,但必须显式报告 software 和 degraded。能力结果至少包含 capture_backend、adapter_type、surface_type、encoder_backend、hardware_encoder_verified、hardware_pipeline_verified、hardware_path_verified、cpu_map_count、max_width、max_height、max_fps 和 concurrent_sessions。`hardware_encoder_verified` 只代表真实短样本编码成功;DDA staging/readback 仍有 CPU map,因此该路径的 `hardware_pipeline_verified` 和 `hardware_path_verified` 必须为 false。若请求硬性要求 4K/120 或硬件编码,而本地只有 software 路径,Agent 返回 HARDWARE_ENCODER_UNAVAILABLE,不得静默降级。
|
||||||
|
|
||||||
|
Windows Headless 媒体契约固定为 SDR 8-bit:video codec 为 H.264/AVC、H.265/HEVC 或 AV1,输入格式为 NV12,audio codec 为 Opus。IDD 不提供本项目的 HDR10/10-bit 能力,不得出现 P010、Main10 或 HDR metadata。
|
||||||
|
|
||||||
|
Windows Headless 视频使用 Hysteria2 不可靠 datagram,允许丢失。缺少一个分片时丢弃整个帧;过期帧不重传;首帧非 IDR、sequence 中断、discontinuity、解码失败或关键帧损坏时,经独立可靠控制流发送带 session ID 的 `request_keyframe`。Client 以 250 ms 最小间隔合并请求,Agent 以容量 1 的队列合并请求并通过 `CODECAPI_AVEncVideoForceKeyFrame` 请求编码器输出 IDR。收到新 codec config 和 IDR 后才能恢复普通帧。
|
||||||
|
|
||||||
|
本地 encoded-ring descriptor 只能经 `\\.\pipe\RemoteDesk\...` 命名空间交付。Pipe DACL 只授权当前用户 SID,拒绝远程客户端并启用首实例保护;双方从进程环境取得同一个随机 32 字节 base64url bootstrap,Agent 为每个连接发送 32 字节随机 nonce,Go bridge 返回 `HMAC-SHA256(key, "RemoteDesk Windows Agent pipe auth v1\\0" || nonce)`。bootstrap 不进入命令行或 JSON,双方读取后从各自环境删除;认证和 hello 共用有界握手时限。仅 `authenticated: true` 的连接可作为生产 descriptor 来源,ring 路径、session ID、generation、owner PID 和 geometry 必须全部校验后才能映射。
|
||||||
|
|
||||||
|
RDA1 音频 datagram 使用固定 40 字节 little-endian header:magic/version/reserved/header_bytes、stream ID、generation、sequence、PTS、duration_ms 和 flags,后接一个不超过 4 KiB 的独立 Opus packet。reserved 必须为 0,当前仅定义 discontinuity flag,duration 仅允许 10/20/40/60 ms;未知 flags、零 identity、空 payload 和超限 packet 必须在进入 jitter buffer 前拒绝。音频 packet 不依赖 RDV1 frame ID,不因视频分片缺失而等待或丢弃。
|
||||||
|
|
||||||
|
输入控制使用独立可靠高优先级流,带 input_seq 和 layout_generation。输入发送不等待视频帧、编码完成、媒体 ACK 或音频播放。文件和剪贴板使用独立可靠低优先级流,不能阻塞输入、音频或视频。
|
||||||
|
|
||||||
|
音视频共享 session clock/QPC 映射和 generation,但独立采集、编码、队列和发送。客户端以音频播放时钟同步视频,视频落后时丢弃旧帧,不为追求完整帧率而增加播放延迟。
|
||||||
|
|
||||||
|
Hysteria2 音频包使用独立的 RDA1 datagram,不进入视频分片重组器。固定 40 字节头包含 stream、generation、sequence、100ns media PTS、duration(10/20/40/60 ms)和 flags,Opus payload 上限为 4 KiB。音频包独立丢弃或播放,视频缺片不能阻塞音频,音频丢包也不能阻塞视频。Windows compatibility runtime 已使用独立 worker 完成 WASAPI loopback、48 kHz 双声道 20 ms Opus、独立 mmap ring、Go RDA1 发送,以及 viewer 有界 jitter/FEC/PLC/WASAPI playback 的源码接线;统一 session-start QPC、长时间漂移校正和真实设备验收仍是发布前条件。
|
||||||
|
|
||||||
## 9. WebRTC 信令
|
## 9. WebRTC 信令
|
||||||
|
|
||||||
- Offer/Answer 和 ICE candidate 放入 Protobuf 信令消息。
|
- Offer/Answer 和 ICE candidate 放入 Protobuf 信令消息。
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# RemoteDesk 实施路线
|
# RemoteDesk 实施路线
|
||||||
|
|
||||||
|
> 全 Rust 架构迁移的当前执行顺序和验收标准见
|
||||||
|
> [全 Rust 架构迁移计划](all-rust-migration.md)。该计划依据
|
||||||
|
> [ADR 0001](adr/0001-all-rust-runtime.md),优先于下文旧版里程碑中的
|
||||||
|
> Tauri、WebView、Go、Hysteria2 和兼容模式描述。
|
||||||
|
|
||||||
路线按风险优先,而不是按界面完成度排序。最先验证 Agent 与客户端原生 helper 的双端零拷贝、IronRDP、Wayland Portal、Xorg DRI3、PTY 和差网络自适应等核心技术。
|
路线按风险优先,而不是按界面完成度排序。最先验证 Agent 与客户端原生 helper 的双端零拷贝、IronRDP、Wayland Portal、Xorg DRI3、PTY 和差网络自适应等核心技术。
|
||||||
|
|
||||||
## M0:技术验证
|
## M0:技术验证
|
||||||
|
|||||||
@@ -1,5 +1,11 @@
|
|||||||
# RemoteDesk 技术栈规划
|
# RemoteDesk 技术栈规划
|
||||||
|
|
||||||
|
> 当前决策:所有发布运行时使用 Rust;Agent 是服务端;原生客户端使用
|
||||||
|
> winit、wgpu 和 egui;实时传输使用 str0m WebRTC。Go、Hysteria2、
|
||||||
|
> Tauri 和 WebView 不属于目标组件。详见
|
||||||
|
> [ADR 0001](adr/0001-all-rust-runtime.md)。下文保留的旧技术栈内容仅用于
|
||||||
|
> 迁移对照,后续随代码迁移删除。
|
||||||
|
|
||||||
## 1. 混合架构
|
## 1. 混合架构
|
||||||
|
|
||||||
RemoteDesk Windows 客户端采用 Web UI 与 Rust 原生会话进程混合方案。JavaScript 负责管理界面和终端;Linux 桌面 WebRTC、Windows RDP 和 GPU 呈现均由 Rust 原生 helper 承载,原始视频像素不进入 WebView。
|
RemoteDesk Windows 客户端采用 Web UI 与 Rust 原生会话进程混合方案。JavaScript 负责管理界面和终端;Linux 桌面 WebRTC、Windows RDP 和 GPU 呈现均由 Rust 原生 helper 承载,原始视频像素不进入 WebView。
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ mod edge_signaling;
|
|||||||
mod features;
|
mod features;
|
||||||
mod path_validation;
|
mod path_validation;
|
||||||
mod transcript;
|
mod transcript;
|
||||||
|
mod windows_media;
|
||||||
|
|
||||||
pub use displays::{
|
pub use displays::{
|
||||||
DISPLAY_ID_LENGTH, DisplayAxis, DisplayDescriptor, DisplayId, DisplayLayout,
|
DISPLAY_ID_LENGTH, DisplayAxis, DisplayDescriptor, DisplayId, DisplayLayout,
|
||||||
@@ -32,3 +33,13 @@ pub use path_validation::{
|
|||||||
pub use transcript::{
|
pub use transcript::{
|
||||||
NegotiationTranscriptV1, TRANSCRIPT_DOMAIN_V1, TranscriptBuildError, TranscriptInputV1,
|
NegotiationTranscriptV1, TRANSCRIPT_DOMAIN_V1, TranscriptBuildError, TranscriptInputV1,
|
||||||
};
|
};
|
||||||
|
pub use windows_media::{
|
||||||
|
AUDIO_DATAGRAM_HEADER_BYTES, AUDIO_DATAGRAM_MAX_BYTES, AudioPacket, AudioPacketFlags,
|
||||||
|
AudioStreamConfig, EncoderBackend, EncoderCapability, MAX_AUDIO_PACKET_BYTES,
|
||||||
|
MAX_VIDEO_FRAGMENT_PAYLOAD, MAX_VIDEO_FRAME_BYTES, MAX_VIDEO_FRAME_FRAGMENTS, MediaPacketFlags,
|
||||||
|
SelectedEncoder, VIDEO_DATAGRAM_HEADER_BYTES, VIDEO_DATAGRAM_MAX_BYTES, VideoAccessUnit,
|
||||||
|
VideoCodec, VideoModePolicy, VideoPacket, VideoPixelFormat, VideoProfile, VideoReassembler,
|
||||||
|
VideoRequest, VideoStreamConfig, WindowsCaptureBackend, WindowsMediaConfigError,
|
||||||
|
WindowsMediaDatagramError, WindowsMediaPacketError, WindowsMediaSelectionError,
|
||||||
|
decode_audio_datagram, decode_video_datagram, select_encoder,
|
||||||
|
};
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ pub type VerifierChallenge = [u8; VERIFIER_CHALLENGE_LENGTH];
|
|||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
pub enum ZeroCopyPolicy {
|
pub enum ZeroCopyPolicy {
|
||||||
RequiredEndToEnd,
|
RequiredEndToEnd,
|
||||||
Compatibility,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
@@ -316,9 +315,6 @@ impl StrictPathCoordinator {
|
|||||||
cross_adapter_copy_count: u64,
|
cross_adapter_copy_count: u64,
|
||||||
verification_source: VerificationSource,
|
verification_source: VerificationSource,
|
||||||
) -> Result<(), PathReportError> {
|
) -> Result<(), PathReportError> {
|
||||||
if self.policy == ZeroCopyPolicy::Compatibility {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
if !status.satisfies_strict_zero_copy() {
|
if !status.satisfies_strict_zero_copy() {
|
||||||
return Err(PathReportError::PathDoesNotSatisfyStrictPolicy(status));
|
return Err(PathReportError::PathDoesNotSatisfyStrictPolicy(status));
|
||||||
}
|
}
|
||||||
@@ -530,10 +526,10 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn compatibility_mode_accepts_software_paths_but_still_needs_both_reports() {
|
fn strict_mode_rejects_software_paths() {
|
||||||
let expected = context(1, 2, 3);
|
let expected = context(1, 2, 3);
|
||||||
let mut coordinator =
|
let mut coordinator =
|
||||||
StrictPathCoordinator::new(ZeroCopyPolicy::Compatibility, expected.clone());
|
StrictPathCoordinator::new(ZeroCopyPolicy::RequiredEndToEnd, expected.clone());
|
||||||
let mut agent = agent_report(&expected);
|
let mut agent = agent_report(&expected);
|
||||||
agent.status = MemoryPathStatus::Software;
|
agent.status = MemoryPathStatus::Software;
|
||||||
agent.surface_type = MemorySurfaceType::Cpu;
|
agent.surface_type = MemorySurfaceType::Cpu;
|
||||||
@@ -541,15 +537,9 @@ mod tests {
|
|||||||
agent.verification_source = VerificationSource::Unverified;
|
agent.verification_source = VerificationSource::Unverified;
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
coordinator.accept_agent_report(agent),
|
coordinator.accept_agent_report(agent),
|
||||||
Ok(StrictPathState::Negotiating)
|
Err(PathReportError::PathDoesNotSatisfyStrictPolicy(
|
||||||
);
|
MemoryPathStatus::Software
|
||||||
|
))
|
||||||
let mut client = client_report(&expected);
|
|
||||||
client.status = MemoryPathStatus::CpuUpload;
|
|
||||||
client.surface_type = MemorySurfaceType::Cpu;
|
|
||||||
assert_eq!(
|
|
||||||
coordinator.accept_client_report(client),
|
|
||||||
Ok(StrictPathState::Connected)
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -78,7 +78,6 @@ impl NegotiationTranscriptV1 {
|
|||||||
7,
|
7,
|
||||||
&[match input.zero_copy_policy {
|
&[match input.zero_copy_policy {
|
||||||
ZeroCopyPolicy::RequiredEndToEnd => 1,
|
ZeroCopyPolicy::RequiredEndToEnd => 1,
|
||||||
ZeroCopyPolicy::Compatibility => 2,
|
|
||||||
}],
|
}],
|
||||||
);
|
);
|
||||||
append_field(&mut output, 8, &input.client_capabilities_digest);
|
append_field(&mut output, 8, &input.client_capabilities_digest);
|
||||||
@@ -145,15 +144,6 @@ mod tests {
|
|||||||
.unwrap()
|
.unwrap()
|
||||||
.stable_hash_input();
|
.stable_hash_input();
|
||||||
|
|
||||||
let mut changed = transcript_input();
|
|
||||||
changed.zero_copy_policy = ZeroCopyPolicy::Compatibility;
|
|
||||||
assert_ne!(
|
|
||||||
original,
|
|
||||||
NegotiationTranscriptV1::new(changed, RequiredFeatures::ALL_KNOWN)
|
|
||||||
.unwrap()
|
|
||||||
.stable_hash_input()
|
|
||||||
);
|
|
||||||
|
|
||||||
let mut changed = transcript_input();
|
let mut changed = transcript_input();
|
||||||
changed.selected_config_hash = [99; 32];
|
changed.selected_config_hash = [99; 32];
|
||||||
assert_ne!(
|
assert_ne!(
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -30,3 +30,93 @@ UDP session:
|
|||||||
|
|
||||||
`--insecure` is intentionally opt-in and should only be used for a local test
|
`--insecure` is intentionally opt-in and should only be used for a local test
|
||||||
certificate. No TCP listener, TCP proxy, or SOCKS5 endpoint is created.
|
certificate. No TCP listener, TCP proxy, or SOCKS5 endpoint is created.
|
||||||
|
|
||||||
|
## Rust encoded ring bridge
|
||||||
|
|
||||||
|
The Windows Agent compatibility encoder can publish H.264 Annex-B access units
|
||||||
|
to its session-bound memory-mapped ring. The Hysteria2 client can consume that
|
||||||
|
ring directly and send RDV1 video datagrams on the same authenticated UDP
|
||||||
|
session:
|
||||||
|
|
||||||
|
```text
|
||||||
|
remotedesk-hysteria2-agent --mode client \
|
||||||
|
--listen 127.0.0.1:39503 \
|
||||||
|
--server relay.example:39502 --password 'change-me' \
|
||||||
|
--server-name relay.example \
|
||||||
|
--agent-udp 127.0.0.1:39502 \
|
||||||
|
--encoded-ring C:\Users\user\AppData\Local\Temp\remotedesk-session-...ring \
|
||||||
|
--ring-stream-id 1 --ring-generation 7
|
||||||
|
```
|
||||||
|
|
||||||
|
`--encoded-ring` is optional and is intended for the authenticated session
|
||||||
|
orchestrator, which obtains the path, stream ID, and generation from the Rust
|
||||||
|
control response. The Go consumer validates the ring header and geometry,
|
||||||
|
copies each encoded access unit before releasing the slot, fragments it into
|
||||||
|
RDV1, and drops video on cancellation or transport failure. It never maps or
|
||||||
|
transmits raw BGRA/NV12 surfaces.
|
||||||
|
|
||||||
|
For production local orchestration, start the Agent and bridge with the same
|
||||||
|
random 32-byte base64url bootstrap. The Agent removes its inherited copy from
|
||||||
|
the environment after startup, restricts the pipe and ring DACL to the current
|
||||||
|
Windows user, rejects remote pipe clients, and authenticates each connection
|
||||||
|
with a nonce-bound HMAC-SHA256 challenge:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
$token = New-Object byte[] 32
|
||||||
|
[Security.Cryptography.RandomNumberGenerator]::Fill($token)
|
||||||
|
$env:REMOTEDESK_AGENT_PIPE_TOKEN = [Convert]::ToBase64String($token).TrimEnd('=').Replace('+','-').Replace('/','_')
|
||||||
|
Start-Process -WindowStyle Hidden remotedesk-windows-agent -ArgumentList 'pipe'
|
||||||
|
|
||||||
|
remotedesk-hysteria2-agent --mode client `
|
||||||
|
--server relay.example:39502 --password 'change-me' `
|
||||||
|
--server-name relay.example --agent-udp 127.0.0.1:39502 `
|
||||||
|
--windows-agent-pipe '\\.\pipe\RemoteDesk\windows-agent' `
|
||||||
|
--session-id session-1 --ring-fps 120 --ring-stream-id 1
|
||||||
|
Remove-Item Env:REMOTEDESK_AGENT_PIPE_TOKEN
|
||||||
|
```
|
||||||
|
|
||||||
|
`--windows-agent-control` remains available only for a same-machine development
|
||||||
|
bridge. Its address must be a literal loopback IP:
|
||||||
|
|
||||||
|
```text
|
||||||
|
remotedesk-hysteria2-agent --mode client \
|
||||||
|
--server relay.example:39502 --password 'change-me' \
|
||||||
|
--server-name relay.example --agent-udp 127.0.0.1:39502 \
|
||||||
|
--windows-agent-control 127.0.0.1:39501 \
|
||||||
|
--session-id session-1 --ring-fps 120 --ring-stream-id 1
|
||||||
|
```
|
||||||
|
|
||||||
|
The receiver-side Windows viewer binds only loopback UDP, pins the first valid
|
||||||
|
RDV1 source, uses bounded reassembly and a one-frame UI mailbox, and presents
|
||||||
|
H.264 through Media Foundation/D3D11:
|
||||||
|
|
||||||
|
```text
|
||||||
|
remotedesk-windows-agent-viewer --target 10.0.0.20:39501 \
|
||||||
|
--session-id session-1 --fps 120 --media-udp 127.0.0.1:39502
|
||||||
|
```
|
||||||
|
|
||||||
|
RDWF remains the default when `--media-udp` is absent. The authenticated Named
|
||||||
|
Pipe is the production descriptor path; loopback TCP and manual `--encoded-ring`
|
||||||
|
are diagnostic/development paths and do not prove descriptor authenticity.
|
||||||
|
|
||||||
|
## RemoteDesk media datagrams
|
||||||
|
|
||||||
|
Packets beginning with RDV1 are RemoteDesk H.264/HEVC/AV1 video fragments.
|
||||||
|
The relay enforces the complete datagram limit (58-byte header plus at most
|
||||||
|
1200 bytes of payload) and otherwise forwards the bytes without decoding them.
|
||||||
|
Audio and control packets use their own framing and remain opaque to this
|
||||||
|
adapter. A missing video fragment is not retransmitted by this helper; the
|
||||||
|
receiver drops the incomplete access unit after its bounded reassembly timeout.
|
||||||
|
|
||||||
|
The transport is an unreliable media path. Input and control traffic must use a
|
||||||
|
separate reliable path in the Agent session protocol; this UDP adapter does not
|
||||||
|
provide ordering or delivery guarantees for those messages.
|
||||||
|
|
||||||
|
RemoteDesk Opus packets use RDA1: a 40-byte header followed by one bounded
|
||||||
|
Opus packet (maximum 4 KiB). Valid durations are 10/20/40/60 ms. The adapter
|
||||||
|
validates the framing and forwards the packet without reordering or waiting for
|
||||||
|
video. With an authenticated Windows Agent descriptor it independently drains
|
||||||
|
the Opus mmap ring and sends RDA1 datagrams. The Windows viewer uses a bounded
|
||||||
|
jitter queue plus Opus FEC/PLC and WASAPI playback; audio failure does not stop
|
||||||
|
video or input. Real-device validation and precise shared-QPC drift correction
|
||||||
|
remain release requirements.
|
||||||
|
|||||||
@@ -2,7 +2,11 @@ module github.com/remotedesk/remotedesk/transport/hysteria2-agent
|
|||||||
|
|
||||||
go 1.25.0
|
go 1.25.0
|
||||||
|
|
||||||
require github.com/apernet/hysteria/core/v2 v2.12.1
|
require (
|
||||||
|
github.com/Microsoft/go-winio v0.6.2
|
||||||
|
github.com/apernet/hysteria/core/v2 v2.12.1
|
||||||
|
golang.org/x/sys v0.47.0
|
||||||
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/andybalholm/brotli v1.1.0 // indirect
|
github.com/andybalholm/brotli v1.1.0 // indirect
|
||||||
@@ -17,7 +21,6 @@ require (
|
|||||||
golang.org/x/crypto v0.54.0 // indirect
|
golang.org/x/crypto v0.54.0 // indirect
|
||||||
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
|
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
|
||||||
golang.org/x/net v0.57.0 // indirect
|
golang.org/x/net v0.57.0 // indirect
|
||||||
golang.org/x/sys v0.47.0 // indirect
|
|
||||||
golang.org/x/text v0.40.0 // indirect
|
golang.org/x/text v0.40.0 // indirect
|
||||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||||
|
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||||
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
|
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
|
||||||
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
|
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
|
||||||
github.com/apernet/hysteria/core/v2 v2.12.1 h1:7jA/jSC+1tFP8Z9COzI0AdT1p9c0SY7D8vngRd1YX7s=
|
github.com/apernet/hysteria/core/v2 v2.12.1 h1:7jA/jSC+1tFP8Z9COzI0AdT1p9c0SY7D8vngRd1YX7s=
|
||||||
|
|||||||
@@ -5,7 +5,9 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"crypto/tls"
|
"crypto/tls"
|
||||||
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
@@ -46,6 +48,46 @@ func (o localDatagramOutbound) UDP(_ string) (server.UDPConn, error) {
|
|||||||
|
|
||||||
type udpConn struct{ conn *net.UDPConn }
|
type udpConn struct{ conn *net.UDPConn }
|
||||||
|
|
||||||
|
type datagramKind uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
datagramOpaque datagramKind = iota
|
||||||
|
datagramVideo
|
||||||
|
)
|
||||||
|
|
||||||
|
// classifyDatagram keeps the relay payload opaque. Only the RemoteDesk video
|
||||||
|
// framing has a local size contract; control and audio packets are forwarded
|
||||||
|
// without parsing so they can evolve independently.
|
||||||
|
func classifyDatagram(data []byte) (datagramKind, error) {
|
||||||
|
if len(data) >= len(videoDatagramMagic) && string(data[:len(videoDatagramMagic)]) == videoDatagramMagic {
|
||||||
|
if err := validateVideoDatagramSize(data); err != nil {
|
||||||
|
return datagramVideo, err
|
||||||
|
}
|
||||||
|
return datagramVideo, nil
|
||||||
|
}
|
||||||
|
if len(data) >= len(audioDatagramMagic) && string(data[:len(audioDatagramMagic)]) == audioDatagramMagic {
|
||||||
|
if err := validateAudioDatagramSize(data); err != nil {
|
||||||
|
return datagramOpaque, err
|
||||||
|
}
|
||||||
|
if _, err := decodeAudioDatagram(data); err != nil {
|
||||||
|
return datagramOpaque, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return datagramOpaque, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func prepareDatagram(data []byte) ([]byte, datagramKind, error) {
|
||||||
|
if len(data) == 0 {
|
||||||
|
return nil, datagramOpaque, fmt.Errorf("empty RemoteDesk datagram")
|
||||||
|
}
|
||||||
|
kind, err := classifyDatagram(data)
|
||||||
|
if err != nil {
|
||||||
|
return nil, kind, err
|
||||||
|
}
|
||||||
|
// Always copy a UDP read buffer before handing it to Hysteria2.
|
||||||
|
return append([]byte(nil), data...), kind, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (c *udpConn) ReadFrom(b []byte) (int, string, error) {
|
func (c *udpConn) ReadFrom(b []byte) (int, string, error) {
|
||||||
n, addr, err := c.conn.ReadFromUDP(b)
|
n, addr, err := c.conn.ReadFromUDP(b)
|
||||||
if addr == nil {
|
if addr == nil {
|
||||||
@@ -66,9 +108,30 @@ func main() {
|
|||||||
agent := flag.String("agent-udp", env("REMOTEDESK_AGENT_UDP", "127.0.0.1:39502"), "local Agent UDP endpoint")
|
agent := flag.String("agent-udp", env("REMOTEDESK_AGENT_UDP", "127.0.0.1:39502"), "local Agent UDP endpoint")
|
||||||
serverName := flag.String("server-name", env("REMOTEDESK_HYSTERIA2_SERVER_NAME", "localhost"), "TLS server name (client mode)")
|
serverName := flag.String("server-name", env("REMOTEDESK_HYSTERIA2_SERVER_NAME", "localhost"), "TLS server name (client mode)")
|
||||||
insecure := flag.Bool("insecure", false, "disable TLS certificate verification (client mode only)")
|
insecure := flag.Bool("insecure", false, "disable TLS certificate verification (client mode only)")
|
||||||
|
ringPath := flag.String("encoded-ring", os.Getenv("REMOTEDESK_ENCODED_RING"), "local Rust encoded-access-unit ring path (optional)")
|
||||||
|
ringStreamID := flag.Uint("ring-stream-id", 0, "RemoteDesk video stream ID for the encoded ring")
|
||||||
|
ringGeneration := flag.Uint64("ring-generation", 0, "RemoteDesk encoded ring generation")
|
||||||
|
agentControl := flag.String("windows-agent-control", os.Getenv("REMOTEDESK_WINDOWS_AGENT_CONTROL"), "loopback Windows Agent TCP control endpoint (optional)")
|
||||||
|
agentPipe := flag.String("windows-agent-pipe", os.Getenv("REMOTEDESK_WINDOWS_AGENT_PIPE"), "authenticated local Windows Agent Named Pipe (recommended)")
|
||||||
|
sessionID := flag.String("session-id", os.Getenv("REMOTEDESK_SESSION_ID"), "encoded desktop session ID")
|
||||||
|
ringFPS := flag.Uint("ring-fps", 30, "encoded desktop frame rate (1-120)")
|
||||||
|
allowSoftwareFallback := flag.Bool("allow-software-fallback", true, "allow Media Foundation software H.264 fallback")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
if *mode == "client" {
|
if *mode == "client" {
|
||||||
runClient(*listen, *serverAddr, *password, *agent, *serverName, *insecure)
|
if *agentControl != "" && *agentPipe != "" {
|
||||||
|
log.Fatal("--windows-agent-control and --windows-agent-pipe are mutually exclusive")
|
||||||
|
}
|
||||||
|
controlEndpoint := *agentPipe
|
||||||
|
if controlEndpoint == "" {
|
||||||
|
controlEndpoint = *agentControl
|
||||||
|
}
|
||||||
|
if uint64(*ringStreamID) > uint64(^uint32(0)) {
|
||||||
|
log.Fatal("--ring-stream-id exceeds uint32")
|
||||||
|
}
|
||||||
|
if *ringFPS == 0 || *ringFPS > 120 {
|
||||||
|
log.Fatal("--ring-fps must be between 1 and 120")
|
||||||
|
}
|
||||||
|
runClient(*listen, *serverAddr, *password, *agent, *serverName, *insecure, *ringPath, uint32(*ringStreamID), *ringGeneration, controlEndpoint, *sessionID, uint8(*ringFPS), *allowSoftwareFallback)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if *mode != "server" {
|
if *mode != "server" {
|
||||||
@@ -118,7 +181,7 @@ func main() {
|
|||||||
time.Sleep(50 * time.Millisecond)
|
time.Sleep(50 * time.Millisecond)
|
||||||
}
|
}
|
||||||
|
|
||||||
func runClient(localListen, serverAddr, password, remoteAgent, serverName string, insecure bool) {
|
func runClient(localListen, serverAddr, password, remoteAgent, serverName string, insecure bool, ringPath string, ringStreamID uint32, ringGeneration uint64, agentControl, sessionID string, ringFPS uint8, allowSoftwareFallback bool) {
|
||||||
if password == "" {
|
if password == "" {
|
||||||
log.Fatal("--password is required in client mode")
|
log.Fatal("--password is required in client mode")
|
||||||
}
|
}
|
||||||
@@ -152,20 +215,92 @@ func runClient(localListen, serverAddr, password, remoteAgent, serverName string
|
|||||||
log.Fatalf("open Hysteria2 UDP session: %v", err)
|
log.Fatalf("open Hysteria2 UDP session: %v", err)
|
||||||
}
|
}
|
||||||
defer stream.Close()
|
defer stream.Close()
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
var sendMu sync.Mutex
|
||||||
log.Printf("RemoteDesk Hysteria2 client listening on %s (UDP only)", localConn.LocalAddr())
|
log.Printf("RemoteDesk Hysteria2 client listening on %s (UDP only)", localConn.LocalAddr())
|
||||||
log.Printf("authenticated Hysteria2 server %s; remote Agent %s", serverAddr, remoteAgent)
|
log.Printf("authenticated Hysteria2 server %s; remote Agent %s", serverAddr, remoteAgent)
|
||||||
|
var agentControlConnection net.Conn
|
||||||
|
var audioDescriptor *audioRingDescriptor
|
||||||
|
if ringPath == "" && agentControl != "" {
|
||||||
|
connection, descriptor, err := openWindowsEncodedSession(agentControl, sessionID, ringFPS, allowSoftwareFallback)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("start Windows encoded desktop: %v", err)
|
||||||
|
}
|
||||||
|
agentControlConnection = connection
|
||||||
|
ringPath = descriptor.RingPath
|
||||||
|
ringGeneration = descriptor.Generation
|
||||||
|
if ringStreamID == 0 {
|
||||||
|
ringStreamID = 1
|
||||||
|
}
|
||||||
|
audioDescriptor = descriptor.AudioRing
|
||||||
|
log.Printf("Windows encoded desktop session %s generation %d opened", sessionID, ringGeneration)
|
||||||
|
}
|
||||||
|
if agentControlConnection != nil {
|
||||||
|
defer agentControlConnection.Close()
|
||||||
|
}
|
||||||
|
if ringPath != "" {
|
||||||
|
if ringStreamID == 0 || ringGeneration == 0 {
|
||||||
|
log.Fatal("--ring-stream-id and --ring-generation are required with --encoded-ring")
|
||||||
|
}
|
||||||
|
ring, err := openEncodedRing(ringPath)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("open encoded media ring: %v", err)
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
defer ring.Close()
|
||||||
|
err := pumpEncodedRing(ctx, ring, ringStreamID, ringGeneration, func(data []byte) error {
|
||||||
|
sendMu.Lock()
|
||||||
|
defer sendMu.Unlock()
|
||||||
|
return stream.Send(data, remoteAgent)
|
||||||
|
})
|
||||||
|
if err != nil && !errors.Is(err, context.Canceled) {
|
||||||
|
log.Printf("encoded media ring stopped: %v", err)
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
if agentControlConnection != nil {
|
||||||
|
_ = agentControlConnection.Close()
|
||||||
|
}
|
||||||
|
_ = localConn.Close()
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
if audioDescriptor != nil {
|
||||||
|
audioRing, err := openAudioRing(audioDescriptor.RingPath)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("audio ring unavailable, continuing without audio: %v", err)
|
||||||
|
} else {
|
||||||
|
go func() {
|
||||||
|
defer audioRing.Close()
|
||||||
|
err := pumpAudioRing(ctx, audioRing, ringStreamID, audioDescriptor.Generation, func(data []byte) error {
|
||||||
|
sendMu.Lock()
|
||||||
|
defer sendMu.Unlock()
|
||||||
|
return stream.Send(data, remoteAgent)
|
||||||
|
})
|
||||||
|
if err != nil && !errors.Is(err, context.Canceled) {
|
||||||
|
log.Printf("audio media ring stopped: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
}
|
||||||
var peerMu sync.RWMutex
|
var peerMu sync.RWMutex
|
||||||
var peer *net.UDPAddr
|
var peer *net.UDPAddr
|
||||||
go func() {
|
go func() {
|
||||||
for {
|
for {
|
||||||
data, _, err := stream.Receive()
|
data, _, err := stream.Receive()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
cancel()
|
||||||
|
_ = localConn.Close()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
prepared, _, err := prepareDatagram(data)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("drop invalid datagram from Hysteria2: %v", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
peerMu.RLock()
|
peerMu.RLock()
|
||||||
currentPeer := peer
|
currentPeer := peer
|
||||||
if currentPeer != nil {
|
if currentPeer != nil {
|
||||||
_, _ = localConn.WriteToUDP(data, currentPeer)
|
_, _ = localConn.WriteToUDP(prepared, currentPeer)
|
||||||
}
|
}
|
||||||
peerMu.RUnlock()
|
peerMu.RUnlock()
|
||||||
}
|
}
|
||||||
@@ -179,9 +314,17 @@ func runClient(localListen, serverAddr, password, remoteAgent, serverName string
|
|||||||
peerMu.Lock()
|
peerMu.Lock()
|
||||||
peer = sourcePeer
|
peer = sourcePeer
|
||||||
peerMu.Unlock()
|
peerMu.Unlock()
|
||||||
|
prepared, _, err := prepareDatagram(buf[:n])
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("drop invalid local datagram from %s: %v", sourcePeer, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
// The Hysteria2 destination is metadata only. The server's outbound
|
// The Hysteria2 destination is metadata only. The server's outbound
|
||||||
// implementation pins delivery to its configured local Agent UDP port.
|
// implementation pins delivery to its configured local Agent UDP port.
|
||||||
if err := stream.Send(append([]byte(nil), buf[:n]...), remoteAgent); err != nil {
|
sendMu.Lock()
|
||||||
|
err = stream.Send(prepared, remoteAgent)
|
||||||
|
sendMu.Unlock()
|
||||||
|
if err != nil {
|
||||||
log.Printf("send Hysteria2 datagram from %s: %v", sourcePeer, err)
|
log.Printf("send Hysteria2 datagram from %s: %v", sourcePeer, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,352 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/binary"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
videoDatagramMagic = "RDV1"
|
||||||
|
videoDatagramVersion = 1
|
||||||
|
videoDatagramHeaderBytes = 58
|
||||||
|
videoDatagramPayloadMax = 1200
|
||||||
|
videoDatagramMaxBytes = videoDatagramHeaderBytes + videoDatagramPayloadMax
|
||||||
|
videoFrameMaxBytes = 16 * 1024 * 1024
|
||||||
|
videoFrameMaxFragments = 4096
|
||||||
|
videoReassemblyMaxFrames = 8
|
||||||
|
videoReassemblyTimeout = 100 * time.Millisecond
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
audioDatagramMagic = "RDA1"
|
||||||
|
audioDatagramVersion = 1
|
||||||
|
audioDatagramHeaderSize = 40
|
||||||
|
audioPacketMaxBytes = 4 * 1024
|
||||||
|
audioDatagramMaxBytes = audioDatagramHeaderSize + audioPacketMaxBytes
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
videoFlagConfig uint16 = 1 << 0
|
||||||
|
videoFlagKeyFrame uint16 = 1 << 1
|
||||||
|
videoFlagFirstFragment uint16 = 1 << 2
|
||||||
|
videoFlagLastFragment uint16 = 1 << 3
|
||||||
|
videoFlagDiscontinuity uint16 = 1 << 4
|
||||||
|
)
|
||||||
|
|
||||||
|
const audioFlagDiscontinuity uint16 = 1 << 0
|
||||||
|
|
||||||
|
// VideoAccessUnit is a complete encoded frame before transport fragmentation.
|
||||||
|
// The Hysteria2 relay must treat the resulting datagrams as opaque bytes.
|
||||||
|
type VideoAccessUnit struct {
|
||||||
|
StreamID uint32
|
||||||
|
Generation uint64
|
||||||
|
Sequence uint64
|
||||||
|
FrameID uint64
|
||||||
|
PTS uint64
|
||||||
|
DTS uint64
|
||||||
|
Flags uint16
|
||||||
|
Payload []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// AudioPacket is one independently decodable Opus packet. Audio is never
|
||||||
|
// reassembled with video and is forwarded on its own priority path.
|
||||||
|
type AudioPacket struct {
|
||||||
|
StreamID uint32
|
||||||
|
Generation uint64
|
||||||
|
Sequence uint64
|
||||||
|
PTS uint64
|
||||||
|
DurationMS uint16
|
||||||
|
Flags uint16
|
||||||
|
Payload []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func encodeAudioDatagram(packet AudioPacket) ([]byte, error) {
|
||||||
|
if err := validateAudioPacket(packet); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
output := make([]byte, audioDatagramHeaderSize+len(packet.Payload))
|
||||||
|
copy(output, audioDatagramMagic)
|
||||||
|
output[4] = audioDatagramVersion
|
||||||
|
binary.LittleEndian.PutUint16(output[6:8], audioDatagramHeaderSize)
|
||||||
|
binary.LittleEndian.PutUint32(output[8:12], packet.StreamID)
|
||||||
|
binary.LittleEndian.PutUint64(output[12:20], packet.Generation)
|
||||||
|
binary.LittleEndian.PutUint64(output[20:28], packet.Sequence)
|
||||||
|
binary.LittleEndian.PutUint64(output[28:36], packet.PTS)
|
||||||
|
binary.LittleEndian.PutUint16(output[36:38], packet.DurationMS)
|
||||||
|
binary.LittleEndian.PutUint16(output[38:40], packet.Flags)
|
||||||
|
copy(output[audioDatagramHeaderSize:], packet.Payload)
|
||||||
|
return output, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeAudioDatagram(input []byte) (AudioPacket, error) {
|
||||||
|
if len(input) < audioDatagramHeaderSize || len(input) > audioDatagramMaxBytes {
|
||||||
|
return AudioPacket{}, errors.New("audio datagram size is invalid")
|
||||||
|
}
|
||||||
|
if string(input[:4]) != audioDatagramMagic || input[4] != audioDatagramVersion || input[5] != 0 {
|
||||||
|
return AudioPacket{}, errors.New("audio datagram version is invalid")
|
||||||
|
}
|
||||||
|
headerBytes := int(binary.LittleEndian.Uint16(input[6:8]))
|
||||||
|
if headerBytes != audioDatagramHeaderSize || headerBytes > len(input) {
|
||||||
|
return AudioPacket{}, errors.New("audio datagram header is invalid")
|
||||||
|
}
|
||||||
|
packet := AudioPacket{
|
||||||
|
StreamID: binary.LittleEndian.Uint32(input[8:12]),
|
||||||
|
Generation: binary.LittleEndian.Uint64(input[12:20]),
|
||||||
|
Sequence: binary.LittleEndian.Uint64(input[20:28]),
|
||||||
|
PTS: binary.LittleEndian.Uint64(input[28:36]),
|
||||||
|
DurationMS: binary.LittleEndian.Uint16(input[36:38]),
|
||||||
|
Flags: binary.LittleEndian.Uint16(input[38:40]),
|
||||||
|
Payload: input[headerBytes:],
|
||||||
|
}
|
||||||
|
if err := validateAudioPacket(packet); err != nil {
|
||||||
|
return AudioPacket{}, err
|
||||||
|
}
|
||||||
|
return packet, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateAudioPacket(packet AudioPacket) error {
|
||||||
|
if packet.StreamID == 0 || packet.Generation == 0 || packet.Sequence == 0 {
|
||||||
|
return errors.New("audio packet identity is invalid")
|
||||||
|
}
|
||||||
|
if packet.DurationMS != 10 && packet.DurationMS != 20 && packet.DurationMS != 40 && packet.DurationMS != 60 {
|
||||||
|
return errors.New("audio packet duration is invalid")
|
||||||
|
}
|
||||||
|
if len(packet.Payload) == 0 || len(packet.Payload) > audioPacketMaxBytes {
|
||||||
|
return errors.New("audio packet payload size is invalid")
|
||||||
|
}
|
||||||
|
if packet.Flags & ^audioFlagDiscontinuity != 0 {
|
||||||
|
return errors.New("audio packet flags are invalid")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type videoDatagram struct {
|
||||||
|
StreamID uint32
|
||||||
|
Generation uint64
|
||||||
|
Sequence uint64
|
||||||
|
FrameID uint64
|
||||||
|
PTS uint64
|
||||||
|
DTS uint64
|
||||||
|
FragmentID uint16
|
||||||
|
FragmentCount uint16
|
||||||
|
Flags uint16
|
||||||
|
Payload []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func fragmentVideoAccessUnit(unit VideoAccessUnit) ([][]byte, error) {
|
||||||
|
if unit.StreamID == 0 || unit.Generation == 0 {
|
||||||
|
return nil, errors.New("video stream identity is invalid")
|
||||||
|
}
|
||||||
|
if len(unit.Payload) == 0 || len(unit.Payload) > videoFrameMaxBytes {
|
||||||
|
return nil, errors.New("video access unit size is invalid")
|
||||||
|
}
|
||||||
|
fragmentCount := (len(unit.Payload) + videoDatagramPayloadMax - 1) / videoDatagramPayloadMax
|
||||||
|
if fragmentCount > videoFrameMaxFragments {
|
||||||
|
return nil, errors.New("video access unit has too many fragments")
|
||||||
|
}
|
||||||
|
|
||||||
|
datagrams := make([][]byte, 0, fragmentCount)
|
||||||
|
for fragmentID, offset := 0, 0; offset < len(unit.Payload); fragmentID++ {
|
||||||
|
end := min(offset+videoDatagramPayloadMax, len(unit.Payload))
|
||||||
|
flags := unit.Flags
|
||||||
|
if fragmentID == 0 {
|
||||||
|
flags |= videoFlagFirstFragment
|
||||||
|
}
|
||||||
|
if end == len(unit.Payload) {
|
||||||
|
flags |= videoFlagLastFragment
|
||||||
|
}
|
||||||
|
packet := videoDatagram{
|
||||||
|
StreamID: unit.StreamID,
|
||||||
|
Generation: unit.Generation,
|
||||||
|
Sequence: unit.Sequence,
|
||||||
|
FrameID: unit.FrameID,
|
||||||
|
PTS: unit.PTS,
|
||||||
|
DTS: unit.DTS,
|
||||||
|
FragmentID: uint16(fragmentID),
|
||||||
|
FragmentCount: uint16(fragmentCount),
|
||||||
|
Flags: flags,
|
||||||
|
Payload: unit.Payload[offset:end],
|
||||||
|
}
|
||||||
|
datagrams = append(datagrams, encodeVideoDatagram(packet))
|
||||||
|
offset = end
|
||||||
|
}
|
||||||
|
return datagrams, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func encodeVideoDatagram(packet videoDatagram) []byte {
|
||||||
|
output := make([]byte, videoDatagramHeaderBytes+len(packet.Payload))
|
||||||
|
copy(output, videoDatagramMagic)
|
||||||
|
output[4] = videoDatagramVersion
|
||||||
|
binary.LittleEndian.PutUint16(output[6:8], videoDatagramHeaderBytes)
|
||||||
|
binary.LittleEndian.PutUint32(output[8:12], packet.StreamID)
|
||||||
|
binary.LittleEndian.PutUint64(output[12:20], packet.Generation)
|
||||||
|
binary.LittleEndian.PutUint64(output[20:28], packet.Sequence)
|
||||||
|
binary.LittleEndian.PutUint64(output[28:36], packet.FrameID)
|
||||||
|
binary.LittleEndian.PutUint64(output[36:44], packet.PTS)
|
||||||
|
binary.LittleEndian.PutUint64(output[44:52], packet.DTS)
|
||||||
|
binary.LittleEndian.PutUint16(output[52:54], packet.FragmentID)
|
||||||
|
binary.LittleEndian.PutUint16(output[54:56], packet.FragmentCount)
|
||||||
|
binary.LittleEndian.PutUint16(output[56:58], packet.Flags)
|
||||||
|
copy(output[videoDatagramHeaderBytes:], packet.Payload)
|
||||||
|
return output
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeVideoDatagram(input []byte) (videoDatagram, error) {
|
||||||
|
if len(input) < videoDatagramHeaderBytes || len(input) > videoDatagramHeaderBytes+videoDatagramPayloadMax {
|
||||||
|
return videoDatagram{}, errors.New("video datagram size is invalid")
|
||||||
|
}
|
||||||
|
if string(input[:4]) != videoDatagramMagic || input[4] != videoDatagramVersion {
|
||||||
|
return videoDatagram{}, errors.New("video datagram version is invalid")
|
||||||
|
}
|
||||||
|
headerBytes := int(binary.LittleEndian.Uint16(input[6:8]))
|
||||||
|
if headerBytes != videoDatagramHeaderBytes || headerBytes > len(input) {
|
||||||
|
return videoDatagram{}, errors.New("video datagram header is invalid")
|
||||||
|
}
|
||||||
|
packet := videoDatagram{
|
||||||
|
StreamID: binary.LittleEndian.Uint32(input[8:12]),
|
||||||
|
Generation: binary.LittleEndian.Uint64(input[12:20]),
|
||||||
|
Sequence: binary.LittleEndian.Uint64(input[20:28]),
|
||||||
|
FrameID: binary.LittleEndian.Uint64(input[28:36]),
|
||||||
|
PTS: binary.LittleEndian.Uint64(input[36:44]),
|
||||||
|
DTS: binary.LittleEndian.Uint64(input[44:52]),
|
||||||
|
FragmentID: binary.LittleEndian.Uint16(input[52:54]),
|
||||||
|
FragmentCount: binary.LittleEndian.Uint16(input[54:56]),
|
||||||
|
Flags: binary.LittleEndian.Uint16(input[56:58]),
|
||||||
|
Payload: input[headerBytes:],
|
||||||
|
}
|
||||||
|
if packet.StreamID == 0 || packet.Generation == 0 || packet.FragmentCount == 0 ||
|
||||||
|
packet.FragmentCount > videoFrameMaxFragments || packet.FragmentID >= packet.FragmentCount ||
|
||||||
|
len(packet.Payload) == 0 || len(packet.Payload) > videoDatagramPayloadMax {
|
||||||
|
return videoDatagram{}, errors.New("video datagram fields are invalid")
|
||||||
|
}
|
||||||
|
if packet.Flags&videoFlagFirstFragment != 0 != (packet.FragmentID == 0) ||
|
||||||
|
packet.Flags&videoFlagLastFragment != 0 != (packet.FragmentID+1 == packet.FragmentCount) {
|
||||||
|
return videoDatagram{}, errors.New("video datagram fragment flags are invalid")
|
||||||
|
}
|
||||||
|
return packet, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type videoFrameKey struct {
|
||||||
|
streamID uint32
|
||||||
|
generation uint64
|
||||||
|
frameID uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
type videoFrameAssembly struct {
|
||||||
|
created time.Time
|
||||||
|
sequence uint64
|
||||||
|
pts uint64
|
||||||
|
dts uint64
|
||||||
|
flags uint16
|
||||||
|
fragmentCount uint16
|
||||||
|
fragments map[uint16][]byte
|
||||||
|
bytes int
|
||||||
|
}
|
||||||
|
|
||||||
|
// videoReassembler is intentionally bounded. It never waits for a missing
|
||||||
|
// datagram past the deadline and never retransmits an expired video frame.
|
||||||
|
type videoReassembler struct {
|
||||||
|
frames map[videoFrameKey]*videoFrameAssembly
|
||||||
|
}
|
||||||
|
|
||||||
|
func newVideoReassembler() *videoReassembler {
|
||||||
|
return &videoReassembler{frames: make(map[videoFrameKey]*videoFrameAssembly)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *videoReassembler) push(input []byte, now time.Time) (*VideoAccessUnit, error) {
|
||||||
|
packet, err := decodeVideoDatagram(input)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
r.expire(now)
|
||||||
|
key := videoFrameKey{streamID: packet.StreamID, generation: packet.Generation, frameID: packet.FrameID}
|
||||||
|
assembly := r.frames[key]
|
||||||
|
if assembly == nil {
|
||||||
|
if len(r.frames) >= videoReassemblyMaxFrames {
|
||||||
|
r.dropOldest()
|
||||||
|
}
|
||||||
|
assembly = &videoFrameAssembly{
|
||||||
|
created: now,
|
||||||
|
sequence: packet.Sequence,
|
||||||
|
pts: packet.PTS,
|
||||||
|
dts: packet.DTS,
|
||||||
|
flags: packet.Flags &^ (videoFlagFirstFragment | videoFlagLastFragment),
|
||||||
|
fragmentCount: packet.FragmentCount,
|
||||||
|
fragments: make(map[uint16][]byte, packet.FragmentCount),
|
||||||
|
}
|
||||||
|
r.frames[key] = assembly
|
||||||
|
} else if assembly.fragmentCount != packet.FragmentCount || assembly.sequence != packet.Sequence {
|
||||||
|
delete(r.frames, key)
|
||||||
|
return nil, errors.New("video frame metadata changed during reassembly")
|
||||||
|
}
|
||||||
|
if _, exists := assembly.fragments[packet.FragmentID]; !exists {
|
||||||
|
assembly.fragments[packet.FragmentID] = append([]byte(nil), packet.Payload...)
|
||||||
|
assembly.bytes += len(packet.Payload)
|
||||||
|
}
|
||||||
|
if assembly.bytes > videoFrameMaxBytes || len(assembly.fragments) != int(assembly.fragmentCount) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
payload := make([]byte, 0, assembly.bytes)
|
||||||
|
for index := uint16(0); index < assembly.fragmentCount; index++ {
|
||||||
|
fragment, ok := assembly.fragments[index]
|
||||||
|
if !ok {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
payload = append(payload, fragment...)
|
||||||
|
}
|
||||||
|
delete(r.frames, key)
|
||||||
|
return &VideoAccessUnit{
|
||||||
|
StreamID: packet.StreamID,
|
||||||
|
Generation: packet.Generation,
|
||||||
|
Sequence: assembly.sequence,
|
||||||
|
FrameID: packet.FrameID,
|
||||||
|
PTS: assembly.pts,
|
||||||
|
DTS: assembly.dts,
|
||||||
|
Flags: assembly.flags | (packet.Flags & (videoFlagConfig | videoFlagKeyFrame | videoFlagDiscontinuity)),
|
||||||
|
Payload: payload,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *videoReassembler) expire(now time.Time) {
|
||||||
|
for key, assembly := range r.frames {
|
||||||
|
if now.Sub(assembly.created) >= videoReassemblyTimeout {
|
||||||
|
delete(r.frames, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *videoReassembler) dropOldest() {
|
||||||
|
var oldest videoFrameKey
|
||||||
|
var oldestTime time.Time
|
||||||
|
for key, assembly := range r.frames {
|
||||||
|
if oldestTime.IsZero() || assembly.created.Before(oldestTime) {
|
||||||
|
oldest, oldestTime = key, assembly.created
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !oldestTime.IsZero() {
|
||||||
|
delete(r.frames, oldest)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func min(a, b int) int {
|
||||||
|
if a < b {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateVideoDatagramSize(input []byte) error {
|
||||||
|
if len(input) > videoDatagramMaxBytes {
|
||||||
|
return fmt.Errorf("video datagram exceeds %d bytes", videoDatagramMaxBytes)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateAudioDatagramSize(input []byte) error {
|
||||||
|
if len(input) > audioDatagramMaxBytes {
|
||||||
|
return fmt.Errorf("audio datagram exceeds %d bytes", audioDatagramMaxBytes)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
)
|
||||||
|
|
||||||
|
// EncodedRingRecord is the cross-process contract produced by the Rust
|
||||||
|
// Windows agent. PTS is the Media Foundation 100-ns media timeline.
|
||||||
|
type EncodedRingRecord struct {
|
||||||
|
Sequence uint64
|
||||||
|
PTS uint64
|
||||||
|
Kind uint32
|
||||||
|
KeyFrame bool
|
||||||
|
Payload []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncodedRing is intentionally a single-consumer interface. The producer may
|
||||||
|
// drop a video record when this queue is full; the consumer must never block
|
||||||
|
// input or audio while polling it.
|
||||||
|
type EncodedRing interface {
|
||||||
|
Pop() (*EncodedRingRecord, error)
|
||||||
|
Close() error
|
||||||
|
}
|
||||||
|
|
||||||
|
// AudioRing is a separate consumer contract even though its mmap slot layout
|
||||||
|
// matches the video ring. Keeping the interfaces distinct prevents audio from
|
||||||
|
// accidentally sharing video backpressure or drop policy.
|
||||||
|
type AudioRing interface {
|
||||||
|
Pop() (*EncodedRingRecord, error)
|
||||||
|
Close() error
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateRingRecord(record *EncodedRingRecord) error {
|
||||||
|
if record == nil || record.Sequence == 0 || len(record.Payload) == 0 || len(record.Payload) > videoFrameMaxBytes {
|
||||||
|
return errors.New("encoded ring record is invalid")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// pumpEncodedRing drains the ring into RDV1 datagrams. The callback is the
|
||||||
|
// already-authenticated Hysteria2 UDP stream; this function never retries a
|
||||||
|
// failed video packet.
|
||||||
|
func pumpEncodedRing(ctx context.Context, ring EncodedRing, streamID uint32, generation uint64, send func([]byte) error) error {
|
||||||
|
if streamID == 0 || generation == 0 {
|
||||||
|
return errors.New("encoded ring stream identity is invalid")
|
||||||
|
}
|
||||||
|
if ring == nil || send == nil {
|
||||||
|
return errors.New("encoded ring pump arguments are invalid")
|
||||||
|
}
|
||||||
|
ticker := newMediaPollTicker()
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
drained := false
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
record, err := ring.Pop()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("read encoded media ring: %w", err)
|
||||||
|
}
|
||||||
|
if record == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
drained = true
|
||||||
|
if err := validateRingRecord(record); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
flags := uint16(0)
|
||||||
|
if record.KeyFrame {
|
||||||
|
flags |= videoFlagKeyFrame
|
||||||
|
}
|
||||||
|
datagrams, err := fragmentVideoAccessUnit(VideoAccessUnit{
|
||||||
|
StreamID: streamID, Generation: generation, Sequence: record.Sequence,
|
||||||
|
FrameID: record.Sequence, PTS: record.PTS, DTS: record.PTS,
|
||||||
|
Flags: flags, Payload: record.Payload,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("fragment encoded media: %w", err)
|
||||||
|
}
|
||||||
|
for _, datagram := range datagrams {
|
||||||
|
if err := send(datagram); err != nil {
|
||||||
|
return fmt.Errorf("send encoded video datagram: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if drained {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-ticker.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pumpAudioRing sends independent Opus packets as RDA1 datagrams. Audio does
|
||||||
|
// not wait for video and does not use video frame reassembly.
|
||||||
|
func pumpAudioRing(ctx context.Context, ring AudioRing, streamID uint32, generation uint64, send func([]byte) error) error {
|
||||||
|
if streamID == 0 || generation == 0 {
|
||||||
|
return errors.New("audio ring stream identity is invalid")
|
||||||
|
}
|
||||||
|
if ring == nil || send == nil {
|
||||||
|
return errors.New("audio ring pump arguments are invalid")
|
||||||
|
}
|
||||||
|
ticker := newMediaPollTicker()
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
drained := false
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
record, err := ring.Pop()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("read audio media ring: %w", err)
|
||||||
|
}
|
||||||
|
if record == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
drained = true
|
||||||
|
if record.Sequence == 0 || len(record.Payload) == 0 || len(record.Payload) > audioPacketMaxBytes {
|
||||||
|
return errors.New("audio ring record is invalid")
|
||||||
|
}
|
||||||
|
flags := uint16(0)
|
||||||
|
if record.Kind == 2 {
|
||||||
|
flags |= audioFlagDiscontinuity
|
||||||
|
}
|
||||||
|
datagram, err := encodeAudioDatagram(AudioPacket{
|
||||||
|
StreamID: streamID, Generation: generation, Sequence: record.Sequence,
|
||||||
|
PTS: record.PTS, DurationMS: 20, Flags: flags, Payload: record.Payload,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("encode audio datagram: %w", err)
|
||||||
|
}
|
||||||
|
if err := send(datagram); err != nil {
|
||||||
|
return fmt.Errorf("send audio datagram: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if drained {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-ticker.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
type mediaPollTicker struct {
|
||||||
|
*time.Ticker
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMediaPollTicker() *mediaPollTicker {
|
||||||
|
return &mediaPollTicker{Ticker: time.NewTicker(2 * time.Millisecond)}
|
||||||
|
}
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
//go:build !windows
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/binary"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"sync/atomic"
|
||||||
|
"syscall"
|
||||||
|
"unsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
ringMagic = "RDMRING1"
|
||||||
|
ringVersion = uint32(1)
|
||||||
|
ringHeaderBytes = 40
|
||||||
|
slotHeaderBytes = 24
|
||||||
|
maxRingSlots = uint32(256)
|
||||||
|
maxRingSlotBytes = uint32(16 * 1024 * 1024)
|
||||||
|
)
|
||||||
|
|
||||||
|
type mappedEncodedRing struct {
|
||||||
|
file *os.File
|
||||||
|
path string
|
||||||
|
data []byte
|
||||||
|
slotCount uint32
|
||||||
|
slotBytes uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
func openEncodedRing(path string) (EncodedRing, error) {
|
||||||
|
file, err := os.OpenFile(path, os.O_RDWR, 0)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("open encoded ring: %w", err)
|
||||||
|
}
|
||||||
|
stat, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
_ = file.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if stat.Size() < ringHeaderBytes {
|
||||||
|
_ = file.Close()
|
||||||
|
return nil, errors.New("encoded ring is shorter than its header")
|
||||||
|
}
|
||||||
|
data, err := syscall.Mmap(int(file.Fd()), 0, int(stat.Size()), syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
|
||||||
|
if err != nil {
|
||||||
|
_ = file.Close()
|
||||||
|
return nil, fmt.Errorf("map encoded ring: %w", err)
|
||||||
|
}
|
||||||
|
if string(data[:8]) != ringMagic || binary.LittleEndian.Uint32(data[8:12]) != ringVersion {
|
||||||
|
_ = syscall.Munmap(data)
|
||||||
|
_ = file.Close()
|
||||||
|
return nil, errors.New("encoded ring header is invalid")
|
||||||
|
}
|
||||||
|
slots, slotBytes := binary.LittleEndian.Uint32(data[12:16]), binary.LittleEndian.Uint32(data[16:20])
|
||||||
|
expected := uint64(ringHeaderBytes) + uint64(slots)*uint64(slotHeaderBytes+slotBytes)
|
||||||
|
if slots < 2 || slots > maxRingSlots || slots&(slots-1) != 0 || slotBytes == 0 || slotBytes > maxRingSlotBytes || expected != uint64(len(data)) {
|
||||||
|
_ = syscall.Munmap(data)
|
||||||
|
_ = file.Close()
|
||||||
|
return nil, errors.New("encoded ring geometry is invalid")
|
||||||
|
}
|
||||||
|
return &mappedEncodedRing{file: file, path: path, data: data, slotCount: slots, slotBytes: slotBytes}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func openAudioRing(path string) (AudioRing, error) {
|
||||||
|
ring, err := openEncodedRing(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return ring, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *mappedEncodedRing) Pop() (*EncodedRingRecord, error) {
|
||||||
|
read := atomic.LoadUint64((*uint64)(unsafe.Pointer(&r.data[32])))
|
||||||
|
write := atomic.LoadUint64((*uint64)(unsafe.Pointer(&r.data[24])))
|
||||||
|
if read == write {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
start := ringHeaderBytes + int(read%uint64(r.slotCount))*int(slotHeaderBytes+r.slotBytes)
|
||||||
|
length := binary.LittleEndian.Uint32(r.data[start+20 : start+24])
|
||||||
|
if length == 0 || length > r.slotBytes {
|
||||||
|
return nil, errors.New("encoded ring record length is invalid")
|
||||||
|
}
|
||||||
|
record := &EncodedRingRecord{Sequence: binary.LittleEndian.Uint64(r.data[start : start+8]), PTS: binary.LittleEndian.Uint64(r.data[start+8 : start+16]), Kind: binary.LittleEndian.Uint32(r.data[start+16 : start+20]), Payload: append([]byte(nil), r.data[start+slotHeaderBytes:start+slotHeaderBytes+int(length)]...)}
|
||||||
|
record.KeyFrame = record.Kind == 2
|
||||||
|
if record.Sequence == 0 {
|
||||||
|
return nil, errors.New("encoded ring record sequence is invalid")
|
||||||
|
}
|
||||||
|
atomic.StoreUint64((*uint64)(unsafe.Pointer(&r.data[32])), read+1)
|
||||||
|
return record, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *mappedEncodedRing) Close() error {
|
||||||
|
err := syscall.Munmap(r.data)
|
||||||
|
if r.file != nil {
|
||||||
|
if closeErr := r.file.Close(); err == nil {
|
||||||
|
err = closeErr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if removeErr := os.Remove(r.path); err == nil && removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) {
|
||||||
|
err = removeErr
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
//go:build windows
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/binary"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"sync/atomic"
|
||||||
|
"unsafe"
|
||||||
|
|
||||||
|
"golang.org/x/sys/windows"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
ringMagic = "RDMRING1"
|
||||||
|
ringVersion = uint32(1)
|
||||||
|
ringHeaderBytes = 40
|
||||||
|
slotHeaderBytes = 24
|
||||||
|
maxRingSlots = uint32(256)
|
||||||
|
maxRingSlotBytes = uint32(16 * 1024 * 1024)
|
||||||
|
)
|
||||||
|
|
||||||
|
type mappedEncodedRing struct {
|
||||||
|
file *os.File
|
||||||
|
path string
|
||||||
|
mapping windows.Handle
|
||||||
|
address uintptr
|
||||||
|
data []byte
|
||||||
|
slotCount uint32
|
||||||
|
slotBytes uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
func openEncodedRing(path string) (EncodedRing, error) {
|
||||||
|
file, err := os.OpenFile(path, os.O_RDWR, 0)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("open encoded ring: %w", err)
|
||||||
|
}
|
||||||
|
closeOnError := true
|
||||||
|
defer func() {
|
||||||
|
if closeOnError {
|
||||||
|
_ = file.Close()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
stat, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("stat encoded ring: %w", err)
|
||||||
|
}
|
||||||
|
length := stat.Size()
|
||||||
|
if length < ringHeaderBytes {
|
||||||
|
return nil, errors.New("encoded ring is shorter than its header")
|
||||||
|
}
|
||||||
|
mapping, err := windows.CreateFileMapping(windows.Handle(file.Fd()), nil, windows.PAGE_READWRITE, 0, 0, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("map encoded ring file: %w", err)
|
||||||
|
}
|
||||||
|
address, err := windows.MapViewOfFile(mapping, windows.FILE_MAP_READ|windows.FILE_MAP_WRITE, 0, 0, uintptr(length))
|
||||||
|
if err != nil {
|
||||||
|
_ = windows.CloseHandle(mapping)
|
||||||
|
return nil, fmt.Errorf("map encoded ring view: %w", err)
|
||||||
|
}
|
||||||
|
if uintptr(length) > uintptr(^uint(0)>>1) {
|
||||||
|
_ = windows.UnmapViewOfFile(address)
|
||||||
|
_ = windows.CloseHandle(mapping)
|
||||||
|
return nil, errors.New("encoded ring is too large for this process")
|
||||||
|
}
|
||||||
|
data := unsafe.Slice((*byte)(unsafe.Pointer(address)), int(length))
|
||||||
|
if string(data[:8]) != ringMagic || binary.LittleEndian.Uint32(data[8:12]) != ringVersion {
|
||||||
|
_ = windows.UnmapViewOfFile(address)
|
||||||
|
_ = windows.CloseHandle(mapping)
|
||||||
|
return nil, errors.New("encoded ring header magic or version is invalid")
|
||||||
|
}
|
||||||
|
slots := binary.LittleEndian.Uint32(data[12:16])
|
||||||
|
slotBytes := binary.LittleEndian.Uint32(data[16:20])
|
||||||
|
if slots < 2 || slots > maxRingSlots || slots&(slots-1) != 0 || slotBytes == 0 || slotBytes > maxRingSlotBytes {
|
||||||
|
_ = windows.UnmapViewOfFile(address)
|
||||||
|
_ = windows.CloseHandle(mapping)
|
||||||
|
return nil, errors.New("encoded ring geometry is invalid")
|
||||||
|
}
|
||||||
|
expected := uint64(ringHeaderBytes) + uint64(slots)*uint64(slotHeaderBytes+slotBytes)
|
||||||
|
if expected != uint64(length) {
|
||||||
|
_ = windows.UnmapViewOfFile(address)
|
||||||
|
_ = windows.CloseHandle(mapping)
|
||||||
|
return nil, errors.New("encoded ring file size does not match its header")
|
||||||
|
}
|
||||||
|
closeOnError = false
|
||||||
|
return &mappedEncodedRing{file: file, path: path, mapping: mapping, address: address, data: data, slotCount: slots, slotBytes: slotBytes}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func openAudioRing(path string) (AudioRing, error) {
|
||||||
|
ring, err := openEncodedRing(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return ring, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *mappedEncodedRing) Pop() (*EncodedRingRecord, error) {
|
||||||
|
read := atomic.LoadUint64((*uint64)(unsafe.Pointer(&r.data[32])))
|
||||||
|
write := atomic.LoadUint64((*uint64)(unsafe.Pointer(&r.data[24])))
|
||||||
|
if read == write {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
offset := ringHeaderBytes + (read%uint64(r.slotCount))*uint64(slotHeaderBytes+r.slotBytes)
|
||||||
|
start := int(offset)
|
||||||
|
if start+slotHeaderBytes > len(r.data) {
|
||||||
|
return nil, errors.New("encoded ring slot header is out of bounds")
|
||||||
|
}
|
||||||
|
length := binary.LittleEndian.Uint32(r.data[start+20 : start+24])
|
||||||
|
if length == 0 || length > r.slotBytes || uint64(start+slotHeaderBytes)+uint64(length) > uint64(len(r.data)) {
|
||||||
|
return nil, errors.New("encoded ring record length is invalid")
|
||||||
|
}
|
||||||
|
payload := append([]byte(nil), r.data[start+slotHeaderBytes:start+slotHeaderBytes+int(length)]...)
|
||||||
|
record := &EncodedRingRecord{
|
||||||
|
Sequence: binary.LittleEndian.Uint64(r.data[start : start+8]),
|
||||||
|
PTS: binary.LittleEndian.Uint64(r.data[start+8 : start+16]),
|
||||||
|
Kind: binary.LittleEndian.Uint32(r.data[start+16 : start+20]),
|
||||||
|
Payload: payload,
|
||||||
|
}
|
||||||
|
record.KeyFrame = record.Kind == 2
|
||||||
|
if record.Sequence == 0 {
|
||||||
|
return nil, errors.New("encoded ring record sequence is invalid")
|
||||||
|
}
|
||||||
|
atomic.StoreUint64((*uint64)(unsafe.Pointer(&r.data[32])), read+1)
|
||||||
|
return record, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *mappedEncodedRing) Close() error {
|
||||||
|
if r.address != 0 {
|
||||||
|
_ = windows.UnmapViewOfFile(r.address)
|
||||||
|
r.address = 0
|
||||||
|
}
|
||||||
|
if r.mapping != 0 {
|
||||||
|
_ = windows.CloseHandle(r.mapping)
|
||||||
|
r.mapping = 0
|
||||||
|
}
|
||||||
|
if r.file != nil {
|
||||||
|
err := r.file.Close()
|
||||||
|
if removeErr := os.Remove(r.path); err == nil && removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) {
|
||||||
|
err = removeErr
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,302 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/binary"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type testEncodedRing struct {
|
||||||
|
records []*EncodedRingRecord
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *testEncodedRing) Pop() (*EncodedRingRecord, error) {
|
||||||
|
if len(r.records) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
record := r.records[0]
|
||||||
|
r.records = r.records[1:]
|
||||||
|
return record, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*testEncodedRing) Close() error { return nil }
|
||||||
|
|
||||||
|
func TestVideoAccessUnitRoundTripThroughFragments(t *testing.T) {
|
||||||
|
payload := bytes.Repeat([]byte{0x5a}, videoDatagramPayloadMax*2+17)
|
||||||
|
unit := VideoAccessUnit{
|
||||||
|
StreamID: 1,
|
||||||
|
Generation: 2,
|
||||||
|
Sequence: 3,
|
||||||
|
FrameID: 4,
|
||||||
|
PTS: 5,
|
||||||
|
DTS: 6,
|
||||||
|
Flags: videoFlagKeyFrame,
|
||||||
|
Payload: payload,
|
||||||
|
}
|
||||||
|
datagrams, err := fragmentVideoAccessUnit(unit)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(datagrams) != 3 {
|
||||||
|
t.Fatalf("got %d datagrams, want 3", len(datagrams))
|
||||||
|
}
|
||||||
|
|
||||||
|
reassembler := newVideoReassembler()
|
||||||
|
var recovered *VideoAccessUnit
|
||||||
|
now := time.Unix(100, 0)
|
||||||
|
for _, datagram := range datagrams {
|
||||||
|
recovered, err = reassembler.push(datagram, now)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if recovered == nil {
|
||||||
|
t.Fatal("frame was not reassembled")
|
||||||
|
}
|
||||||
|
if !bytes.Equal(recovered.Payload, payload) || recovered.Flags&videoFlagKeyFrame == 0 {
|
||||||
|
t.Fatal("reassembled frame does not match source")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVideoReassemblerExpiresIncompleteFrame(t *testing.T) {
|
||||||
|
datagrams, err := fragmentVideoAccessUnit(VideoAccessUnit{
|
||||||
|
StreamID: 1,
|
||||||
|
Generation: 1,
|
||||||
|
Sequence: 1,
|
||||||
|
FrameID: 1,
|
||||||
|
PTS: 1,
|
||||||
|
DTS: 1,
|
||||||
|
Flags: videoFlagKeyFrame,
|
||||||
|
Payload: bytes.Repeat([]byte{1}, videoDatagramPayloadMax+1),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reassembler := newVideoReassembler()
|
||||||
|
now := time.Unix(200, 0)
|
||||||
|
if _, err := reassembler.push(datagrams[0], now); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := reassembler.push(datagrams[0], now.Add(videoReassemblyTimeout+time.Nanosecond)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(reassembler.frames) != 1 {
|
||||||
|
t.Fatalf("expected only the new incomplete assembly, got %d", len(reassembler.frames))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVideoDatagramRejectsInvalidFragmentFlags(t *testing.T) {
|
||||||
|
datagrams, err := fragmentVideoAccessUnit(VideoAccessUnit{
|
||||||
|
StreamID: 1,
|
||||||
|
Generation: 1,
|
||||||
|
Sequence: 1,
|
||||||
|
FrameID: 1,
|
||||||
|
Payload: []byte{1},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
datagrams[0][56] = 0
|
||||||
|
datagrams[0][57] = 0
|
||||||
|
if _, err := decodeVideoDatagram(datagrams[0]); err == nil {
|
||||||
|
t.Fatal("invalid first/last flags were accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVideoDatagramSizeUsesCompleteDatagramLimit(t *testing.T) {
|
||||||
|
if err := validateVideoDatagramSize(make([]byte, videoDatagramMaxBytes)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := validateVideoDatagramSize(make([]byte, videoDatagramMaxBytes+1)); err == nil {
|
||||||
|
t.Fatal("oversized datagram was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareDatagramKeepsOpaquePacketsUnparsed(t *testing.T) {
|
||||||
|
input := []byte("audio-or-control")
|
||||||
|
prepared, kind, err := prepareDatagram(input)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if kind != datagramOpaque || !bytes.Equal(prepared, input) {
|
||||||
|
t.Fatalf("unexpected opaque packet result: kind=%d payload=%q", kind, prepared)
|
||||||
|
}
|
||||||
|
prepared[0] = 'X'
|
||||||
|
if input[0] == 'X' {
|
||||||
|
t.Fatal("prepared datagram aliases the UDP input buffer")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareDatagramRecognizesVideoPackets(t *testing.T) {
|
||||||
|
datagrams, err := fragmentVideoAccessUnit(VideoAccessUnit{
|
||||||
|
StreamID: 1, Generation: 1, Sequence: 1, FrameID: 1, Payload: []byte{1},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, kind, err := prepareDatagram(datagrams[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if kind != datagramVideo {
|
||||||
|
t.Fatalf("got datagram kind %d, want video", kind)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAudioPacketRoundTripIsIndependent(t *testing.T) {
|
||||||
|
input, err := encodeAudioDatagram(AudioPacket{
|
||||||
|
StreamID: 1, Generation: 2, Sequence: 3, PTS: 4, DurationMS: 20,
|
||||||
|
Payload: []byte{0x01, 0x02, 0x03},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
decoded, err := decodeAudioDatagram(input)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if decoded.StreamID != 1 || decoded.Generation != 2 || decoded.Sequence != 3 ||
|
||||||
|
decoded.PTS != 4 || decoded.DurationMS != 20 || !bytes.Equal(decoded.Payload, []byte{1, 2, 3}) {
|
||||||
|
t.Fatalf("decoded audio packet does not match source: %+v", decoded)
|
||||||
|
}
|
||||||
|
prepared, kind, err := prepareDatagram(input)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if kind != datagramOpaque || !bytes.Equal(prepared, input) {
|
||||||
|
t.Fatalf("audio packet was not preserved as opaque transport data")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAudioDatagramRejectsInvalidDurationAndSize(t *testing.T) {
|
||||||
|
if _, err := encodeAudioDatagram(AudioPacket{
|
||||||
|
StreamID: 1, Generation: 1, Sequence: 1, PTS: 1, DurationMS: 15, Payload: []byte{1},
|
||||||
|
}); err == nil {
|
||||||
|
t.Fatal("invalid audio duration was accepted")
|
||||||
|
}
|
||||||
|
if err := validateAudioDatagramSize(make([]byte, audioDatagramMaxBytes+1)); err == nil {
|
||||||
|
t.Fatal("oversized audio datagram was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAudioDatagramRejectsReservedByteAndUnknownFlags(t *testing.T) {
|
||||||
|
encoded, err := encodeAudioDatagram(AudioPacket{
|
||||||
|
StreamID: 1, Generation: 2, Sequence: 3, PTS: 4,
|
||||||
|
DurationMS: 20, Payload: []byte("opus"),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
encoded[5] = 1
|
||||||
|
if _, err := decodeAudioDatagram(encoded); err == nil {
|
||||||
|
t.Fatal("audio reserved byte was accepted")
|
||||||
|
}
|
||||||
|
encoded[5] = 0
|
||||||
|
binary.LittleEndian.PutUint16(encoded[38:40], 2)
|
||||||
|
if _, err := decodeAudioDatagram(encoded); err == nil {
|
||||||
|
t.Fatal("unknown audio flags were accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPumpEncodedRingFragmentsAndMarksKeyFrame(t *testing.T) {
|
||||||
|
payload := bytes.Repeat([]byte{0x42}, videoDatagramPayloadMax+3)
|
||||||
|
ring := &testEncodedRing{records: []*EncodedRingRecord{{
|
||||||
|
Sequence: 9, PTS: 123, KeyFrame: true, Payload: payload,
|
||||||
|
}}}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
var datagrams [][]byte
|
||||||
|
send := func(data []byte) error {
|
||||||
|
datagrams = append(datagrams, append([]byte(nil), data...))
|
||||||
|
if len(datagrams) == 2 {
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
err := pumpEncodedRing(ctx, ring, 7, 8, send)
|
||||||
|
if err != context.Canceled {
|
||||||
|
t.Fatalf("pump returned %v, want context cancellation", err)
|
||||||
|
}
|
||||||
|
if len(datagrams) != 2 {
|
||||||
|
t.Fatalf("got %d datagrams, want 2", len(datagrams))
|
||||||
|
}
|
||||||
|
first, err := decodeVideoDatagram(datagrams[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if first.Flags&videoFlagKeyFrame == 0 || first.PTS != 123 || first.FrameID != 9 {
|
||||||
|
t.Fatalf("ring metadata was not preserved: %+v", first)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPumpEncodedRingRejectsInvalidRecord(t *testing.T) {
|
||||||
|
ring := &testEncodedRing{records: []*EncodedRingRecord{{Sequence: 0, Payload: []byte{1}}}}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
err := pumpEncodedRing(ctx, ring, 1, 1, func([]byte) error { return nil })
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("invalid encoded ring record was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPumpAudioRingEncodesRDA1AndDiscontinuity(t *testing.T) {
|
||||||
|
ring := &testEncodedRing{records: []*EncodedRingRecord{{
|
||||||
|
Sequence: 7,
|
||||||
|
PTS: 400_000,
|
||||||
|
Kind: 2,
|
||||||
|
Payload: []byte("opus"),
|
||||||
|
}}}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
var datagram []byte
|
||||||
|
err := pumpAudioRing(ctx, ring, 11, 12, func(data []byte) error {
|
||||||
|
datagram = append([]byte(nil), data...)
|
||||||
|
cancel()
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != context.Canceled {
|
||||||
|
t.Fatalf("pump returned %v, want context cancellation", err)
|
||||||
|
}
|
||||||
|
packet, err := decodeAudioDatagram(datagram)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if packet.StreamID != 11 || packet.Generation != 12 || packet.Sequence != 7 ||
|
||||||
|
packet.PTS != 400_000 || packet.DurationMS != 20 ||
|
||||||
|
packet.Flags&audioFlagDiscontinuity == 0 || !bytes.Equal(packet.Payload, []byte("opus")) {
|
||||||
|
t.Fatalf("audio ring metadata was not preserved: %+v", packet)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPumpAudioRingRejectsInvalidRecords(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
record *EncodedRingRecord
|
||||||
|
}{
|
||||||
|
{name: "zero sequence", record: &EncodedRingRecord{Payload: []byte{1}}},
|
||||||
|
{name: "empty payload", record: &EncodedRingRecord{Sequence: 1}},
|
||||||
|
{
|
||||||
|
name: "oversized payload",
|
||||||
|
record: &EncodedRingRecord{
|
||||||
|
Sequence: 1,
|
||||||
|
Payload: make([]byte, audioPacketMaxBytes+1),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
ring := &testEncodedRing{records: []*EncodedRingRecord{test.record}}
|
||||||
|
err := pumpAudioRing(
|
||||||
|
context.Background(),
|
||||||
|
ring,
|
||||||
|
1,
|
||||||
|
1,
|
||||||
|
func([]byte) error { return nil },
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("invalid audio ring record was accepted")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,254 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"crypto/hmac"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
agentControlLineMax = 16 * 1024
|
||||||
|
agentPipeTokenEnv = "REMOTEDESK_AGENT_PIPE_TOKEN"
|
||||||
|
agentPipeAuthDomain = "RemoteDesk Windows Agent pipe auth v1\x00"
|
||||||
|
)
|
||||||
|
|
||||||
|
type mediaRingDescriptor struct {
|
||||||
|
SchemaVersion uint16 `json:"schema_version"`
|
||||||
|
SessionID string `json:"session_id"`
|
||||||
|
Generation uint64 `json:"generation"`
|
||||||
|
OwnerPID uint32 `json:"owner_pid"`
|
||||||
|
RingPath string `json:"ring_path"`
|
||||||
|
SlotCount uint32 `json:"slot_count"`
|
||||||
|
SlotBytes uint32 `json:"slot_bytes"`
|
||||||
|
VideoTransport string `json:"video_transport"`
|
||||||
|
AudioTransport string `json:"audio_transport"`
|
||||||
|
AudioRing *audioRingDescriptor `json:"audio_ring,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type audioRingDescriptor struct {
|
||||||
|
SchemaVersion uint16 `json:"schema_version"`
|
||||||
|
SessionID string `json:"session_id"`
|
||||||
|
Generation uint64 `json:"generation"`
|
||||||
|
OwnerPID uint32 `json:"owner_pid"`
|
||||||
|
RingPath string `json:"ring_path"`
|
||||||
|
SlotCount uint32 `json:"slot_count"`
|
||||||
|
SlotBytes uint32 `json:"slot_bytes"`
|
||||||
|
AudioTransport string `json:"audio_transport"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type encodedDesktopOpened struct {
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Error string `json:"error"`
|
||||||
|
VideoCodec string `json:"video_codec"`
|
||||||
|
MediaTransport string `json:"media_transport"`
|
||||||
|
Ring mediaRingDescriptor `json:"ring"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type agentHello struct {
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Authenticated bool `json:"authenticated"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type agentAuthChallenge struct {
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
ProtocolVersion uint8 `json:"protocol_version"`
|
||||||
|
Nonce string `json:"nonce"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func openWindowsEncodedSession(address, sessionID string, fps uint8, allowSoftwareFallback bool) (net.Conn, mediaRingDescriptor, error) {
|
||||||
|
if err := validateAgentControlAddress(address); err != nil {
|
||||||
|
return nil, mediaRingDescriptor{}, err
|
||||||
|
}
|
||||||
|
if !validSessionID(sessionID) || fps == 0 || fps > 120 {
|
||||||
|
return nil, mediaRingDescriptor{}, errors.New("Windows Agent encoded session parameters are invalid")
|
||||||
|
}
|
||||||
|
isPipe := isWindowsAgentPipe(address)
|
||||||
|
connection, err := dialWindowsAgentControl(address, 2*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mediaRingDescriptor{}, fmt.Errorf("connect Windows Agent control: %w", err)
|
||||||
|
}
|
||||||
|
closeOnError := true
|
||||||
|
defer func() {
|
||||||
|
if closeOnError {
|
||||||
|
_ = connection.Close()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
_ = connection.SetDeadline(time.Now().Add(10 * time.Second))
|
||||||
|
reader := bufio.NewReaderSize(connection, agentControlLineMax+1)
|
||||||
|
if isPipe {
|
||||||
|
if err := authenticateWindowsAgentPipe(reader, connection); err != nil {
|
||||||
|
return nil, mediaRingDescriptor{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var hello agentHello
|
||||||
|
if err := readBoundedAgentJSON(reader, &hello); err != nil || hello.Kind != "windows_agent_hello" || (isPipe && !hello.Authenticated) {
|
||||||
|
return nil, mediaRingDescriptor{}, errors.New("Windows Agent control hello is invalid")
|
||||||
|
}
|
||||||
|
request := struct {
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
SessionID string `json:"session_id"`
|
||||||
|
CaptureMode string `json:"capture_mode"`
|
||||||
|
VideoCodec string `json:"video_codec"`
|
||||||
|
FramesPerSecond uint8 `json:"frames_per_second"`
|
||||||
|
AllowSoftwareFallback bool `json:"allow_software_fallback"`
|
||||||
|
}{
|
||||||
|
Kind: "open_encoded_desktop", SessionID: sessionID,
|
||||||
|
CaptureMode: "compatibility", VideoCodec: "h264", FramesPerSecond: fps,
|
||||||
|
AllowSoftwareFallback: allowSoftwareFallback,
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(request)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mediaRingDescriptor{}, err
|
||||||
|
}
|
||||||
|
encoded = append(encoded, '\n')
|
||||||
|
if _, err := connection.Write(encoded); err != nil {
|
||||||
|
return nil, mediaRingDescriptor{}, fmt.Errorf("open Windows encoded desktop: %w", err)
|
||||||
|
}
|
||||||
|
var opened encodedDesktopOpened
|
||||||
|
if err := readBoundedAgentJSON(reader, &opened); err != nil {
|
||||||
|
return nil, mediaRingDescriptor{}, err
|
||||||
|
}
|
||||||
|
if opened.Kind != "encoded_desktop_opened" || opened.VideoCodec != "h264" ||
|
||||||
|
opened.MediaTransport != "shared_memory_encoded_access_units" {
|
||||||
|
return nil, mediaRingDescriptor{}, fmt.Errorf("Windows encoded desktop unavailable: %s", opened.Error)
|
||||||
|
}
|
||||||
|
if err := validateMediaRingDescriptor(opened.Ring, sessionID); err != nil {
|
||||||
|
return nil, mediaRingDescriptor{}, err
|
||||||
|
}
|
||||||
|
_ = connection.SetDeadline(time.Time{})
|
||||||
|
closeOnError = false
|
||||||
|
return connection, opened.Ring, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func dialWindowsAgentControl(address string, timeout time.Duration) (net.Conn, error) {
|
||||||
|
if isWindowsAgentPipe(address) {
|
||||||
|
return dialWindowsAgentPipe(address, timeout)
|
||||||
|
}
|
||||||
|
return net.DialTimeout("tcp", address, timeout)
|
||||||
|
}
|
||||||
|
|
||||||
|
func authenticateWindowsAgentPipe(reader *bufio.Reader, connection net.Conn) error {
|
||||||
|
var challenge agentAuthChallenge
|
||||||
|
if err := readBoundedAgentJSON(reader, &challenge); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if challenge.Kind != "agent_auth_challenge" || challenge.ProtocolVersion != 1 {
|
||||||
|
return errors.New("Windows Agent Pipe challenge is invalid")
|
||||||
|
}
|
||||||
|
nonce, err := base64.RawURLEncoding.DecodeString(challenge.Nonce)
|
||||||
|
if err != nil || len(nonce) != 32 {
|
||||||
|
return errors.New("Windows Agent Pipe nonce is invalid")
|
||||||
|
}
|
||||||
|
encodedToken, ok := os.LookupEnv(agentPipeTokenEnv)
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("%s is required for authenticated Windows Agent Pipe", agentPipeTokenEnv)
|
||||||
|
}
|
||||||
|
_ = os.Unsetenv(agentPipeTokenEnv)
|
||||||
|
key, err := base64.RawURLEncoding.DecodeString(encodedToken)
|
||||||
|
if err != nil || len(key) != 32 {
|
||||||
|
return errors.New("Windows Agent Pipe token must be 32-byte base64url")
|
||||||
|
}
|
||||||
|
mac := windowsAgentPipeMAC(key, nonce)
|
||||||
|
request := struct {
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
ProtocolVersion uint8 `json:"protocol_version"`
|
||||||
|
MAC string `json:"mac"`
|
||||||
|
}{
|
||||||
|
Kind: "agent_auth",
|
||||||
|
ProtocolVersion: 1,
|
||||||
|
MAC: base64.RawURLEncoding.EncodeToString(mac),
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(request)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
encoded = append(encoded, '\n')
|
||||||
|
if _, err := connection.Write(encoded); err != nil {
|
||||||
|
return fmt.Errorf("authenticate Windows Agent Pipe: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func windowsAgentPipeMAC(key, nonce []byte) []byte {
|
||||||
|
mac := hmac.New(sha256.New, key)
|
||||||
|
_, _ = mac.Write([]byte(agentPipeAuthDomain))
|
||||||
|
_, _ = mac.Write(nonce)
|
||||||
|
return mac.Sum(nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func readBoundedAgentJSON(reader *bufio.Reader, output any) error {
|
||||||
|
line, err := reader.ReadSlice('\n')
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("read Windows Agent control: %w", err)
|
||||||
|
}
|
||||||
|
if len(line) == 0 || len(line) > agentControlLineMax || line[len(line)-1] != '\n' {
|
||||||
|
return errors.New("Windows Agent control response exceeds its bound")
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(line, output); err != nil {
|
||||||
|
return errors.New("Windows Agent control response is invalid JSON")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateAgentControlAddress(address string) error {
|
||||||
|
if isWindowsAgentPipe(address) {
|
||||||
|
if len(address) > 256 || !strings.HasPrefix(strings.ToLower(address), `\\.\pipe\remotedesk\`) || strings.Contains(address, "..") {
|
||||||
|
return errors.New("Windows Agent Pipe name is outside the RemoteDesk namespace")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
host, port, err := net.SplitHostPort(address)
|
||||||
|
if err != nil || port == "" {
|
||||||
|
return errors.New("Windows Agent control must be an IP socket address")
|
||||||
|
}
|
||||||
|
ip := net.ParseIP(host)
|
||||||
|
if ip == nil || !ip.IsLoopback() {
|
||||||
|
return errors.New("Windows Agent control must use a loopback IP")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func isWindowsAgentPipe(address string) bool {
|
||||||
|
return strings.HasPrefix(strings.ToLower(address), `\\.\pipe\`)
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateMediaRingDescriptor(descriptor mediaRingDescriptor, sessionID string) error {
|
||||||
|
if descriptor.SchemaVersion != 1 || descriptor.SessionID != sessionID || descriptor.Generation == 0 || descriptor.OwnerPID == 0 ||
|
||||||
|
descriptor.VideoTransport != "shared_memory_encoded_access_units" || descriptor.AudioTransport != "separate_transport" ||
|
||||||
|
!filepath.IsAbs(descriptor.RingPath) || descriptor.SlotCount < 2 || descriptor.SlotCount > maxRingSlots ||
|
||||||
|
descriptor.SlotCount&(descriptor.SlotCount-1) != 0 || descriptor.SlotBytes == 0 || descriptor.SlotBytes > maxRingSlotBytes {
|
||||||
|
return errors.New("Windows Agent media ring descriptor is invalid")
|
||||||
|
}
|
||||||
|
if descriptor.AudioRing != nil {
|
||||||
|
audio := descriptor.AudioRing
|
||||||
|
if audio.SchemaVersion != 1 || audio.SessionID != sessionID || audio.Generation != descriptor.Generation ||
|
||||||
|
audio.OwnerPID != descriptor.OwnerPID || audio.AudioTransport != "shared_memory_opus_packets" ||
|
||||||
|
!filepath.IsAbs(audio.RingPath) || audio.SlotCount < 2 || audio.SlotCount > maxRingSlots ||
|
||||||
|
audio.SlotCount&(audio.SlotCount-1) != 0 || audio.SlotBytes == 0 || audio.SlotBytes > maxRingSlotBytes {
|
||||||
|
return errors.New("Windows Agent audio ring descriptor is invalid")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validSessionID(value string) bool {
|
||||||
|
if len(value) == 0 || len(value) > 128 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, character := range []byte(value) {
|
||||||
|
if !((character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z') ||
|
||||||
|
(character >= '0' && character <= '9') || character == '-' || character == '_') {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"net"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestOpenWindowsEncodedSessionBindsDescriptorToRequest(t *testing.T) {
|
||||||
|
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer listener.Close()
|
||||||
|
serverError := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
connection, err := listener.Accept()
|
||||||
|
if err != nil {
|
||||||
|
serverError <- err
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer connection.Close()
|
||||||
|
if _, err := connection.Write([]byte("{\"kind\":\"windows_agent_hello\"}\n")); err != nil {
|
||||||
|
serverError <- err
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var request struct {
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
SessionID string `json:"session_id"`
|
||||||
|
FramesPerSecond uint8 `json:"frames_per_second"`
|
||||||
|
AllowSoftwareFallback bool `json:"allow_software_fallback"`
|
||||||
|
}
|
||||||
|
if err := readBoundedAgentJSON(bufio.NewReader(connection), &request); err != nil {
|
||||||
|
serverError <- err
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if request.Kind != "open_encoded_desktop" || request.SessionID != "session-1" || request.FramesPerSecond != 60 || !request.AllowSoftwareFallback {
|
||||||
|
serverError <- &testControlError{"encoded desktop request mismatch"}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
response := encodedDesktopOpened{
|
||||||
|
Kind: "encoded_desktop_opened", VideoCodec: "h264",
|
||||||
|
MediaTransport: "shared_memory_encoded_access_units",
|
||||||
|
Ring: mediaRingDescriptor{
|
||||||
|
SchemaVersion: 1, SessionID: "session-1", Generation: 7, OwnerPID: 42,
|
||||||
|
RingPath: filepath.Join(t.TempDir(), "media.ring"), SlotCount: 16,
|
||||||
|
SlotBytes: 8 * 1024 * 1024, VideoTransport: "shared_memory_encoded_access_units",
|
||||||
|
AudioTransport: "separate_transport",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(response)
|
||||||
|
if err == nil {
|
||||||
|
encoded = append(encoded, '\n')
|
||||||
|
_, err = connection.Write(encoded)
|
||||||
|
}
|
||||||
|
serverError <- err
|
||||||
|
}()
|
||||||
|
connection, descriptor, err := openWindowsEncodedSession(listener.Addr().String(), "session-1", 60, true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer connection.Close()
|
||||||
|
if descriptor.Generation != 7 || descriptor.SessionID != "session-1" {
|
||||||
|
t.Fatalf("unexpected descriptor: %+v", descriptor)
|
||||||
|
}
|
||||||
|
if err := <-serverError; err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWindowsAgentControlRejectsRemoteAndMismatchedDescriptor(t *testing.T) {
|
||||||
|
if validateAgentControlAddress("192.0.2.10:39501") == nil {
|
||||||
|
t.Fatal("remote Windows Agent control address was accepted")
|
||||||
|
}
|
||||||
|
descriptor := mediaRingDescriptor{
|
||||||
|
SchemaVersion: 1, SessionID: "other", Generation: 1, OwnerPID: 1,
|
||||||
|
RingPath: filepath.Join(t.TempDir(), "media.ring"), SlotCount: 16,
|
||||||
|
SlotBytes: 1024, VideoTransport: "shared_memory_encoded_access_units",
|
||||||
|
AudioTransport: "separate_transport",
|
||||||
|
}
|
||||||
|
if validateMediaRingDescriptor(descriptor, "session-1") == nil {
|
||||||
|
t.Fatal("descriptor for another session was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWindowsAgentPipeAuthenticationMatchesRustWireVector(t *testing.T) {
|
||||||
|
key := make([]byte, 32)
|
||||||
|
nonce := make([]byte, 32)
|
||||||
|
for index := range key {
|
||||||
|
key[index] = byte(index)
|
||||||
|
nonce[index] = byte(index + 32)
|
||||||
|
}
|
||||||
|
actual := base64.RawURLEncoding.EncodeToString(windowsAgentPipeMAC(key, nonce))
|
||||||
|
const expected = "KinBDOBvVhz0eRv37iahq8BO0z8QdiI0Qnx7HZmcxIk"
|
||||||
|
if actual != expected {
|
||||||
|
t.Fatalf("unexpected Agent Pipe MAC: %s", actual)
|
||||||
|
}
|
||||||
|
if validateAgentControlAddress(`\\.\pipe\RemoteDesk\windows-agent`) != nil {
|
||||||
|
t.Fatal("RemoteDesk Agent Pipe was rejected")
|
||||||
|
}
|
||||||
|
if validateAgentControlAddress(`\\.\pipe\Other\windows-agent`) == nil {
|
||||||
|
t.Fatal("Pipe outside the RemoteDesk namespace was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type testControlError struct{ message string }
|
||||||
|
|
||||||
|
func (e *testControlError) Error() string { return e.message }
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
//go:build !windows
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"net"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func dialWindowsAgentPipe(_ string, _ time.Duration) (net.Conn, error) {
|
||||||
|
return nil, errors.New("Windows Agent Named Pipe requires Windows")
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
//go:build windows
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/Microsoft/go-winio"
|
||||||
|
)
|
||||||
|
|
||||||
|
func dialWindowsAgentPipe(name string, timeout time.Duration) (net.Conn, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||||
|
defer cancel()
|
||||||
|
return winio.DialPipeContext(ctx, name)
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user