From e728f38d798f2c46cec2a4e07834301c16ecb084 Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Wed, 23 Sep 2026 03:04:42 +0500 Subject: [PATCH 1/2] perf(desktop): recycle encoder input buffer, SIMD decode output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The encoder reused to allocate a fresh ~3 MiB I420 buffer per frame; it now keeps a YUVBuffer and rewrites it via read_bgra8 when dimensions match (~190 MB/s of alloc churn removed at 60 fps). - Decode emitted I420→RGB8 then expanded to BGRA with a scalar loop. write_rgba8 does the conversion in one SIMD pass (AVX2 on x86-64) and the R↔B swap for the RawFrame BGRA contract vectorizes in place. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- CHANGELOG.md | 10 ++++++ crates/rds-desktop/src/codec/openh264.rs | 39 +++++++++++++++--------- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4637edd..2121b7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## [Unreleased] +- Desktop capture/encode throughput: + - X11 capture uses MIT-SHM (`CreateSegment` fd-passing, server ≥1.2) + when available — the pixmap lands in a shared segment instead of an + ~8 MiB serialized `GetImage` reply per 1080p frame; plain `GetImage` + remains the fallback for remote/older servers. + - The encoder recycles its I420 input buffer across frames (~3 MiB + per frame at 1080p — ~190 MB/s of alloc churn at 60 fps removed). + - Decode output goes I420→RGBA in one SIMD pass (`write_rgba8`, + AVX2 on x86-64) plus an in-place R↔B swap — replacing an RGB8 + scratch buffer plus a scalar expand. - Desktop encode path: - OpenH264 now runs `ScreenContentRealTime` — the correct usage type for desktop content (text/sharp edges, not camera footage); adaptive diff --git a/crates/rds-desktop/src/codec/openh264.rs b/crates/rds-desktop/src/codec/openh264.rs index b4fd927..cbd208d 100644 --- a/crates/rds-desktop/src/codec/openh264.rs +++ b/crates/rds-desktop/src/codec/openh264.rs @@ -27,6 +27,10 @@ pub struct H264Encoder { pending_bitrate: Option, fps: f32, want_idr: bool, + /// Recycled I420 input buffer — a 1080p frame is ~3 MiB, so a fresh + /// allocation per frame at 60 fps is ~190 MB/s of pure alloc churn. + /// Rebuilt only when frame dimensions change. + yuv_buf: Option, } impl H264Encoder { @@ -38,6 +42,7 @@ impl H264Encoder { pending_bitrate: None, fps, want_idr: true, + yuv_buf: None, }) } @@ -84,11 +89,24 @@ impl Encoder for H264Encoder { self.inner.force_intra_frame(); self.want_idr = false; } - let yuv = bgra_to_i420(frame); + let (w, h) = (frame.width as usize, frame.height as usize); + let stride = frame.stride as usize; + let yuv = if stride.is_multiple_of(4) && frame.data.len() >= stride * h { + match self.yuv_buf.take() { + Some(mut buf) if buf.dimensions() == (w, h) => { + buf.read_bgra8(StridedBgra(frame)); + buf + } + _ => YUVBuffer::from_bgra8_source(StridedBgra(frame)), + } + } else { + bgra_to_i420_scalar(frame) + }; let stream = self .inner .encode(&yuv) .map_err(|e| DesktopError::Encode(e.to_string()))?; + self.yuv_buf = Some(yuv); // The flag the writer's collapse trusts must be the truth on // the wire, not a schedule assumption: OpenH264 decides when // forced and periodic IDRs actually land, so read the NALs. @@ -171,20 +189,13 @@ impl Decoder for H264Decoder { return Ok(None); }; let (w, h) = yuv.dimensions(); - let mut rgb = vec![0u8; w * h * 3]; - yuv.write_rgb8(&mut rgb); - // Expand RGB to BGRA for a uniform RawFrame layout. + // I420→RGBA in one SIMD pass (AVX2 on x86-64), then swap R↔B + // in place for the RawFrame BGRA contract — the swap vectorizes + // trivially and avoids a separate 3-byte-per-pixel scratch. let mut bgra = vec![0u8; w * h * 4]; - for (dst, src) in bgra - .as_chunks_mut::<4>() - .0 - .iter_mut() - .zip(rgb.as_chunks::<3>().0) - { - dst[0] = src[2]; - dst[1] = src[1]; - dst[2] = src[0]; - dst[3] = 0xFF; + yuv.write_rgba8(&mut bgra); + for px in bgra.as_chunks_mut::<4>().0.iter_mut() { + px.swap(0, 2); } Ok(Some(RawFrame { width: w as u32, From ac5ed675e64b48383480d88d4210c8ca216239fc Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Wed, 23 Sep 2026 03:04:51 +0500 Subject: [PATCH 2/2] perf(desktop): MIT-SHM shared-memory X11 capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GetImage serializes every frame through the X socket — ~8 MiB per 1080p frame — plus a reply round-trip. With MIT-SHM >= 1.2 the server writes the pixmap into a shared segment (CreateSegment fd passing, mapped via memmap2) and replies with a completion event, so capture is one memcpy off mapped memory. The backend probes SHM at connect and falls back to plain GetImage on older servers, remote displays (ssh -X), or any runtime failure. Verified live against Xvfb (shm segment present, repeated captures correct). Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- Cargo.lock | 10 ++ crates/rds-desktop/Cargo.toml | 3 +- crates/rds-desktop/src/capture/x11.rs | 139 ++++++++++++++++++++++++-- 3 files changed, 145 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8954c69..3bdbb35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1858,6 +1858,15 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2902,6 +2911,7 @@ name = "rds-desktop" version = "0.1.0" dependencies = [ "bytes", + "memmap2", "noq", "openh264", "rds-bench", diff --git a/crates/rds-desktop/Cargo.toml b/crates/rds-desktop/Cargo.toml index 3ceab79..1c5f24f 100644 --- a/crates/rds-desktop/Cargo.toml +++ b/crates/rds-desktop/Cargo.toml @@ -8,10 +8,11 @@ repository.workspace = true [features] default = [] -x11 = ["dep:x11rb", "dep:openh264"] +x11 = ["dep:x11rb", "dep:openh264", "dep:memmap2"] [dependencies] bytes.workspace = true +memmap2 = { version = "0.9", optional = true } openh264 = { version = "0.9", optional = true } rds-core.workspace = true rds-net.workspace = true diff --git a/crates/rds-desktop/src/capture/x11.rs b/crates/rds-desktop/src/capture/x11.rs index 90acd0c..b59a8be 100644 --- a/crates/rds-desktop/src/capture/x11.rs +++ b/crates/rds-desktop/src/capture/x11.rs @@ -1,18 +1,43 @@ -//! X11 capture via `x11rb`: `GetImage` polling for frames. +//! X11 capture via `x11rb`: MIT-SHM shared-memory `GetImage` where the +//! server supports it, plain `GetImage` polling as the portable +//! fallback. //! -//! This is the portable baseline backend. The MIT-SHM zero-copy path, -//! `ext-image-copy-capture` and the DRM/KMS tap are scheduled behind the -//! same `Capturer` trait; GetImage polling is what every X server already -//! supports. Input injection lives in `crate::input::x11`. +//! MIT-SHM removes the dominant capture cost: the pixmap is written +//! into a shared segment the client mmaps, so a 1080p frame costs one +//! completion event plus a memcpy instead of ~8 MiB serialized through +//! the X socket. `CreateSegment` (SHM ≥ 1.2 fd passing) is preferred — +//! the server allocates and owns nothing persists on the client side. +//! `ext-image-copy-capture` and the DRM/KMS tap remain scheduled behind +//! the same `Capturer` trait. Input injection lives in +//! `crate::input::x11`. +//! +//! `mmap` on the server-provided segment is the one unsafe call in this +//! module (workspace convention: FFI paths allow `unsafe_code` locally). +#![allow(unsafe_code)] + +use std::os::fd::OwnedFd; use bytes::Bytes; +use memmap2::{MmapMut, MmapOptions}; use rds_core::{DesktopCaps, DisplayInfo}; -use x11rb::connection::Connection as _; +use x11rb::connection::{Connection as _, RequestConnection as _}; +use x11rb::protocol::shm::{self, ConnectionExt as _, Seg}; use x11rb::protocol::xproto::{ConnectionExt, GetImageReply, ImageFormat}; use x11rb::rust_connection::RustConnection; use crate::{Capturer, DesktopError, RawFrame}; +/// MIT-SHM segment the server fills in place — the reply is a +/// completion event, not a serialized pixmap. +struct ShmPath { + seg: Seg, + map: MmapMut, + len: usize, + /// The mapping stays valid without it, but keeping the fd makes + /// ownership/lifetime explicit. + _fd: OwnedFd, +} + /// Polls one X11 screen (root window) into `RawFrame`s. pub struct X11Capturer { conn: RustConnection, @@ -20,6 +45,7 @@ pub struct X11Capturer { width: u16, height: u16, screen: usize, + shm: Option, } impl X11Capturer { @@ -35,18 +61,84 @@ impl X11Capturer { setup.roots[idx].height_in_pixels, ); let _ = default_screen; + let shm = Self::try_shm(&conn, width, height); Ok(Self { conn, root, width, height, screen: idx, + shm, + }) + } + + /// MIT-SHM ≥ 1.2 (fd passing): the server creates the segment and + /// hands back an fd we mmap. `None` on older servers, remote + /// displays, or any setup failure — the caller falls back to plain + /// `GetImage`. + fn try_shm(conn: &RustConnection, width: u16, height: u16) -> Option { + conn.extension_information(shm::X11_EXTENSION_NAME).ok()??; + let ver = conn.shm_query_version().ok()?.reply().ok()?; + if (ver.major_version, ver.minor_version) < (1, 2) { + return None; + } + let len = usize::from(width) * usize::from(height) * 4; + let seg = conn.generate_id().ok()?; + let reply = conn + .shm_create_segment(seg, len as u32, false) + .ok()? + .reply() + .ok()?; + let fd: OwnedFd = reply.shm_fd; + // SAFETY: `fd` is a fresh server-created segment of exactly + // `len` bytes; nothing else mutates its size. + let map = unsafe { MmapOptions::new().len(len).map_mut(&fd) }.ok()?; + Some(ShmPath { + seg, + map, + len, + _fd: fd, }) } } impl Capturer for X11Capturer { fn capture(&mut self) -> Result { + if let Some(shm) = &self.shm { + // `reply()` waits for the ShmCompletion event the server + // sends after filling the segment. + let filled = self + .conn + .shm_get_image( + self.root, + 0, + 0, + self.width, + self.height, + !0, + ImageFormat::Z_PIXMAP.into(), + shm.seg, + 0, + ) + .map_err(|e| DesktopError::Capture(e.to_string())) + .and_then(|c| c.reply().map_err(|e| DesktopError::Capture(e.to_string()))); + match filled { + Ok(_) => { + return Ok(RawFrame { + width: u32::from(self.width), + height: u32::from(self.height), + stride: u32::from(self.width) * 4, + data: Bytes::copy_from_slice(&shm.map[..shm.len]), + }); + } + Err(e) => { + tracing::warn!("MIT-SHM capture failed ({e}); falling back to GetImage"); + } + } + } + // Reached only when the shm attempt failed at runtime — stop + // paying for attempts that can no longer succeed. + self.shm = None; let reply: GetImageReply = self .conn .get_image( @@ -79,6 +171,14 @@ impl Capturer for X11Capturer { } } +impl Drop for X11Capturer { + fn drop(&mut self) { + if let Some(shm) = self.shm.take() { + let _ = self.conn.shm_detach(shm.seg); + } + } +} + /// Displays visible over X11. pub fn capabilities() -> Result { match X11Capturer::new(0) { @@ -89,3 +189,30 @@ pub fn capabilities() -> Result { Err(e) => Err(e), } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Real capture needs an X server — skipped silently when `$DISPLAY` + /// is unset (CI has none). On a local server (`:N`) MIT-SHM must be + /// present; remote `host:N` displays legitimately lack it. + #[test] + fn capture_roundtrip() { + let Some(display) = std::env::var_os("DISPLAY") else { + return; + }; + let local = display.to_string_lossy().starts_with(':'); + let Ok(mut cap) = X11Capturer::new(0) else { + return; + }; + if local { + assert!(cap.shm.is_some(), "local X server without MIT-SHM 1.2?"); + } + for _ in 0..3 { + let frame = cap.capture().unwrap(); + assert_eq!(frame.data.len(), (frame.width * frame.height * 4) as usize); + assert_eq!(frame.stride, frame.width * 4); + } + } +}