diff --git a/CHANGELOG.md b/CHANGELOG.md index dc61e4b..fbdf585 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## [Unreleased] +- Transport tuning and relay redundancy: + - BBRv3 congestion control on both backends (paced, bufferbloat- + resistant) instead of the loss-based Cubic default — better + latency under load for interactive desktop and bulk sync. + - 4 MiB stream receive window / 32 MiB connection send window + (upstream defaults target ~100 Mbps × 100 ms): large keyframes and + sync chunk streams no longer stall on high-BDP links. + - `--relay` is repeatable on `rds` and `rds-agent`; endpoints probe + all configured relays, home on the fastest and fail over + automatically — the iroh-recommended ≥2-relay production topology. - Relay TLS and lifecycle polish: - `rds-server`/`rds-relay` gain native TLS on the relay listener: `--tls-cert/--tls-key` for PEM files (rustls `ring` provider), or diff --git a/crates/rds-agent/src/main.rs b/crates/rds-agent/src/main.rs index 91479b6..e172ddc 100644 --- a/crates/rds-agent/src/main.rs +++ b/crates/rds-agent/src/main.rs @@ -16,9 +16,10 @@ struct Cli { /// Path to the endpoint secret key (created if missing). #[arg(long)] key_file: Option, - /// Custom relay URL; default is the n0 public relays. + /// Custom relay URL; default is the n0 public relays. Repeatable — + /// ≥2 relays give automatic client-side failover. #[arg(long)] - relay: Option, + relay: Vec, /// Transport backend: `iroh` (default) or `noq` (with the /// `transport-noq` feature). #[arg(long, default_value = "iroh")] @@ -111,9 +112,7 @@ async fn main() -> anyhow::Result<()> { backend, ..Default::default() }; - if let Some(url) = &cli.relay { - config = config.with_relay(url)?; - } + config = config.with_relays(&cli.relay)?; let endpoint = bind_endpoint(config).await?; endpoint.online().await; diff --git a/crates/rds-agent/tests/e2e.rs b/crates/rds-agent/tests/e2e.rs index eeea88d..d850340 100644 --- a/crates/rds-agent/tests/e2e.rs +++ b/crates/rds-agent/tests/e2e.rs @@ -131,7 +131,7 @@ async fn unauthorized_peer_is_rejected() { async fn direct_connection_without_relay() { // LAN-style path: no relay, direct UDP addresses in the ticket. let agent_ep = bind_endpoint(EndpointConfig { - relay: None, + relays: Vec::new(), ..Default::default() }) .await @@ -554,7 +554,7 @@ async fn sync_unconfigured_is_refused() { async fn direct_connection_noq_backend() { let config = || EndpointConfig { backend: rds_net::Backend::Noq, - relay: None, + relays: Vec::new(), ..Default::default() }; let agent_ep = bind_endpoint(config()).await.unwrap(); diff --git a/crates/rds-cli/src/main.rs b/crates/rds-cli/src/main.rs index 60bb250..6398f14 100644 --- a/crates/rds-cli/src/main.rs +++ b/crates/rds-cli/src/main.rs @@ -12,9 +12,10 @@ struct Cli { /// Path to the endpoint secret key. #[arg(long, global = true)] key_file: Option, - /// Custom relay URL; default is the n0 public relays. + /// Custom relay URL; default is the n0 public relays. Repeatable — + /// ≥2 relays give automatic client-side failover. #[arg(long, global = true)] - relay: Option, + relay: Vec, /// Transport backend: `iroh` (default) or `noq` (with the /// `transport-noq` feature). #[arg(long, global = true, default_value = "iroh")] @@ -127,9 +128,7 @@ async fn main() -> anyhow::Result<()> { backend, ..Default::default() }; - if let Some(url) = &cli.relay { - config = config.with_relay(url)?; - } + config = config.with_relays(&cli.relay)?; let endpoint = bind_endpoint(config).await?; let directory = cli.server.map(rds_discovery::client::Client::new); let grant = cli diff --git a/crates/rds-net/Cargo.toml b/crates/rds-net/Cargo.toml index 18b5b74..3a62971 100644 --- a/crates/rds-net/Cargo.toml +++ b/crates/rds-net/Cargo.toml @@ -14,7 +14,9 @@ data-encoding.workspace = true ed25519-dalek = { workspace = true, features = ["pkcs8"] } iroh.workspace = true noq = { workspace = true, optional = true } -noq-proto = { workspace = true, optional = true } +# Non-optional: iroh runs on noq internally, so it is in the tree either +# way — needed for transport tuning (BBRv3, windows) on the iroh backend. +noq-proto.workspace = true rand = { workspace = true, optional = true } postcard = { workspace = true, features = ["alloc", "use-std"] } rds-core.workspace = true @@ -37,7 +39,6 @@ metrics = [] ## Owned transport backend on noq (WS1). Off by default until C1 parity. transport-noq = [ "dep:noq", - "dep:noq-proto", "dep:tokio-stream", "dep:blake3", "dep:rand", diff --git a/crates/rds-net/src/backends/iroh.rs b/crates/rds-net/src/backends/iroh.rs index b25d5fa..7eac355 100644 --- a/crates/rds-net/src/backends/iroh.rs +++ b/crates/rds-net/src/backends/iroh.rs @@ -20,23 +20,35 @@ use crate::EndpointConfig; /// lookup — so a private deployment does not publish to third-party DNS. /// Without one, `presets::N0` gives the public relays plus DNS/Pkarr lookup. pub async fn bind_endpoint(config: EndpointConfig) -> anyhow::Result { - let mut builder = match (&config.relay, config.discovery) { - (Some(url), _) => Endpoint::builder(iroh::endpoint::presets::Minimal) - .relay_mode(RelayMode::Custom(RelayMap::from_iter([url.clone()]))), - (None, true) => Endpoint::builder(iroh::endpoint::presets::N0), + let mut builder = match (config.relays.is_empty(), config.discovery) { + (false, _) => Endpoint::builder(iroh::endpoint::presets::Minimal).relay_mode( + RelayMode::Custom(RelayMap::from_iter(config.relays.clone())), + ), + (true, true) => Endpoint::builder(iroh::endpoint::presets::N0), // No relay, no lookup: Minimal binds a plain QUIC socket. - (None, false) => Endpoint::builder(iroh::endpoint::presets::Minimal), + (true, false) => Endpoint::builder(iroh::endpoint::presets::Minimal), }; if let Some(key) = config.secret_key { builder = builder.secret_key(key); } + // Tuning on top of iroh's multipath-aware defaults: + // - BBRv3: paced, bufferbloat-resistant — the low-latency choice for + // interactive desktop + bulk sync over real WAN paths (upstream + // default is loss-based Cubic). + // - 4 MiB stream receive window: upstream tunes for ~100 Mbps x + // 100 ms; a larger per-stream window keeps a big keyframe or sync + // chunk stream from stalling on high-BDP links. + // - 32 MiB connection send window keeps several bulk streams busy. + let mut transport = iroh::endpoint::QuicTransportConfig::builder() + .congestion_controller_factory(std::sync::Arc::new( + noq_proto::congestion::Bbr3Config::default(), + )) + .stream_receive_window(noq_proto::VarInt::from_u32(4 * 1024 * 1024)) + .send_window(32 * 1024 * 1024); if let Some(max_paths) = config.max_multipath_paths { - builder = builder.transport_config( - iroh::endpoint::QuicTransportConfig::builder() - .max_concurrent_multipath_paths(max_paths) - .build(), - ); + transport = transport.max_concurrent_multipath_paths(max_paths); } + builder = builder.transport_config(transport.build()); // iroh manages its own sockets; a single bind address is all it // accepts. Multi-interface binding is a `noq`-backend capability. if let Some(addr) = config.bind_addrs.first() { diff --git a/crates/rds-net/src/backends/noq/mod.rs b/crates/rds-net/src/backends/noq/mod.rs index 055c09b..239bcd2 100644 --- a/crates/rds-net/src/backends/noq/mod.rs +++ b/crates/rds-net/src/backends/noq/mod.rs @@ -62,6 +62,12 @@ fn transport_config(max_multipath_paths: Option) -> Arc, - /// Custom relay URL. `None` uses the backend's default relay set - /// (n0 public relays for iroh). - pub relay: Option, + /// Custom relay URLs. Empty uses the backend's default relay set + /// (n0 public relays for iroh). Multiple relays give the client + /// automatic failover — production deployments should run ≥2. + pub relays: Vec, /// Publish/resolve addresses via the backend's lookup services /// (iroh: n0 DNS + pkarr). `false` binds the `Minimal` preset — /// dialing uses exactly the `EndpointAddr` given, which is what @@ -116,7 +117,7 @@ impl Default for EndpointConfig { backend: Backend::default(), secret_key: None, bind_addrs: Vec::new(), - relay: None, + relays: Vec::new(), discovery: true, max_multipath_paths: None, #[cfg(feature = "transport-noq")] @@ -130,7 +131,20 @@ impl EndpointConfig { /// Relay URL string, e.g. `https://relay.example.com` or `http://127.0.0.1:3340`. pub fn with_relay(mut self, url: &str) -> anyhow::Result { use std::str::FromStr; - self.relay = Some(RelayUrl::from_str(url)?); + self.relays.push(RelayUrl::from_str(url)?); + Ok(self) + } + + /// Multiple relay URLs — the client fails over between them. + pub fn with_relays(mut self, urls: I) -> anyhow::Result + where + I: IntoIterator, + S: AsRef, + { + use std::str::FromStr; + for url in urls { + self.relays.push(RelayUrl::from_str(url.as_ref())?); + } Ok(self) } diff --git a/docs/architecture.md b/docs/architecture.md index ab899ca..24b389e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -241,9 +241,17 @@ connection. ### Stability measures - Relay-first connect (works on any egress-only network), in-band - hole-punch upgrade — both handled by iroh. + hole-punch upgrade — both handled by iroh. `--relay` is repeatable: + multiple custom relays give automatic client-side failover, which is + the iroh-recommended production topology (≥2 relays). - QUIC connection migration survives NAT rebinding/Wi-Fi↔LTE moves. - Agent reconnects to relay with backoff; CLI can pin `--relay`. +- QUIC transport tuning on both backends: BBRv3 congestion control + (paced, bufferbloat-resistant — vs loss-based Cubic default), + 4 MiB stream receive window / 32 MiB connection send window so a + large keyframe or sync chunk stream does not stall on high-BDP + links (upstream defaults target ~100 Mbps × 100 ms). iroh's own + multipath keep-alive and path idle-timeout defaults are preserved. - Serialized frame sends + collapse + mid-send stale reset bound worst-case latency under loss: queues stay near-empty and the residual tail is retransmit physics, not queueing. diff --git a/docs/deployment.md b/docs/deployment.md index 5637b57..439f443 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -64,6 +64,13 @@ table inet rds { accepts direct paths opportunistically (hole-punched or via the endpoint's discovered addresses). +**Relay redundancy.** `--relay` is repeatable on both `rds` and +`rds-agent`; iroh probes all configured relays, homes on the +lowest-latency one, and fails over automatically. Production should run +≥2 `rds-relay`/`rds-server` instances in different failure domains and +list all their URLs — no LB or failover plumbing is needed on the relay +side, since every endpoint carries the full list. + **TLS on the relay.** `rds-server` serves the iroh relay protocol (WebSocket over HTTP) on 3340 in plaintext: relayed payloads are end-to-end-encrypted QUIC the relay cannot read, and relay admission is