From a6365bb98f605367d3a653435d8bdf7bbe1862ac Mon Sep 17 00:00:00 2001 From: grumbach Date: Wed, 28 Oct 2026 15:36:19 +0900 Subject: [PATCH 01/15] feat(swarm): host a swarm rendezvous on every node Clients reading the same file share its records over WebRTC, BitTorrent style. Every node now introduces them and stays the last seed: a record no client has comes from the nodes as before. There is no new binary and no new port. Design: ADR-0026. The rendezvous (`src/swarm.rs`): - Rooms are keyed by a file's root DataMap address. Readers announce at the 2 closest nodes to it that host rooms, on the connections they already have (QUIC for native clients, WebRTC Direct for browsers). - The node names each member from its connection, hands each offer to a different member, and passes an answer back only for an offer it delivered to the answerer, once, before it expires (the protocol's 70 s signal lifetime). - An offer or answer is passed on with only the candidates another reader may try: no public address but its sender's own host, as the node reaches it on that request, no local address or mDNS name unless the node reaches the sender at a local address itself, and at most 16 (ant-protocol's `constrain_swarm_sdp`), so a stranger cannot point readers' connectivity checks at a third party or at their own home networks. A stored SDP is charged the memory it holds. - A native membership continued over a connection from another address counts against that address from then on, if it has room. - Native announces may wait up to 20 s for a signal. Browser announces never wait, so they hold no browser request slot. - A node hosts rooms only once browsers can reach it: until its browser listener has an endpoint and a STUN server to name, it refuses announces as `NotHosting`, so native readers pass it over at once and meet browsers at the closest nodes both reach. - Native requests are signed and checked for freshness and replay. A request naming another node is refused before it touches that history. - Membership numbers let the node keep each connection's newest membership. Ended memberships are remembered for as long as a request sent before they ended can be taken, so a late request cannot end, take over or revive one. - Native replies go only on the connection the request came on, through saorsa-core's `send_message_on_channel`. - Hard limits with no configuration: - rooms, members per room and in all, and rooms per connection; - members, histories, ended memberships and stored SDP bytes per host (an IPv4 address or an IPv6 /64); - unanswered offers per member and waiting announces. - The WebRTC Direct socket answers plain STUN Binding requests for hosts that announced in the last 60 s, rate limited in saorsa-transport. Routing: `NativeSwarmRouter` in `src/node.rs` (shared with the devnet) handles native swarm requests on their own task budget. `src/web_rtc.rs` handles browser ones before the storage check, so nodes that store no chunks host rooms too, and advertises `swarm_rendezvous`. Browser swarm requests run on a request budget of their own (64 a second, 8 per source): one within it gives the reads' budget back the token it took on arrival, one arriving when the reads' budget is spent is taken on the swarm's, and one past the swarm's budget is refused without closing the channel the reads share, so a room's announces never spend the request budget reads need, nor lose the channel to it. --- ...ents-share-records-nodes-introduce-them.md | 738 +++++ docs/adr/README.md | 1 + src/devnet.rs | 20 + src/lib.rs | 1 + src/node.rs | 237 +- src/storage/handler.rs | 39 + src/swarm.rs | 2889 +++++++++++++++++ src/web_rtc.rs | 464 ++- 8 files changed, 4363 insertions(+), 26 deletions(-) create mode 100644 docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md create mode 100644 src/swarm.rs diff --git a/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md new file mode 100644 index 00000000..1328f83c --- /dev/null +++ b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md @@ -0,0 +1,738 @@ +# ADR-0026: Clients reading the same file share its records; nodes introduce them and stay the last seed + +- **Status:** Proposed +- **Date:** 2026-10-27 +- **Decision owners:** +- **Reviewers:** +- **Supersedes:** none +- **Superseded by:** none +- **Related:** [ADR-0015](./ADR-0015-direct-browser-clients-over-webrtc-direct.md) + (browser sessions over WebRTC Direct, whose UDP socket also answers STUN + here), [ADR-0023](./ADR-0023-spread-hot-reads-and-refuse-full-browser-sessions.md) + (option 4, "browser peer-to-peer sharing", deferred to its own ADR: this + one), [RFC 8489](https://www.rfc-editor.org/rfc/rfc8489) (STUN), + [RFC 8445](https://www.rfc-editor.org/rfc/rfc8445) (ICE), + [RFC 8831](https://www.rfc-editor.org/rfc/rfc8831) (WebRTC data channels) +- **Implementation:** ant-protocol (swarm messages), saorsa-transport (plain + STUN Binding answers on the WebRTC Direct socket), saorsa-core (a send on + one named connection only), ant-node (the swarm rendezvous), ant-client + (the swarm in every client, native and browser) + +## Context + +Every reader of a file asks the same close group for the same records. ADR-0023 +spread hot reads across the group and made full browser sessions refuse +cleanly, but the group's capacity is still the ceiling: a node admits a bounded +number of browser sessions and bulk replies, so a popular file (a film many +people watch at once, a release everyone fetches) is served at the rate seven +nodes can serve it, however many readers there are. + +Readers of one file hold exactly the records the next reader needs. Records are +content addressed, every client checks each record against its address +(`ant-core/src/record.rs`), and public records are byte-identical for every +reader. So a record from another reader is as good as one from a node, once it +passes the check. BitTorrent peers share pieces the same way. What clients lack +is a way to find each other and connect: + +- Browsers cannot accept connections; two browsers connect only through WebRTC, + which needs a signalling channel to swap an offer and an answer, and a STUN + server to learn the public address the other side can reach. +- Native clients run in `NodeMode::Client`, are not DHT members, and are + usually behind NAT too. +- The network has no third-party service to lean on. A WebTorrent tracker or a + public STUN server would learn which addresses read which file, and would be + a single point of failure the network does not otherwise have. + +The nodes closest to a file are already the parties every reader talks to. They +can introduce readers to each other on the connections those readers already +have, and answer their STUN requests on the WebRTC Direct UDP socket they +already run. + +## Decision Drivers + +- Load on the close group should grow with the data, not with the number of + readers. +- No new binary and no third-party service: nodes and clients only. +- Every client takes part, browser and native alike, always: a swarm nobody + joins serves nobody. +- A peer can never make a client accept wrong bytes, and a slow or hostile + peer can only cost a short wait before the nodes are asked. +- A node's cost is small and bounded by hard limits, with no switch to turn it + off or tune it. +- A release that adds this must not break older clients or older nodes, in + either order of rollout. + +## Considered Options + +1. **A separate signalling service** next to the nodes (built as `ant-signal` + and reviewed, then dropped). It works, but it is one more binary to run and + secure, someone has to run it, and readers would depend on it. +2. **Third-party WebTorrent trackers and public STUN servers.** Free to use, + but they see who reads what, and the network would stop working the way it + should whenever they do. +3. **Peer records in the DHT** ("I hold file X, reach me at Y"). Clients are not + DHT members, browsers cannot be reached at an address, and two clients would + still need someone to carry their offer and answer. +4. **Nodes relay the data** (a TURN-like role). Every byte would still cross a + node, so nothing is offloaded. +5. **Nodes introduce readers and answer their STUN; readers share records over + WebRTC data channels; nodes stay the last seed** (chosen). + +## Decision + +We adopt option 5. + +### Rooms + +The readers of one file form a *room*. Its key is the address the file's +root DataMap would have as a public record: `compute_address(rmp_serde(root))`. +Every reader derives the same key whichever way it reached the file (by public +address, through a manifest, which may hold a shrunk or a root map, or from a +private map), once it has resolved a shrunk map to its root. For a file whose +public map is its root, the key is the file's public address. A reader +announces itself at the `SWARM_RENDEZVOUS_NODES` (2) nodes closest to the key +that host rooms, so two readers whose views of the close group differ +slightly still share one, and readers still meet while some of the closest +nodes predate rooms: it looks up the 8 nodes closest to the key and passes +over those that host none (see Older nodes). A node hosts rooms only once +browsers can reach it: until its browser listener has an endpoint, and so a +STUN server to name, it refuses every announce as `NotHosting`, and a client +passes it over at once, with no pause. Native readers so pass over the same +nodes browsers never see (a relay-only node, or one +built without the listener), and meet the browsers at the closest nodes +both can reach, rather than at nodes no browser comes to and that give them +no STUN. + +### Wire (ant-protocol, `src/swarm.rs`) + +Six `ChunkMessageBody` variants are appended after the pointer messages and +ADR-0020's get-or-closer pair (14 and 15), at discriminants 16 to 21; existing +variants keep their discriminants. + +| Request | Reply | Purpose | +|---|---|---| +| `SwarmAnnounceRequest { room, node, membership, offers, wait_ms }` | `SwarmAnnounceResponse::{Joined, Refused, Error}` | Join or stay in a room, hand over WebRTC offers, collect signals | +| `SwarmAnswerRequest { room, node, to, offer_id, sdp }` | `SwarmAnswerResponse::{Delivered, Unmatched, Error}` | Answer an offer the node delivered | +| `SwarmLeaveRequest { room, node, membership }` | `SwarmLeaveResponse::{Left, Error}` | Leave | + +`node` is the peer id of the rendezvous node the request is for. +`membership` names one of the client's memberships of the room: a number that +grows each time the client joins the room anew. A node compares only the +numbers of one connection identity (a native client's peer id, made afresh +each run, or one browser connection). It keeps each connection's newest +membership of a room: an announce naming a newer one starts a member in place +of the old (whose signals and offers go with it), and a leave ends the +membership it names, or an older one in its place. Admitting a membership +marks every older one of that connection in that room as replaced. A +membership the client ended (left, or replaced by a newer one) is remembered for twice the +10-minute signed-time window and 2 s more, as long as a request sent before +it ended can still be taken (one per connection and room however often it +ends a newer one; at most 64 per connection, 256 per host and 16384 in all, +past which the connection's, host's or node's oldest goes, so the newest +ends are the ones kept; a forgotten one can come back only as a member that +is dropped once silent): an announce naming it, or an older one, is +refused +(`SwarmRefusal::Superseded`), even one that arrives after its own leave. So a +request of a membership the client left, however late it arrives, cannot end +or take over a newer one, nor bring the old one back. A membership dropped +because its connection closed or it went silent is not ended: the client +carries on in it over a new connection. A waiting announce whose member was +replaced or dropped meanwhile is answered `Superseded`, not `Joined`. + +`Joined` carries the member's id, the room's size, how many offers went out, +the signals waiting for the member (offers from others, answers to its own), +and the node's STUN URL (`stun:`). + +Limits both sides check: at most 8 offers per announce, 16 KiB per SDP, 32 +signals and 48 KiB of SDP per reply (inside the browser transport's 64 KiB +small-response bound, asserted at compile time), at most 4 STUN URLs of at +most 256 bytes, and a wait of at most 20 s. `MAX_SWARM_REQUEST_BYTES` bounds +an encoded request, so a node refuses a longer one from its length before +decoding it, and a client validates every reply before acting on it, +announce, answer and leave alike, an error's text included (at most 512 +bytes). + +**Candidates.** An offer or answer is another client's, and its receiver +sends ICE connectivity checks to every candidate in it before anything proves +who made it. Both sides therefore pass each SDP through the protocol's +`constrain_swarm_sdp`: UDP candidates only, naming an IP address that is not +unspecified, multicast or broadcast, or an mDNS `.local` name, at most 16. A +node relaying an SDP also drops every candidate whose address is public and +not the sender's own host, as the node reaches it on that request (an IPv4 +address, or an IPv6 /64). Local addresses and mDNS names stay only where the +node and the client reach each other at local addresses (a network of their +own, such as a test network or a LAN deployment): the node keeps them only +for a sender it reaches at a local address, and a client applies them only +from a node it reached at one, loopback ones only over loopback. On a network +reached at public addresses, readers meet by their public (server-reflexive) +addresses alone, so two readers behind one NAT need it to hairpin. A stranger +in a room therefore cannot point other readers' checks at a third party, nor +at the readers' own home networks, and a hostile node can point them at 16 +addresses at most per SDP. Receivers read SDP leniently (webrtc-rs trims an +attribute's value and skips a carriage return before a line, Chrome takes an +attribute name followed by a space, Firefox names attributes in any case), so +a line any of them may read as a candidate or a DTLS fingerprint stays only +in its one standard form (`a=candidate:` or `a=fingerprint:`, then a value +with no whitespace around it), and a line with a carriage return anywhere but +before its line feed is dropped: what a receiver reads is exactly what was +checked. The filtered SDP holds no more memory than its length, and the node +charges a stored SDP the memory it holds. Honest SDPs only lose candidates, +so a link's binding to both ends' fingerprints does not change. + +**Decoding.** The offers, inbox and STUN server lists are refused while +decoding once past their limits, so a request or reply of a legal length +cannot make its receiver build thousands of entries before `validate` runs; +and a message with bytes after its end is refused, so a small reply cannot be +padded out to the wire limit. Every enum in +the module is encoded by variant position; a test pins each tag, and variants +may only be appended. + +### The node's part (ant-node, `src/swarm.rs`) + +- **Identity.** The node names each member itself, from the authenticated + connection (a native peer id, or the number of a browser connection on this + node). A browser's session is its connection. A native client's request is + signed by its key but not bound to the connection it came on. So every + request names its rendezvous node, and a node refuses one naming another: + a request one node saw cannot be replayed at the room's other node. A node + also drops a native request whose signed time is more than 10 minutes from + its own clock, more than 5 s older than that client's newest, or whose id + it has seen, so one cannot be replayed at the same node either. A request + naming another node is answered with an error before it touches that + history, so a node that saw a client's request cannot send it on to make + the client's later requests elsewhere look stale. Only an + announce or a leave starts a client's history (an answer comes from a + member, which announced first; a leave may overtake the announce it ends); + a host keeps at most 256 histories, 16384 in all, + the oldest made forgotten first, and a history goes once nothing it holds + could pass the 10-minute check. +- **Introductions.** Each offer goes to a different other member, picked at + random. An answer is passed on only to the member whose offer the node + delivered to the answerer, once, before it expires. Unanswered offers, and + signals waiting for their member, expire after the protocol's + `SWARM_SIGNAL_TTL_MS` (70 s), which a client checks at compile time against + the slowest answer any client gives: 11 s to its next announce (a + browser's 3 s pause and one 8 s step to make that round's offers), 30 s + for an announce that may already be on its way (a native one may wait + 20 s, with 10 s for the request and reply), 8 s to make the answer and + 10 s for the answer's request. An offering client keeps its offer for its + own announce's time on each way, the poll gap and the node's 70 s: 141 s + natively, 101 s in a browser, whose announces do not wait. It drops + expired offers before and after every announce, and all of them before a + pause after a failure, so an unanswered offer holds its place in the + room's connection limits at most that long and one announce more. A link + a client answered has 79 s to open and be proven, since its answer still + has to reach the offerer; one it offered has 20 s once it has the answer. A member that stops + announcing is dropped after 60 s; a connection that closes leaves all its + rooms at once. +- **Long poll, native only.** A native client's announce with nothing waiting + may be held until a signal arrives, the wait ends, the member is dropped, or + the node stops; one announce per member waits, and no lock is held + meanwhile. Native swarm requests are routed apart from chunk requests, with + their own task budget, so a waiting announce never delays a GET. A browser's + announce is never held: it would keep one of the browser session's few + request slots, and its byte reservation, from the browser's reads. Browsers + ask again after 1 s while a signal is likely on its way (they just acted on + one, or made an offer within the last 10 s) and after 3 s otherwise: in a + room larger than its members' links can hold, most offers go unanswered, + and their makers wait at the slower pace. At a node's browser listener, + swarm requests run on a budget of their own (64 a second in all, 8 from one + source): every request takes a token from the budget the reads share as + it arrives, and a swarm request the swarm's budget has room for gives it + back, so announces never spend what reads need; one past the swarm's + budget keeps that token and is refused, without closing the channel the + reads use, and its maker backs off as from any refusal. A request that + arrives once the reads' budget is spent is taken on the swarm's instead, + since only the opened frame says whether it is a swarm request: one that + is goes on, and anything else gives the swarm its token back and is + refused as over the request rate, as before. +- **Closing.** A connection that closes leaves all its rooms at once. No + request of it still being handled can act after the close (announce, answer + or leave), nor any request older than a minute. A member that goes takes + with it its signals still queued for others and the offers others made to + it, since none can be answered now. +- **Hard limits**, no configuration: 4096 rooms, 256 members per room, 16384 + members in all, 4 rooms per connection, 64 members per host (an IPv4 + address or an IPv6 /64, since a native client's identities cost nothing to + make), 32 unanswered offers per member, + 16 MiB of stored SDP (2 MiB for the offers and answers of one host's + members), 1024 waiting announces (more are answered at once), and + a fixed number of swarm tasks (more requests are dropped). +- **Every node.** A node hosts the rendezvous whether it stores chunks or + not, and devnets route native swarm requests the way nodes do. +- **Replies.** A reply to a native client whose connection closed meanwhile, + or from a node stopping, is dropped rather than sent, and a reply goes + only on a connection the client still has, so the node never dials a + client that left. A signed message is not bound to the connection it came + on, and saorsa-core maps a client's identity to every connection a message + signed by it arrived on: a node that saw a client's message could send it + on to another node, which would then take the sender's connection for the + client's too. So a native reply goes on the connection the request came + on and no other, through saorsa-core's new `send_message_on_channel` + (never another of the peer's connections, never a dial). Other protocols' + replies still go by peer id; moving them to the same call is a separate + change. Offers are handed on in order, so `offers_delivered` names exactly + the first ones, and the client drops the rest at once. +- **STUN.** The WebRTC Direct UDP socket also answers a plain STUN Binding + request (no USERNAME, no comprehension-required attribute, a valid + FINGERPRINT if present) with XOR-MAPPED-ADDRESS. The answer (32 bytes for a + 20-byte IPv4 request, 44 for IPv6) is larger than the request, and a UDP + source can be forged, so it is bounded three ways: only hosts (an IPv4 + address or an IPv6 /64, as the responder's own limits count them) that + announced in a room within the last 60 s are answered, so a victim's + address is answered only by the few nodes of its own rooms; each source (an + IPv4 address or an IPv6 /64) gets at most 10 answers a second (burst 20, + 4096 sources tracked); and the socket sends at most 2000 answers a second + in all, after a burst of at most 4000. A request counts against both + limits or neither: only requests that would be answered count, so one + source over its rate, or sending malformed requests, costs the others + nothing, and a request the socket's limit refuses costs its source + nothing. Every check runs on the raw bytes without allocating, the answer + is built only once both limits grant it, and a refused plain request is + dropped rather than passed to the connection handling that shares the + socket. A request with a comprehension-required + attribute is left alone rather than answered with `420 Unknown Attribute`: + no ICE agent sends one, and an error answer is one more reply a forged + source could draw. Policy lookups take a read lock, so the packet path + rarely waits on announces. +- **Capability.** A node advertises `swarm_rendezvous` in its browser HELLO. + +### The client's part (ant-client, `ant-core/src/swarm/`) + +Every client takes part, always, on both builds: a browser through its own +`RTCPeerConnection`, a native client through webrtc-rs (the stack +saorsa-transport's WebRTC Direct already uses). + +- **Joining.** Only a valid root map has a room (not a child map, one whose + records index, with at least the 3 records a file has), checked before + joining, so a bad map never takes a valid room's place; and only one with + at most 262,144 records (about 1 TiB of 4 MiB records), all of which a + room holds, so a room always shares its whole file. A larger file is read + from the nodes alone. Every read of a file joins its file's room as soon + as it has the root map (native `file_download*` and `data_download*`; + in a browser, opening a reader, a public or private download, and a + stream), and a browser reader joins again on a read if reading other + files has left the room since. Joining starts the room's own task and + returns at once: the other readers are found beside the reads from the + nodes, which never wait for that (see Reading), so a slow or failing + discovery leaves the reads from the nodes as they were. The small DataMap + records come from the nodes. A room joined by a read asks the nodes for + nothing (no lookup, no announce) until that read's first record of the file + is in, or for 30 s if none comes: its discovery so never competes with a + download's first record for the nodes, the client's connections or its + CPU. (With discovery started at the open, the first chunk came 0.3 to + 6.5 s later in 6 of 7 lab pairs whose nodes were the bottleneck.) A room + joined on request (`join_swarm`) starts at once. A client's read of a + file's record keeps the file's room in use for as long as it runs, however + long the nodes take. Joining past 4 rooms leaves the room unread longest, once no + read has used it for 30 s; with every room still read, the join waits (at + most 2 do) and is made at a later read of one of the file's records once a + room has gone unread, the file meanwhile read from the nodes. A client + reading more files than that at once so cannot make every read a new + membership (a key, a lookup, links that never open), and a long download's + room is never left for a file opened beside it; at most 8 rooms left at a time tell their nodes so (each + leave bounded to 10 s in all), and one left past that leaves quietly, its + places there expiring after 60 s. A room left tells its nodes even if it + never saw its join confirmed (a reply can be lost). Two places are not + told: a room left quietly, and a node a rendezvous place stops using after + 3 failures or 10 minutes (the place may come back to it in the same + membership). Those memberships are not ended at the node; they go after + 60 s of silence, and a late announce of one can bring it back only until + it is silent that long again. A room joined again is a new membership: + its number is newer, so nothing the old one still sends can end it, and + links made for the old one are not kept by the new one. A room left or shut + down drops its record list at once. `Client::join_swarm` and the browser's + `joinSwarm` let an application join before reading, so the client meets + the others first. A page tells which file each link and count belongs to + from the channel's label, from the `Swarm room ` a browser read + reports when it resolves its file, and from per-room counts + (`swarm_room_stats`, `swarmRoomStats`). +- **One swarm per identity.** Rendezvous nodes know a native client's + memberships by its node identity, so clients built on one native node + share one swarm. A membership continued over a connection from another + address counts against that address from then on, if it has room. Each + browser client has its own sessions, and its own swarm. +- **Room-scoped records.** A room knows its file's records: those its root + map lists. A record is asked of peers, kept, announced and served only + within a room whose file holds it, so a peer never learns what else a + client read. Reads of any other record (existence checks, a `chunk get` by + address, DataMap records, a shrunk map's wrappers included) neither ask + peers nor keep the record. +- **Proven links.** The room's key is the root map's address; next to it a + client derives the room's *secret* from the root map's bytes, which a node + that knows only the key of a private file's room does not have. Each end + of a new link sends `Proof`: a keyed BLAKE3 MAC, under the secret, of its + role (offerer or answerer), the offer's id, and the DTLS fingerprints of + the offer and the answer as both ends saw them. Each WebRTC stack checks + the peer's DTLS certificate against those fingerprints, so a node that put + itself in the middle, with an offer and an answer of its own, cannot pass + one client's proof on to the other. The secret proves nothing about which + member a link reaches, since every member knows it, so each membership of + a room also has an ML-DSA-65 key of its own: `Hello` carries the public key + (1952 bytes), the member's name in the room is a hash of it, and `Proof` + also carries the member's signature (3309 bytes) over its role, the offer's + id and the same fingerprints. A member that names itself by another's key + cannot sign for it, and a signature made for one link proves nothing on + another, so no member can pose as another to settle away its links or take + its names. A link opens only after the peer's + `Hello` and then its `Proof`, which must check out within 20 s of the + answer on a link this client offered, or 79 s on one it answered (which + starts before the offerer has had the answer): until then + no `Have` is sent, nothing is served, no duplicate is settled, and any + other message ends the link. A public file's secret is open to anyone with + its address, as its records are. A client keeps one link per member, as + the node that introduced it names it, so a member cannot fill the room's + links by connecting again and again under new names: one answer at a time + per member and 8 in all, each held until the node has taken the answer, + taken before any negotiation; until its proof, one link each + way per member (two clients offering to each other at once make + one each way, and settle on one once open); once open, a member linked + again under another name keeps its first link. +- **STUN.** A client uses only the STUN URLs a rendezvous node names at an + IP address the client reached that node on, two per node: natively the + remote address of an authenticated connection to it, in a browser the + host of a WebRTC Direct session with it. The addresses a node advertises + are its own claim, so it cannot send a client's STUN requests, and with + them its address, to another host. With no such connection, none of that + node's STUN URLs are used. A native connection through a relay is named + by the relay's address, a host the client already sends to. +- **Meeting.** The two rendezvous places never use the same node: a place + whose lookup puts the other place's node at its own position takes the + first node of the lookup the other does not use. Offers go through every + rendezvous node, as many as that node + has members the client has no link or connection being made to there, + each node counted alone (so two readers whose views of the close group + share only one node still meet there); offers from others are answered + while the client wants links (8 per room, 16 in all, counting the + connections being made). A link made through one node to a peer that is + also a member of the other settles with its duplicate, and the link kept + then counts at both nodes, so the nodes do not keep doubling each other's + offers. A room looks its nodes up once for both places, and one lookup, + found or failed, serves them for 30 s; a place takes a node again when its membership ends + (after 3 failures in a row, or 10 minutes), so a node that goes is + replaced while the other carries on; its STUN URLs are dropped as the + membership ends. Two links between the same pair (met + through both nodes, or crossing offers) settle on the one both ends pick, + told apart by the name the client has in that room alone (from its key for + that membership), so peers meeting it in two rooms cannot tell it is the + same client by its name or key. Its addresses may still let them link + the two: each connection's WebRTC candidates carry whatever host and + public addresses it gathers (see below for native offers), which are often + the same across rooms, though not reliably so (shared NATs, changed + interfaces, browsers' mDNS names). The swarm gives no unlinkability at the + network level. The end that + closes a duplicate stops using it at once but keeps its connection 2 s + more, still counted against the link limits: the other end may not have this end's proof on that link yet, and + without it would see a link that failed rather than a duplicate, never + learn that both reach the same peer, and offer that member again every + few seconds (seen between a native and a browser client). Signalling is + non-trickle, with up to 3 s of ICE gathering, and a browser takes the page's + ICE gathering turn after any connection to a node waiting for it (see + Discovery is bounded). Every negotiation step (making an offer, answering + one, applying an answer) ends after 8 s, or as soon + as the room is left or the client shuts down, and a step that ends closes + the connection it was making; the swarm is never held across one, so a + native client dropped meanwhile still shuts its swarm down. A connection + being made through either node counts at both until it is a link, so a + reader starting out does not offer through both at once to a peer of + both. An announce is bounded as a whole, waiting for the transport + included, so a busy browser session cannot hold a round's offers past + their time. A native + announce that comes back empty well before the wait it asked for (the node + had no place for it to wait) is followed by a 2 s pause, not another at + once, and a native client starts announces at one node at least 1 s apart + whatever the replies hold. A room starts at most 8 answers to other + members' offers in 20 s, so a node that hands out offers of members it + makes up, well formed or not, costs a bounded number of negotiations + however fast it replies. A link has exactly one data + channel, labelled `ant-swarm-`: the tag is 16 hex digits derived + from the room's secret, so it names the room and says nothing else, and an + answerer refuses a channel labelled for another room. Another channel + opened by the peer, a text message, or a message longer than the wire + allows ends the link before anything is copied. +- **Sharing.** Over the data channel, peers say which records they hold + (`Have`), ask (`Get`), and answer with 16 KiB `Piece`s, `Missing` or `Busy`. + Every message is bounded before anything is allocated for it; every piece + but a record's last is a full frame; a channel holds at most 257 messages + unread (a bound of 256 plus the one sender's place) and, unsent, 256 KiB + plus the message that crossed it, about 272 KiB. +- **Reading.** A record the client's swarm holds already (checked when it + came) is used first. Otherwise (and except for two native downloads whose + data chunks come from nodes only: one with download diagnostics, whose + figures count the nodes' answers, and the peer-report download, which + still shares what it reads and may resolve a large file's map as any + other read does) a read asks peers only for a record a linked + peer in the record's room said it holds. Peers complement the nodes and + never hold a read up: a read that asks a peer gives it the read's hedge + delay (3 s in a browser, 1 s natively: the time a node gets before a + second is asked). Past that, the peer keeps the read while, at its average + rate so far, the whole record arrives within the read's patience, and for + the patience at most (measured on the clock from when the peer was asked, + so timers that fire late on a busy page do not stretch it): as long as reads that went to the nodes have lately + taken to have their records (a running average), at least twice the hedge + delay, and never past the 12 s a peer has to send a record. Then, or as + soon as the peer falls off that pace or fails, the read asks the nodes + beside it, and the first checked record wins: a node read that ends + without one (the nodes refuse, are busy or have none) leaves a peer still + sending to finish within its own deadline, and only then is the read + answered as the nodes answered; the node limiter still learns of the + nodes' miss. A slow or failing peer so + delays a read by the patience at most, which is about what a node read + started then would take, and by the hedge delay unless it was on pace to + beat one. (A node read started while the peer is still sending costs the + node's bytes even when the peer wins; with a fixed 3 s window, six + staggered readers in the lab used 1 % more node bytes than nodes alone.) A browser read still waiting + for its turn at the nodes, with no GET sent, takes its record from a + linked peer that offers it meanwhile (it looks every 250 ms) while it + keeps its place in that queue: if the peer fails or is slower, the node + read goes on as if no peer had been asked. A read asks up to 2 peers in + all, however often it asks (a + peer whose record fails its check counts as one; a link closed meanwhile as the + duplicate of one kept does not, and the read goes on to its twin, waiting + up to 2 s for the twin to say what it holds), with 2 s to send each next + piece, a deadline of 2 s plus the record's size at 512 KiB/s, and 12 s for + the read in all, from its first ask; a peer that stalls or runs past its record's time is + passed over for 30 s (the read's own time running out does not count + against it). A + browser download asks for its records in an order of its own, picked at + random, as BitTorrent picks pieces: a download keeps up to 8 records in + flight (4 at first outside Chromium), and readers of one file who start together and ask in the same + order would hold the same records at every moment, and want the same next + ones, so none would have a record another is still waiting for. A + download delivers its chunks in any order, so its order costs it nothing; + a stream or a range read asks in the order its reader reads. A client asks one link for one record at a time: in + Chromium, two records sent at once on one data channel, a few pieces of + each in turn, stalled it for seconds (a 2 MiB record took 3.6 to 5.3 s + that way, 66 to 656 ms alone). One more read may wait its turn at the + client, where the wait is seen, keeping the link busy back to back; reads + past that go to the nodes rather than queue behind a peer that may be + slower than they are (on a local network, queueing made the last of four + readers starting 1.5 s apart 11.5 s slower than with no peers at all); at most 8 records come from peers at once, and reads past that go + to the nodes, as do the reads of a hidden browser page (a tab in the + background), whose timers the browser slows to a second or more; a page + hidden mid-read leaves the peers for the nodes at once, on its + `visibilitychange` event. A record being received holds only the pieces that came + (grown as they arrive, doubling from 64 KiB and never past the record's + size), so a peer that names a large record and stops costs at most about + twice what it sent, and never less than 64 KiB, and records being + received hold at most 8 records' bytes. A record that fails its check ends + that link; pieces out of order end it too. +- **Existence checks never ask peers.** `chunk_exists` (used before uploads to + skip stored records) asks only nodes, and records from peers stay out of the + chunk cache that answers it: another client holding a record says nothing + about whether the network stores it. +- **The node limiter sees only nodes.** A record from a peer does not feed the + adaptive fetch limiter, which measures how the nodes are coping, including + the reads browser read-ahead holds for; after a fallback, only the time the + nodes took counts. +- **Serving.** Records fetched from nodes, or from peers and checked, go into a + store of 256 MiB (native) or 64 MiB (browser), oldest dropped first, and so + do records a read finds in the chunk cache, so records read before the + room was joined are offered too. Records a client already holds when it + joins a room are told to the room's links as the room learns its file's + records, so a link that opened first still hears of them. A client sends at + most 4 records at once, and starts no send past 1 while reads of its own + are running (sends already running when a read starts finish; the count + is taken in one step, so two links never both take the last place): a + record sent costs the sender's CPU and line, which its own download needs + first + (in the lab, readers each held to one CPU core and sending 4 at once while + downloading took up to 40 % longer than nodes alone; sending 1 at a time, + they were no slower); a client done reading sends 4, + 2 per link, one more than a peer asks for at once (a record the peer gave + up on counts until its next piece sees that, and the peer's next ask may + come first), and answers `Busy` past that; an ask cancelled and made again under the same id is a new transfer, + and the old one stops. A record, or any other message, + that takes more than 20 s to send (a peer that stopped reading) ends the + link. Messages take turns on a link in order, and a record lets the + others, and newly arrived asks, have the channel every 64 KiB, so records + sent at once interleave rather than one taking every turn (a browser send + that need not wait never yields by itself; the browser yields through a + posted message, not a timer, which background tabs slow). A browser waits on a full + channel for + its `bufferedamountlow` event rather than a timer: browsers slow timers in + tabs that are not in front to one a second, which would cap a tab left + seeding at a quarter of a megabyte a second. +- **Older nodes.** A browser sends swarm messages only to nodes that advertise + `swarm_rendezvous`: an older node closes the channel on a message it cannot + decode, which would fail the reads in flight on it. The browser client's + network adapter must say which nodes do (the client's own does); a client + on an adapter that cannot joins no room and reads from the nodes alone. Older nodes drop the + messages of a native client silently, which costs that client a timeout. + A rendezvous place passes over, for 10 minutes, a node a session says + hosts no rooms (at once, from its capabilities, and still once that + session has closed) and one that never let the client join in a + membership (after its 3 failures), and takes the next closest of the + nodes the room looked up (a room remembers as many such nodes as its + lookups can name in 10 minutes, 160); readers that see the same + nodes pass over the same ones, so they still meet. With none left, a place + tries again less and less often, doubling up to 5 minutes. Reads go to the + nodes as before throughout. (Without this, a room whose closest node + predated rooms never formed: in a lab network with 10 of 56 nodes on the + previous release, six readers starting together met no one.) +- **Discovery is bounded.** Per room: one lookup of its nodes at a time and + at most one per 30 s for both places, two places, each with one announce in + flight (a browser asks again after 1 s while a signal is likely on its way + and after 3 s otherwise, within the node's swarm budget; a native announce + is held at the node for up to its wait, and announces at a node start at + least 1 s apart), + at most 2 offers per announce, each a WebRTC connection with up to 3 s of + ICE gathering and every negotiation step bounded, and at most 8 links; + per room, at most 8 answers started in 20 s; per client: 4 rooms, 16 + links, 8 answers being made, 4 records sent (1 while any of its own reads + runs) and 8 + received at once. In a browser that makes connections take turns to + gather ICE (every engine but Chromium), the swarm's own connections take + the page's turn only while it is free and no connection to a node waits + for it, and a room's lookup asks one node at a time, dials each only then, + and warms no connections to the nodes it learns of; in any browser a room's + lookup never starts the seeds' bootstrap (which opens several connections + at once): it starts from the routes the reads' own lookups found, waiting + up to 10 s for one, and with none (a tab left seeding whose routes went + with its network) from one seed, the seeds tried one at a time in the + discovery turn; discovery so holds at + most one turn at a time, and a read whose connection starts while it does + waits for that one turn, at most 1 s (the longest a connection holds the + turn). `swarm_stats().discovery` (`swarmStats` in a browser) + counts lookups, announces and their failures by cause, the signalling + bytes, offers made, delivered and received, answers sent and received, + channels opened, and links proven or failed. In lab runs of six browser + readers of a 128 MiB file, each staying about 3 minutes after its + download, each reader sent 28-47 KB of swarm requests (one lookup, about + one announce a second) and received 14-22 KB of offers, answers and STUN + names over its whole stay. + A browser's pool keeps 64 node sessions and closes the least recently used + idle one, which may be a rendezvous node's; the node then drops the + membership, with its offers. The client's next announce joins anew under a + new member id, which makes it drop its earlier offers at once rather than + hold their places for their lifetime. Only that announce connects again: + in the discovery turn, which it lets go before it waits on the node, like a + room's lookup; an answer or a leave on a session that has closed fails at + once, since the node has dropped the membership it belonged to. + +## Consequences + +### Positive + +- The close group serves a popular file about once per swarm rather than once + per reader. In the native end-to-end test the second reader of an 8 MiB file + took every record from the first, and the nodes received no GET for any of + them (`ant-core/tests/e2e_swarm.rs`). +- No new service and no third party: the readers' own nodes introduce them, on + connections they already have. +- A peer cannot corrupt a read, and a bad or slow peer costs one short wait. + +### Negative / Trade-offs + +- **Readers of a file learn each other's addresses.** That is how + BitTorrent works, and it is new here: until now only nodes saw a reader's + address. A browser hides its local addresses behind mDNS names; a native + client's offer lists its host addresses. The rendezvous nodes learn which + connections read which file; a reader's close-group nodes could already see + the records it fetches. +- **Private files too.** Two readers of the same private DataMap now meet; the + room key, a hash of the root map, does not reveal the map or the content. + Self-encryption is convergent, so the same content stored privately and + publicly has the same root map and records, and its private and public + readers meet in one room; its nodes could already see them fetch the same + records. +- **Clients upload.** A client sends up to 4 records at once to other readers + for as long as it is in their room, which costs bandwidth and, on phones, + data and battery. +- **Browsers poll.** A browser's announce every 1 to 3 s per room and + rendezvous node is a small request on its control lane; a held announce + would cost it a request slot instead. Native clients long-poll. A room of + a few hundred browsers can fill a node's swarm budget; its members are + then refused and back off, which slows their meeting but never their reads. +- **Nodes browsers cannot reach host no rooms**, so a network of nodes built + without the browser listener has no swarm, natively either. +- **Native clients link webrtc-rs**, which compiles in rustls's ring provider + alongside aws-lc-rs. rustls then cannot pick a process-wide provider by + itself, so the client installs aws-lc-rs, the one QUIC uses, before its + first peer connection, as saorsa-transport already does for nodes. +- **Two readers on different networks may not connect** when both NATs are + symmetric: there is no relay, by design. They read from the nodes. +- **Native readers behind relays get no STUN.** A client uses a node's STUN + servers only at an address it reached that node on. A native client whose + connection to a rendezvous node runs through a relay reached the relay, + not the node's WebRTC Direct address, so it uses none of that node's STUN + servers, and on a network reached at public addresses its host candidates + are dropped too. Two such native readers behind NATs then exchange offers + but cannot link, and read from the nodes. Browsers reach nodes directly + and are not affected. Open: whether a client may check a node's direct + address before trusting its STUN servers, or should prefer a directly + reached rendezvous node. +- **A request taken after its connection closed.** A node can take a native + request its transport queued before the connection closed, after the + close; such a membership, like any a client stops using without leaving, + stays at most 60 s of silence. Tying requests to the connection they came + on needs saorsa-core to tag packets with their connection; it does not. +- **Hidden tabs.** A hidden browser tab reads from the nodes, not peers, and + still serves. Its 20 s send bound runs on timers the browser slows in a + hidden tab (to about a minute at worst), so a peer that stops reading can + hold one of its 4 upload turns that long; stopping uploads when hidden + would end background seeding. + +### Neutral / Operational + +- Nodes count announces, refusals, offers delivered and undelivered, answers + delivered and unmatched (`SwarmService::counters`); clients report + `SwarmStats` (records and bytes from and to peers, rejected records, links, + rooms). +- Rollout order does not matter for correctness. Nodes first lets new clients + share at once; clients first costs native clients the backed-off timeouts + above until their rooms' nodes upgrade. + +## Validation + +- ant-protocol: the variants' discriminants are pinned, and a swarm announce + round-trips; the limits are tested at their bounds. +- saorsa-transport: plain Binding answers carry the source address (IPv4-mapped + sources as IPv4), check and return FINGERPRINT, leave connectivity checks and + other packets to the usual handling, drop refused plain requests, are + answered only for allowed sources within their rate, share one rate per IPv6 + /64, and stop at the socket's overall budget. +- saorsa-core: a send on a named connection is refused for a connection that + is not the peer's, and never falls back or dials. +- ant-node: unit tests for requests naming another node, introductions, answers nobody asked for, long-poll + wake-up and shutdown, caps and leaving, expiry, the STUN allowlist, request + classification from the header, the unanswered-offer cap, browsers never + waiting, one waiting announce per member, a signal already taken not cutting + a later wait short, requests from before a close joining nothing, + replayed or stale native requests, histories started only by announces and + leaves and capped per host, ended memberships refused however late their + announce, kept per connection and host, members capped per host, and a departing member's signals + and the offers made to it going with it. +- ant-client unit tests over an in-memory WebRTC stand-in: a record served to + a second and a third client, nothing asked for a record nobody announced, a + forged record and out-of-order pieces ending the link, a stalled peer costing + one stall and being sent `Cancel`, a dribbling peer running out of time, + `Missing` and `Busy` falling back at once, crossing links settling on one, + the per-link send limit, reads taking turns on one link, a record of another + room neither announced nor served, a full queue closing the channel, and + shutdown. +- ant-client end to end, native, on in-process nodes with real webrtc-rs: a + second reader served records by the first, the nodes asked for every record + it did not send, a reader whose peer left served by the nodes, and readers of + different files not meeting. (The race itself, a peer keeping a read or the + nodes taking it, is timed by unit tests on a paused clock.) +- ant-client end to end in real Chromium (`browser-tests/swarm.spec.js`): two + pages in separate contexts read one file; with nodes that host rooms, the + second takes data records from the first (CI runs this against the ant-node + the client is built with); with the pinned older node CI also runs, + it reads from the nodes and no swarm message is sent. +- A native client and Chromium pages in one room, on a local devnet: a page + took every data record from a native client (webrtc-rs), and a native client + every data record from a page, in 4 of 5 runs; in the other one record ran + out of time and came from the nodes. +- The browsers installed on the machine, started by the operating system + rather than a test driver: Brave took every data record from Chrome in 3 of + 3 runs. Firefox could not be checked locally: the local nodes listen only on + loopback, which Firefox does not use for WebRTC. +- Review triggers: a record from a peer ever reaching a caller unchecked, or + answering an existence check; a node limit that has to grow; the STUN answer + growing relative to the request. + +## Notes for AI-assisted work + +AI tools may help draft this ADR, but **must not mark it Accepted without human review**. Accepted ADRs are immutable: create a new superseding ADR rather than editing an Accepted ADR. diff --git a/docs/adr/README.md b/docs/adr/README.md index 9ae2edf7..c28695f2 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -42,5 +42,6 @@ See [`TOOLING.md`](./TOOLING.md) for `adrs`, `adr-kit`, and AI harness setup. - [ADR-0020: Kademlia-style chunk GET: return the chunk or closer peers](./ADR-0020-kademlia-chunk-get-or-closer.md) - [ADR-0022: Remove the LMDB Chunk Store](./ADR-0022-remove-the-lmdb-chunk-store.md) - [ADR-0023: Spread hot-record reads across the close group, and refuse full browser sessions explicitly](./ADR-0023-spread-hot-reads-and-refuse-full-browser-sessions.md) +- [ADR-0026: Clients reading the same file share its records; nodes introduce them and stay the last seed](./ADR-0026-clients-share-records-nodes-introduce-them.md) - [ADR-0027: Map the browser listener's port through UPnP](./ADR-0027-map-the-browser-listener-port-through-upnp.md) - [ADR-0028: Pressure-gated retention of out-of-range records](./ADR-0028-pressure-gated-retention.md) diff --git a/src/devnet.rs b/src/devnet.rs index 07cbaf4f..1c1ce6fe 100644 --- a/src/devnet.rs +++ b/src/devnet.rs @@ -30,6 +30,10 @@ use tokio::time::Instant; use tokio_util::sync::CancellationToken; use tokio_util::task::TaskTracker; +use crate::node::NativeSwarmRouter; +#[cfg(feature = "webrtc-direct")] +use crate::swarm::SwarmService; + #[cfg(all(feature = "webrtc-direct", feature = "test-utils"))] use crate::ant_protocol::{ChunkMessage, ChunkMessageBody, ChunkPutRequest, ChunkPutResponse}; #[cfg(all(feature = "webrtc-direct", feature = "test-utils"))] @@ -1294,11 +1298,17 @@ impl Devnet { node.index )) })?; + let swarm = node + .ant_protocol + .as_ref() + .map_or_else(SwarmService::new, |protocol| Arc::clone(protocol.swarm())); + swarm.set_node_id(*p2p.peer_id().to_bytes()); let server = crate::web_rtc::spawn( &webrtc_direct_config, &node.data_dir, p2p, node.ant_protocol.clone(), + swarm, self.config .evm_network .as_ref() @@ -1327,8 +1337,18 @@ impl Devnet { let protocol_clone = Arc::clone(protocol); let handlers = node.handlers.clone(); let node_index = node.index; + // Native clients' swarm requests, as a node routes them. + let swarm = NativeSwarmRouter::start( + Arc::clone(protocol.swarm()), + Arc::clone(p2p), + TaskTracker::new(), + self.shutdown.clone(), + ); node.protocol_task = Some(tokio::spawn(async move { while let Ok(event) = events.recv().await { + if swarm.route(&event) { + continue; + } if let P2PEvent::Message { topic, source: Some(source), diff --git a/src/lib.rs b/src/lib.rs index dd07b8e9..df50e185 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -58,6 +58,7 @@ pub mod payment; pub mod pointer; pub mod replication; pub mod storage; +pub mod swarm; pub mod upgrade; #[cfg(feature = "webrtc-direct")] mod web_rtc; diff --git a/src/node.rs b/src/node.rs index 1377e72f..482854d1 100644 --- a/src/node.rs +++ b/src/node.rs @@ -1,6 +1,6 @@ //! Node implementation - thin wrapper around saorsa-core's `P2PNode`. -use crate::ant_protocol::{CHUNK_PROTOCOL_ID, GET_OR_CLOSER_AGENT_TOKEN}; +use crate::ant_protocol::{ChunkMessage, CHUNK_PROTOCOL_ID, GET_OR_CLOSER_AGENT_TOKEN}; use crate::config::{ default_nodes_dir, default_root_dir, NetworkMode, NodeConfig, NODE_IDENTITY_FILENAME, }; @@ -19,19 +19,22 @@ use crate::storage::legacy_artifacts; use crate::storage::traffic as storage_traffic; use crate::storage::MIB; use crate::storage::{AntProtocol, ChunkRequestContext, ChunkStore, ChunkStoreConfig}; +use crate::swarm::{self, SwarmService, SwarmSession}; use crate::upgrade::{ upgrade_cache_dir, AutoApplyUpgrader, BinaryCache, ReleaseCache, UpgradeMonitor, UpgradeResult, }; use rand::Rng; -use saorsa_core::identity::NodeIdentity; +use saorsa_core::identity::{NodeIdentity, PeerId}; use saorsa_core::{ IPDiversityConfig as CoreDiversityConfig, MultiAddr, NodeConfig as CoreNodeConfig, P2PEvent, P2PNode, }; +use saorsa_transport::shared::normalize_socket_addr; +use std::net::IpAddr; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicI32, Ordering}; use std::sync::Arc; -use std::time::Instant; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::sync::mpsc::UnboundedReceiver; use tokio::sync::Semaphore; use tokio::task::JoinHandle; @@ -67,6 +70,204 @@ pub(crate) fn user_agent(serves_chunks: bool) -> String { } } +/// Routes native clients' swarm requests to the rendezvous, apart from chunk +/// requests, and sweeps it: the same for nodes and devnet nodes. +pub(crate) struct NativeSwarmRouter { + swarm: Arc, + /// Swarm requests have their own budget: an announce may wait for a + /// signal, and must never hold a permit chunk requests use. + slots: Arc, + p2p: Arc, + children: TaskTracker, + stopping: CancellationToken, +} + +impl NativeSwarmRouter { + /// Starts sweeping `swarm` every [`SWARM_SWEEP_INTERVAL`] until `stopping`. + pub(crate) fn start( + swarm: Arc, + p2p: Arc, + children: TaskTracker, + stopping: CancellationToken, + ) -> Self { + let sweeping = Arc::clone(&swarm); + let stop = stopping.clone(); + children.spawn(async move { + let mut every = tokio::time::interval(SWARM_SWEEP_INTERVAL); + loop { + tokio::select! { + () = stop.cancelled() => break, + _ = every.tick() => sweeping.sweep(Instant::now()), + } + } + }); + Self { + swarm, + slots: Arc::new(Semaphore::new(swarm::MAX_SWARM_TASKS)), + p2p, + children, + stopping, + } + } + + /// Handles `event` if it is the swarm's: a connection that closed, or a + /// swarm request, handed to its own task. Returns whether it was. + /// Saturated, a request is dropped: the client times out and fetches + /// from nodes alone. + pub(crate) fn route(&self, event: &P2PEvent) -> bool { + if let P2PEvent::PeerDisconnected(peer) = event { + self.swarm + .session_closed(SwarmSession::Native(*peer.to_bytes())); + return true; + } + let P2PEvent::Message { + topic, + source: Some(source), + transport_source, + timestamp, + data, + } = event + else { + return false; + }; + if topic != CHUNK_PROTOCOL_ID || !swarm::is_swarm_request_bytes(data) { + return false; + } + // No legal swarm request is this long; refuse it before decoding. + if data.len() > ant_protocol::MAX_SWARM_REQUEST_BYTES { + debug!( + "Swarm request from {source} dropped: {} bytes is too long", + data.len() + ); + return true; + } + // The reply goes back on the connection the request came on, named + // as the transport names it. + let Some(channel) = transport_source + .as_ref() + .and_then(MultiAddr::socket_addr) + .map(|address| normalize_socket_addr(address).to_string()) + else { + debug!("Swarm request from {source} dropped: no connection to answer on"); + return true; + }; + let Ok(permit) = Arc::clone(&self.slots).try_acquire_owned() else { + debug!("Swarm request from {source} dropped: swarm slots full"); + return true; + }; + let swarm = Arc::clone(&self.swarm); + let p2p = Arc::clone(&self.p2p); + let stopping = self.stopping.clone(); + let request = SwarmRequest { + source: *source, + channel, + source_ip: transport_source.as_ref().and_then(MultiAddr::ip), + signed_at: *timestamp, + received: Instant::now(), + data: data.clone(), + }; + self.children.spawn(async move { + let _permit = permit; + Self::answer(&swarm, &p2p, request, &stopping).await; + }); + true + } + + /// Answers one swarm request and sends the reply. + async fn answer( + swarm: &SwarmService, + p2p: &Arc, + request: SwarmRequest, + stopping: &CancellationToken, + ) { + let SwarmRequest { + source, + channel, + source_ip, + signed_at, + received, + data, + } = request; + let source = &source; + let message = match ChunkMessage::decode(&data) { + Ok(message) => message, + Err(e) => { + debug!("Undecodable swarm request from {source}: {e}"); + return; + } + }; + // The encoded request is not kept while an announce waits. + drop(data); + let now_unix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |since| since.as_secs()); + let peer = *source.to_bytes(); + let starts = swarm::starts_history(&message.body); + // One for another node gets its error reply without touching the + // client's history here. + if swarm.names_this_node(&message.body) + && !swarm.first_sight( + peer, + source_ip, + message.request_id, + signed_at, + now_unix, + starts, + ) + { + debug!("Swarm request from {source} dropped: stale or seen before"); + return; + } + let session = SwarmSession::Native(peer); + let Some(body) = swarm + .handle(session, source_ip, message.body, stopping, received) + .await + else { + return; + }; + // A client that left (or a node stopping) is not dialled to be told. + if stopping.is_cancelled() || swarm.reply_unwanted(session, received) { + return; + } + let reply = ChunkMessage { + request_id: message.request_id, + body, + }; + match reply.encode() { + // Only on the connection the request came on. A signed message + // is not bound to its connection, so a node that saw one of this + // client's could send it here and have the transport take its + // own connection for the client's too: the reply never goes on + // another of the client's connections, nor dials a client that + // left between the check above and the send. + Ok(bytes) => { + if let Err(e) = p2p + .send_message_on_channel(source, &channel, CHUNK_PROTOCOL_ID, bytes) + .await + { + debug!("Failed to send swarm reply to {source}: {e}"); + } + } + Err(e) => warn!("Failed to encode swarm reply: {e}"), + } + } +} + +/// A swarm request from a native client, as it reached the routing loop. +struct SwarmRequest { + source: PeerId, + /// The connection it came on. + channel: String, + source_ip: Option, + /// The client's signed send time, in Unix seconds. + signed_at: u64, + received: Instant, + data: Vec, +} + +/// How often silent swarm members and expired swarm signals are swept. +const SWARM_SWEEP_INTERVAL: Duration = Duration::from_secs(15); + /// Builder for constructing an Ant node. pub struct NodeBuilder { config: NodeConfig, @@ -224,6 +425,11 @@ impl NodeBuilder { _ => None, }; + // Every node hosts the swarm rendezvous, storing chunks or not. + let swarm = ant_protocol + .as_ref() + .map_or_else(SwarmService::new, |protocol| Arc::clone(protocol.swarm())); + swarm.set_node_id(*p2p_arc.peer_id().to_bytes()); let node = RunningNode { config: self.config, p2p_node: p2p_arc, @@ -232,6 +438,7 @@ impl NodeBuilder { events_rx: Some(events_rx), upgrade_monitor, ant_protocol, + swarm, replication_engine, protocol_task: None, protocol_children: TaskTracker::new(), @@ -569,6 +776,9 @@ pub struct RunningNode { upgrade_monitor: Option, /// ANT protocol handler for chunk storage. ant_protocol: Option>, + /// The swarm rendezvous: the storage handler's, or this node's own when + /// it stores nothing. + swarm: Arc, /// Replication engine (manages neighbor sync, verification, audits). replication_engine: Option, /// Protocol message routing background task. @@ -679,6 +889,7 @@ impl RunningNode { &self.config.root_dir, Arc::clone(&self.p2p_node), self.ant_protocol.clone(), + Arc::clone(&self.swarm), &evm_network, self.shutdown.clone(), None, @@ -1072,19 +1283,29 @@ impl RunningNode { /// Subscribes to P2P events and routes incoming chunk protocol messages /// to the `AntProtocol` handler, sending responses back to the sender. fn start_protocol_routing(&mut self) { - let protocol = match self.ant_protocol { - Some(ref p) => Arc::clone(p), - None => return, - }; + let protocol = self.ant_protocol.clone(); let mut events = self.p2p_node.subscribe_events(); let p2p = Arc::clone(&self.p2p_node); let semaphore = Arc::new(Semaphore::new(64)); let children = self.protocol_children.clone(); let stopping = self.shutdown.clone(); + let swarm = NativeSwarmRouter::start( + Arc::clone(&self.swarm), + Arc::clone(&p2p), + children.clone(), + stopping.clone(), + ); self.protocol_task = Some(tokio::spawn(async move { while let Ok(event) = events.recv().await { + if swarm.route(&event) { + continue; + } + // Chunk requests only where chunks are stored. + let Some(protocol) = protocol.as_ref() else { + continue; + }; if let P2PEvent::Message { topic, source: Some(source), @@ -1101,7 +1322,7 @@ impl RunningNode { if let Some((data_type, response_topic)) = handler_info { debug!("Received {data_type} protocol message from {source}"); let received_at = Instant::now(); - let protocol = Arc::clone(&protocol); + let protocol = Arc::clone(protocol); let p2p = Arc::clone(&p2p); let sem = semaphore.clone(); let stopping = stopping.clone(); diff --git a/src/storage/handler.rs b/src/storage/handler.rs index c4d1b075..dfd14af4 100644 --- a/src/storage/handler.rs +++ b/src/storage/handler.rs @@ -46,6 +46,7 @@ use crate::replication::config::K_BUCKET_SIZE; use crate::replication::fresh::FreshWriteEvent; use crate::storage::traffic::{self, ChunkRequestKind, ChunkResponseKey}; use crate::storage::ChunkStore; +use crate::swarm::{SwarmService, SwarmSession}; use ant_protocol::chunk::{PointerGetResponse, PointerPutResponse}; use bytes::Bytes; use parking_lot::RwLock; @@ -411,6 +412,8 @@ pub struct AntProtocol { /// message to a node that does not keep pointers gets a clean refusal /// rather than a silent drop. pointers: Option, + /// Introduces clients downloading the same file to each other. + swarm: Arc, } impl AntProtocol { @@ -449,6 +452,7 @@ impl AntProtocol { fresh_write_tx: None, p2p_node: RwLock::new(None), pointers: None, + swarm: SwarmService::new(), } } @@ -468,6 +472,39 @@ impl AntProtocol { self.pointers.as_ref() } + /// The swarm rendezvous. Every node runs one: it is how clients + /// downloading the same file find each other. + #[must_use] + pub fn swarm(&self) -> &Arc { + &self.swarm + } + + /// Answers an encoded swarm request from `session`, which arrived from + /// `source_ip`, and returns the encoded reply; `None` if `data` is not a + /// swarm request. An announce may wait for a signal until `shutdown`. + /// Callers route swarm requests here apart from chunk requests, which + /// must not wait behind them (see [`crate::swarm::is_swarm_request_bytes`]). + pub async fn handle_swarm_request( + &self, + session: SwarmSession, + source_ip: Option, + data: &[u8], + shutdown: &tokio_util::sync::CancellationToken, + ) -> Option { + let message = ChunkMessage::decode(data).ok()?; + let body = self + .swarm + .handle(session, source_ip, message.body, shutdown, Instant::now()) + .await?; + ChunkMessage { + request_id: message.request_id, + body, + } + .encode() + .ok() + .map(Bytes::from) + } + /// Attach the node's P2P handle for payment live-DHT checks. /// /// Wires the handle into the payment verifier so payment-proof closeness @@ -475,6 +512,8 @@ impl AntProtocol { /// replaces the verifier handle. pub fn attach_p2p_node(&self, node: Arc) { *self.p2p_node.write() = Some(Arc::clone(&node)); + // Swarm requests must name this node. + self.swarm.set_node_id(*node.peer_id().to_bytes()); if let Some(pointers) = &self.pointers { // Pointers take the same self-closeness gate as chunks, judged at // the pointer address because that is what the network routes on. diff --git a/src/swarm.rs b/src/swarm.rs new file mode 100644 index 00000000..65992ab4 --- /dev/null +++ b/src/swarm.rs @@ -0,0 +1,2889 @@ +//! Swarm rendezvous: the node introduces clients that are downloading the +//! same file, so they can share its records with each other. +//! +//! A client announces itself in a *room* (a file) at the nodes closest to +//! the room's key and hands them WebRTC offers. The node passes each offer to +//! another member and carries the answer back; the two clients then talk +//! directly and records never pass through the node. The node stays the last +//! seed: whatever no client has is fetched from it as before. +//! +//! The rules, all enforced here: +//! - members are named by the node ([`MemberId`]), one per connection and +//! room, so a client cannot speak for another; +//! - an answer is passed on only for an offer this node delivered to the +//! answering member, once, before the offer expires; +//! - only `{offer_id, sdp}` is relayed, never anything else a client sent; +//! - rooms, members, rooms per connection, remembered offers, waiting +//! announces and stored SDP bytes are all capped. The caps are fixed: +//! swarm work is small, and a node that is short of room simply refuses, +//! which leaves its clients fetching from nodes alone. +//! +//! Signals for a member wait in a small inbox until its next announce, which +//! may wait for them (a long poll). A waiting announce holds no lock and +//! ends at the deadline, on the first signal, or when the node shuts down. + +use std::collections::{HashMap, VecDeque}; +use std::net::{IpAddr, Ipv6Addr}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use ant_protocol::chunk::{ChunkMessageBody, ProtocolError}; +use ant_protocol::swarm::{ + constrain_swarm_sdp, is_global_ip, MemberId, Membership, NodeId, OfferId, RoomKey, + SwarmAnnounceRequest, SwarmAnnounceResponse, SwarmAnswerRequest, SwarmAnswerResponse, + SwarmLeaveRequest, SwarmLeaveResponse, SwarmRefusal, SwarmSignal, MAX_SWARM_ICE_SERVERS, + MAX_SWARM_ICE_SERVER_BYTES, MAX_SWARM_INBOX, MAX_SWARM_REPLY_SDP_BYTES, SWARM_SIGNAL_TTL_MS, +}; +use parking_lot::{Mutex, RwLock}; +use rand::seq::SliceRandom; +use tokio::sync::Notify; +use tokio_util::sync::CancellationToken; + +use crate::logging::debug; + +/// Rooms one node hosts. +pub const MAX_ROOMS: usize = 4096; +/// Members in one room. +pub const MAX_MEMBERS_PER_ROOM: usize = 256; +/// Members across every room. +pub const MAX_MEMBERS: usize = 16_384; +/// Rooms one connection may be in. +pub const MAX_ROOMS_PER_SESSION: usize = 4; +/// Members from one host (an IPv4 address or an IPv6 /64) across every +/// room: a native client's identities cost nothing to make, so a host could +/// otherwise take every place with new ones. +pub const MAX_MEMBERS_PER_HOST: usize = 64; +/// Signals waiting for one member. +pub const MAX_INBOX_PER_MEMBER: usize = MAX_SWARM_INBOX; +/// Unanswered offers remembered for one member. Past it, its new offers are +/// not passed on until older ones are answered or expire. +pub const MAX_PENDING_PER_MEMBER: usize = 32; +/// SDP bytes held in every inbox together. +pub const MAX_STORED_SDP_BYTES: usize = 16 * 1024 * 1024; +/// SDP bytes held for the offers and answers of one host's members, so one +/// host cannot take the whole budget and stop every other introduction. +pub const MAX_STORED_SDP_BYTES_PER_HOST: usize = MAX_STORED_SDP_BYTES / 8; +/// Announces waiting for a signal at once; past it, announces reply at once. +pub const MAX_WAITING: usize = 1024; +/// A member that has not announced for this long leaves its room. +pub const MEMBER_TTL: Duration = Duration::from_secs(60); +/// How long a delivered offer waits for its answer, and a signal for its +/// member's next announce: the protocol's, which clients check their own +/// answering time against. +pub const SIGNAL_TTL: Duration = Duration::from_millis(SWARM_SIGNAL_TTL_MS); +/// Client addresses remembered as recent announcers, which the node's STUN +/// responder answers. +pub const MAX_STUN_SOURCES: usize = 16_384; +/// Memberships that ended remembered, in all, per connection and per host. +const MAX_ENDED_MEMBERSHIPS: usize = MAX_MEMBERS; +const MAX_ENDED_PER_SESSION: usize = 64; +const MAX_ENDED_PER_HOST: usize = 4 * MAX_ENDED_PER_SESSION; +/// How long an ended membership is remembered: as long as a request sent +/// before it ended can still be taken. A native request's signed time may be +/// up to [`MAX_REQUEST_SKEW`] ahead of this node's clock and is taken until +/// it is that far behind, so twice the skew, and a little for rounding. +const ENDED_MEMBERSHIP_TTL: Duration = Duration::from_secs(2 * MAX_REQUEST_SKEW.as_secs() + 2); +/// Closed connections remembered, and for how long: longer than any request +/// takes to reach its handler. +const MAX_CLOSED_SESSIONS: usize = MAX_MEMBERS; +const CLOSED_SESSION_TTL: Duration = Duration::from_secs(60); +/// How far a native client's signed timestamp may be from this node's clock. +/// A wide window: clients' clocks drift, and a request outside it only costs +/// the client its swarm, not its reads. +pub const MAX_REQUEST_SKEW: Duration = Duration::from_mins(10); +/// How much older than a native client's newest request one of its requests +/// may be, for requests overtaking each other on the way. +const REQUEST_REORDER: u64 = 5; +/// Request ids remembered per native client within [`REQUEST_REORDER`]. +const MAX_RECENT_REQUESTS: usize = 256; +/// Native clients whose requests are remembered. +const MAX_REQUEST_HISTORIES: usize = MAX_MEMBERS; +/// Native clients of one host whose requests are remembered: identities cost +/// nothing to make, so a host could otherwise push every other client's +/// history out. +const MAX_HISTORIES_PER_HOST: usize = 4 * MAX_MEMBERS_PER_HOST; + +/// A native client's recent requests, to refuse one sent again. Its own +/// requests are the only ones that can change it. +#[derive(Default)] +struct RequestHistory { + /// The newest signed timestamp seen. + latest: u64, + /// Ids signed within [`REQUEST_REORDER`] of `latest`, with their times. + recent: VecDeque<(u64, u64)>, + /// When it was made, among all histories. + number: u64, + /// The client's host, if known. + host: Option, +} + +impl RequestHistory { + /// Records a request; `false` if it was here, or is much older than the + /// newest. + fn note(&mut self, request_id: u64, signed_at: u64) -> bool { + if signed_at.saturating_add(REQUEST_REORDER) < self.latest + || self.recent.iter().any(|(_, id)| *id == request_id) + { + return false; + } + self.latest = self.latest.max(signed_at); + self.recent.push_back((signed_at, request_id)); + let oldest = self.latest.saturating_sub(REQUEST_REORDER); + self.recent.retain(|(at, _)| *at >= oldest); + while self.recent.len() > MAX_RECENT_REQUESTS { + self.recent.pop_front(); + } + true + } +} + +/// Native clients' request histories: at most [`MAX_HISTORIES_PER_HOST`] per +/// host and [`MAX_REQUEST_HISTORIES`] in all, the oldest made forgotten first, +/// and each forgotten once nothing it holds could pass the age check. +#[derive(Default)] +struct Histories { + by_peer: HashMap<[u8; 32], RequestHistory>, + /// The histories in the order they were made, numbered, so an entry for + /// a history forgotten since never removes a newer one of its client. + order: VecDeque<([u8; 32], u64)>, + made: u64, + by_host: HashMap, +} + +impl Histories { + fn forget(&mut self, peer: &[u8; 32]) { + let Some(history) = self.by_peer.remove(peer) else { + return; + }; + if let Some(host) = history.host { + if let Some(count) = self.by_host.get_mut(&host) { + *count = count.saturating_sub(1); + if *count == 0 { + self.by_host.remove(&host); + } + } + } + } + + /// Starts `peer`'s history, unless its host has all it may have. With + /// every place taken, the oldest history goes. + fn start(&mut self, peer: [u8; 32], host: Option) -> bool { + let host_full = host.is_some_and(|host| { + self.by_host.get(&host).copied().unwrap_or(0) >= MAX_HISTORIES_PER_HOST + }); + if host_full { + return false; + } + while self.by_peer.len() >= MAX_REQUEST_HISTORIES { + let Some((oldest, number)) = self.order.pop_front() else { + break; + }; + if self.by_peer.get(&oldest).map(|history| history.number) == Some(number) { + self.forget(&oldest); + } + } + self.made = self.made.wrapping_add(1); + self.by_peer.insert( + peer, + RequestHistory { + number: self.made, + host, + ..RequestHistory::default() + }, + ); + self.order.push_back((peer, self.made)); + if let Some(host) = host { + *self.by_host.entry(host).or_default() += 1; + } + true + } + + /// Forgets the histories whose newest request was signed before + /// `oldest`: nothing they hold could pass the age check now. + fn forget_older_than(&mut self, oldest: u64) { + let idle: Vec<[u8; 32]> = self + .by_peer + .iter() + .filter(|(_, history)| history.latest < oldest) + .map(|(peer, _)| *peer) + .collect(); + for peer in &idle { + self.forget(peer); + } + let by_peer = &self.by_peer; + self.order.retain(|(peer, number)| { + by_peer.get(peer).map(|history| history.number) == Some(*number) + }); + } +} + +/// The connection a request came on. Members are tied to it: a native +/// client's authenticated peer id, or a browser connection. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum SwarmSession { + /// A native client, by its authenticated peer id. + Native([u8; 32]), + /// A browser, by its WebRTC Direct connection number on this node. + Browser(u64), +} + +/// Counters for metrics and logs. +#[derive(Debug, Default)] +pub struct SwarmCounters { + /// Announces received. + pub announces: AtomicU64, + /// Answers received. + pub answers: AtomicU64, + /// Offers handed to a member. + pub offers_delivered: AtomicU64, + /// Offers not handed on: no room in an inbox, a pending cap, or the + /// stored-SDP budget. + pub offers_undelivered: AtomicU64, + /// Answers handed back to the member that made the offer. + pub answers_delivered: AtomicU64, + /// Answers with no matching offer, or that could not be kept. + pub answers_unmatched: AtomicU64, + /// Announces refused by a cap. + pub refused: AtomicU64, + /// Members dropped because they stopped announcing. + pub members_expired: AtomicU64, +} + +/// Counts at one moment. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct SwarmStats { + /// Rooms with at least one member. + pub rooms: usize, + /// Members across all rooms. + pub members: usize, + /// Offers delivered and waiting for an answer. + pub pending_offers: usize, + /// SDP bytes held in inboxes. + pub stored_sdp_bytes: usize, + /// Announces waiting for a signal. + pub waiting: usize, +} + +/// One close of a native client's connection. +#[derive(Debug, Clone, Copy)] +struct Closed { + at: Instant, + number: u64, +} + +/// A membership the client ended, see [`State::ended`]. +#[derive(Clone, Copy)] +struct Ended { + membership: Membership, + /// When it was noted, among all ended memberships. + number: u64, + host: Option, +} + +struct Pending { + offer_id: OfferId, + to: MemberId, + expires: Instant, +} + +struct Queued { + signal: SwarmSignal, + bytes: usize, + /// The host of the member that sent it, whose share of the budget it + /// takes. + host: Option, + expires: Instant, +} + +struct Member { + session: SwarmSession, + /// The client's membership this member is: a newer one takes its place, + /// and only a leave naming it ends it. + membership: Membership, + /// The host it announced from, if known: see [`MAX_MEMBERS_PER_HOST`]. + host: Option, + last_seen: Instant, + inbox: VecDeque, + pending: VecDeque, + /// Woken, for every announce waiting then, when a signal arrives or the + /// member is dropped. + wake: Arc, + /// Whether an announce of this member is waiting: one at a time. + waiting: bool, +} + +#[derive(Default)] +struct Room { + members: HashMap, + by_session: HashMap, +} + +#[derive(Default)] +struct State { + rooms: HashMap, + sessions: HashMap>, + /// When native clients' connections last closed, so a request of theirs + /// still being handled cannot act after the close (a browser connection's + /// own token tells), and the closes in the order they happened, to forget + /// the old ones in constant time. + closed: HashMap, + closed_order: VecDeque<(SwarmSession, Closed)>, + /// Numbers closes, so an old queue entry never removes a newer close. + closes: u64, + /// The newest membership of each connection and room the client ended + /// (left, or replaced by a newer one), numbered, and in the order they + /// ended: an announce naming it or an older one is refused. A membership + /// dropped because its connection closed or it went silent is not ended: + /// the client may still be in it, and carry on over a new connection. + ended: HashMap<(SwarmSession, RoomKey), Ended>, + /// When each was noted, oldest first; an entry whose number is no longer + /// its membership's is stale and skipped. + ended_order: VecDeque<((SwarmSession, RoomKey), u64, Instant)>, + ends: u64, + /// Each connection's and each host's ended memberships, oldest first. + ended_by_session: HashMap>, + ended_by_host: HashMap>, + members: usize, + /// Members from each host, see [`MAX_MEMBERS_PER_HOST`]. + by_host: HashMap, + stored_sdp_bytes: usize, + /// Stored SDP bytes by the host that sent them. + sdp_by_host: HashMap, + pending_offers: usize, +} + +/// The node's swarm rendezvous. +#[derive(Default)] +pub struct SwarmService { + state: Mutex, + waiting: AtomicUsize, + counters: SwarmCounters, + /// This node's STUN servers, as handed to members. + ice_servers: RwLock>, + /// Hosts (IPv4 addresses, IPv6 /64s) that announced recently: the only + /// ones the node's STUN responder answers. A client reaches this list + /// only over a connection it proved (QUIC, or DTLS and the PQ handshake), + /// so a spoofed source cannot turn the responder on a third party. + announcers: RwLock>, + /// Native clients' recent requests: a signed request sent again is + /// refused. + histories: Mutex, + /// This node's peer id: requests naming another node are refused, so a + /// signed request this node saw cannot be replayed at another, nor one + /// another node saw here. Until it is known, every request is refused. + node_id: RwLock>, +} + +impl std::fmt::Debug for SwarmService { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SwarmService") + .field("stats", &self.stats()) + .finish() + } +} + +/// Swarm requests handled at once on the native path. Above +/// [`MAX_WAITING`] so announces that cannot wait still get through. +pub const MAX_SWARM_TASKS: usize = MAX_WAITING + 256; + +// The wire discriminants of the swarm requests in `ChunkMessageBody`, after +// the get-or-closer pair at 14 and 15 (ant-protocol pins them in a test). +const ANNOUNCE_REQUEST: u64 = 16; +const ANSWER_REQUEST: u64 = 18; +const LEAVE_REQUEST: u64 = 20; + +/// Whether an encoded `ChunkMessage` is a swarm request, read from its +/// header alone so the message is not decoded twice: a postcard varint +/// request id, then the body's varint discriminant. +#[must_use] +pub fn is_swarm_request_bytes(data: &[u8]) -> bool { + let mut bytes = data.iter().copied(); + let mut read_varint = || -> Option { + let mut value = 0u64; + for shift in (0..70).step_by(7) { + let byte = bytes.next()?; + value |= u64::from(byte & 0x7f).checked_shl(shift)?; + if byte & 0x80 == 0 { + return Some(value); + } + } + None + }; + let request_id = read_varint(); + let discriminant = read_varint(); + request_id.is_some() + && matches!( + discriminant, + Some(ANNOUNCE_REQUEST | ANSWER_REQUEST | LEAVE_REQUEST) + ) +} + +/// An SDP from a member, as this node passes it on: its candidates may name +/// the member's own host, as this node reaches it, and no other public +/// address, at most `MAX_SWARM_CANDIDATES` of them. Local addresses and mDNS +/// names stay only for a member this node reaches at a local address itself +/// (a network of its own, such as a test network): elsewhere they would let a +/// stranger point readers at their own home networks. The member an SDP goes +/// to sends connectivity checks to each candidate before anything proves who +/// made it. +fn relayable_sdp(sdp: &str, host: Option) -> String { + let local = host.is_some_and(|host| !is_global_ip(host)); + constrain_swarm_sdp(sdp, |ip| match ip { + Some(ip) if is_global_ip(ip) => Some(host_of(ip)) == host, + _ => local, + }) +} + +/// The host `ip` counts as: an IPv4 address, or an IPv6 /64, which one +/// client is usually given whole. +fn host_of(ip: IpAddr) -> IpAddr { + match ip.to_canonical() { + IpAddr::V6(v6) => { + let mut octets = v6.octets(); + if let Some(interface) = octets.get_mut(8..) { + interface.fill(0); + } + IpAddr::V6(Ipv6Addr::from(octets)) + } + v4 @ IpAddr::V4(_) => v4, + } +} + +fn other_node() -> ProtocolError { + ProtocolError::Internal("this swarm request is for another node".into()) +} + +/// Whether a native client's request `body` may start its request history +/// (see [`SwarmService::first_sight`]): an announce, or a leave, which may +/// overtake the announce it ends. +#[must_use] +pub fn starts_history(body: &ChunkMessageBody) -> bool { + matches!( + body, + ChunkMessageBody::SwarmAnnounceRequest(_) | ChunkMessageBody::SwarmLeaveRequest(_) + ) +} + +/// Whether a message body is one this service answers. +#[must_use] +pub fn is_swarm_request(body: &ChunkMessageBody) -> bool { + matches!( + body, + ChunkMessageBody::SwarmAnnounceRequest(_) + | ChunkMessageBody::SwarmAnswerRequest(_) + | ChunkMessageBody::SwarmLeaveRequest(_) + ) +} + +impl State { + /// Remembers a native client's close, forgetting closes past + /// [`CLOSED_SESSION_TTL`] and, past [`MAX_CLOSED_SESSIONS`] in the queue, + /// the oldest; the newest is always kept. + fn note_close(&mut self, session: SwarmSession, now: Instant) { + self.closes = self.closes.wrapping_add(1); + let close = Closed { + at: now, + number: self.closes, + }; + self.closed.insert(session, close); + self.closed_order.push_back((session, close)); + self.forget_closes(now); + } + + fn forget_closes(&mut self, now: Instant) { + while let Some((session, close)) = self.closed_order.front().copied() { + let expired = now.saturating_duration_since(close.at) >= CLOSED_SESSION_TTL; + if !expired && self.closed_order.len() <= MAX_CLOSED_SESSIONS { + break; + } + self.closed_order.pop_front(); + // A later close of the same session keeps its own entry. + if self.closed.get(&session).map(|kept| kept.number) == Some(close.number) { + self.closed.remove(&session); + } + } + } + + /// Whether `session`'s connection closed since `received`. + fn closed_since(&self, session: SwarmSession, received: Instant) -> bool { + self.closed + .get(&session) + .is_some_and(|closed| closed.at >= received) + } + + /// Whether a request of `session` received at `received` must not act: + /// its connection closed since, it is older than any close is + /// remembered, or (a browser's) its connection is closing. + fn stale( + &self, + session: SwarmSession, + received: Instant, + closing: &CancellationToken, + now: Instant, + ) -> bool { + closing.is_cancelled() + || now.saturating_duration_since(received) >= CLOSED_SESSION_TTL + || self.closed_since(session, received) + } + + /// Remembers that `membership` of `session` (from `host`) in `room_key` + /// ended, unless a newer one of theirs already did. A connection's place + /// in a room counts once however often it ends a newer membership there; + /// past its own share, or its host's, the oldest of its own (or its + /// host's) goes, so the newest ends are the ones kept. + fn note_ended( + &mut self, + session: SwarmSession, + host: Option, + room_key: RoomKey, + membership: Membership, + now: Instant, + ) { + let key = (session, room_key); + let number = self.ends.wrapping_add(1); + // A newer end of the same place counts as the newest: it goes to the + // back of every queue, under the host it came from now. + if let Some(kept) = self.ended.get(&key) { + if kept.membership >= membership { + return; + } + self.remove_ended(&key); + } + { + let oldest_of_session = self + .ended_by_session + .get(&session) + .filter(|ended| ended.len() >= MAX_ENDED_PER_SESSION) + .and_then(|ended| ended.front().copied()); + if let Some(room) = oldest_of_session { + self.remove_ended(&(session, room)); + } + let oldest_of_host = host + .and_then(|host| self.ended_by_host.get(&host)) + .filter(|ended| ended.len() >= MAX_ENDED_PER_HOST) + .and_then(|ended| ended.front().copied()); + if let Some(oldest) = oldest_of_host { + self.remove_ended(&oldest); + } + while self.ended.len() >= MAX_ENDED_MEMBERSHIPS && self.forget_oldest_ended() {} + self.ended.insert( + key, + Ended { + membership, + number, + host, + }, + ); + self.ended_by_session + .entry(session) + .or_default() + .push_back(room_key); + if let Some(host) = host { + self.ended_by_host.entry(host).or_default().push_back(key); + } + } + self.ends = number; + self.ended_order.push_back((key, number, now)); + self.forget_ended(now); + } + + fn remove_ended(&mut self, key: &(SwarmSession, RoomKey)) { + let Some(ended) = self.ended.remove(key) else { + return; + }; + let (session, room) = *key; + if let Some(rooms) = self.ended_by_session.get_mut(&session) { + rooms.retain(|kept| *kept != room); + if rooms.is_empty() { + self.ended_by_session.remove(&session); + } + } + if let Some(host) = ended.host { + if let Some(keys) = self.ended_by_host.get_mut(&host) { + keys.retain(|kept| kept != key); + if keys.is_empty() { + self.ended_by_host.remove(&host); + } + } + } + } + + /// Whether the queue entry `(key, number)` is its membership's newest. + fn ended_live(&self, key: &(SwarmSession, RoomKey), number: u64) -> bool { + self.ended.get(key).map(|ended| ended.number) == Some(number) + } + + /// Forgets the oldest ended membership; `false` if there is none. + fn forget_oldest_ended(&mut self) -> bool { + while let Some((key, number, _)) = self.ended_order.pop_front() { + if self.ended_live(&key, number) { + self.remove_ended(&key); + return true; + } + } + false + } + + /// Forgets ended memberships past [`ENDED_MEMBERSHIP_TTL`], and drops + /// stale queue entries, so the queue stays near the memberships' number. + fn forget_ended(&mut self, now: Instant) { + while let Some((key, number, at)) = self.ended_order.front().copied() { + let live = self.ended_live(&key, number); + if live && now.saturating_duration_since(at) < ENDED_MEMBERSHIP_TTL { + break; + } + self.ended_order.pop_front(); + if live { + self.remove_ended(&key); + } + } + if self.ended_order.len() > 2 * self.ended.len() + 1024 { + let ended = &self.ended; + self.ended_order + .retain(|(key, number, _)| ended.get(key).map(|kept| kept.number) == Some(*number)); + } + } + + /// Whether `membership` of `session` in `room_key`, or a newer one of + /// theirs, ended. + fn ended_covers( + &self, + session: SwarmSession, + room_key: &RoomKey, + membership: Membership, + ) -> bool { + self.ended + .get(&(session, *room_key)) + .is_some_and(|ended| ended.membership >= membership) + } + + /// Takes `bytes` of the stored-SDP budget, and of `host`'s share of it, + /// if both fit. + fn reserve(&mut self, bytes: usize, host: Option) -> bool { + let Some(total) = self + .stored_sdp_bytes + .checked_add(bytes) + .filter(|total| *total <= MAX_STORED_SDP_BYTES) + else { + return false; + }; + if let Some(host) = host { + let held = self.sdp_by_host.get(&host).copied().unwrap_or(0); + let Some(share) = held + .checked_add(bytes) + .filter(|share| *share <= MAX_STORED_SDP_BYTES_PER_HOST) + else { + return false; + }; + self.sdp_by_host.insert(host, share); + } + self.stored_sdp_bytes = total; + true + } + + fn release(&mut self, bytes: usize, host: Option) { + self.stored_sdp_bytes = self.stored_sdp_bytes.saturating_sub(bytes); + if let Some(host) = host { + if let Some(share) = self.sdp_by_host.get_mut(&host) { + *share = share.saturating_sub(bytes); + if *share == 0 { + self.sdp_by_host.remove(&host); + } + } + } + } + + fn release_all(&mut self, freed: Vec<(usize, Option)>) { + for (bytes, host) in freed { + self.release(bytes, host); + } + } + + /// Forgets a member and everything it held, and every signal of its + /// still queued for others and every offer made to it: none can be + /// answered now, so none keeps an inbox place or the budget. + fn drop_member(&mut self, room_key: &RoomKey, member_id: &MemberId) { + let Some(room) = self.rooms.get_mut(room_key) else { + return; + }; + let Some(member) = room.members.remove(member_id) else { + return; + }; + room.by_session.remove(&member.session); + let mut freed: Vec<(usize, Option)> = member + .inbox + .iter() + .map(|queued| (queued.bytes, queued.host)) + .collect(); + let mut pending = member.pending.len(); + for other in room.members.values_mut() { + other.inbox.retain(|queued| { + let (SwarmSignal::Offer { from, .. } | SwarmSignal::Answer { from, .. }) = + &queued.signal; + let keep = from != member_id; + if !keep { + freed.push((queued.bytes, queued.host)); + } + keep + }); + let before = other.pending.len(); + other.pending.retain(|offer| offer.to != *member_id); + pending += before - other.pending.len(); + } + let empty = room.members.is_empty(); + if empty { + self.rooms.remove(room_key); + } + if let Some(rooms) = self.sessions.get_mut(&member.session) { + rooms.retain(|room| room != room_key); + if rooms.is_empty() { + self.sessions.remove(&member.session); + } + } + self.release_all(freed); + self.pending_offers = self.pending_offers.saturating_sub(pending); + self.members = self.members.saturating_sub(1); + if let Some(host) = member.host { + if let Some(count) = self.by_host.get_mut(&host) { + *count = count.saturating_sub(1); + if *count == 0 { + self.by_host.remove(&host); + } + } + } + // Waiting announces for this member end now rather than at their deadline. + member.wake.notify_waiters(); + } +} + +impl SwarmService { + /// A service with no rooms. + #[must_use] + pub fn new() -> Arc { + Arc::new(Self::default()) + } + + /// Counters for metrics. + pub fn counters(&self) -> &SwarmCounters { + &self.counters + } + + /// Whether a reply to a request of `session` received at `received` + /// should be dropped: the connection closed since, so sending it would + /// only make the node dial a client that left. + pub fn reply_unwanted(&self, session: SwarmSession, received: Instant) -> bool { + self.state.lock().closed_since(session, received) + } + + /// Sets this node's peer id, which every request it answers must name. + pub fn set_node_id(&self, node_id: NodeId) { + *self.node_id.write() = Some(node_id); + } + + /// Sets the STUN servers handed to members (this node's own, once its + /// public address is known). + pub fn set_ice_servers(&self, servers: Vec) { + *self.ice_servers.write() = servers + .into_iter() + .filter(|server| server.len() <= MAX_SWARM_ICE_SERVER_BYTES) + .take(MAX_SWARM_ICE_SERVERS) + .collect(); + } + + /// Whether the STUN responder should answer `ip`: it announced in a room + /// within [`MEMBER_TTL`]. + pub fn stun_allows(&self, ip: IpAddr, now: Instant) -> bool { + self.announcers + .read() + .get(&host_of(ip)) + .is_some_and(|seen| now.saturating_duration_since(*seen) < MEMBER_TTL) + } + + /// Takes a place among the waiting announces, if there is one. + fn admit_waiter(&self) -> bool { + let mut waiting = self.waiting.load(Ordering::Acquire); + loop { + if waiting >= MAX_WAITING { + return false; + } + match self.waiting.compare_exchange_weak( + waiting, + waiting + 1, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return true, + Err(actual) => waiting = actual, + } + } + } + + fn note_announcer(&self, ip: IpAddr, now: Instant) { + // By host (an IPv4 address, or an IPv6 /64, as the STUN responder's + // own limits count), so one host cannot fill the list by announcing + // from address after address of its own. + let ip = host_of(ip); + // Most announces refresh a known address: a read lock does, so the + // STUN responder's lookups on the packet path rarely wait. + if let Some(seen) = self.announcers.read().get(&ip) { + if now.saturating_duration_since(*seen) < MEMBER_TTL / 2 { + return; + } + } + let mut announcers = self.announcers.write(); + if !announcers.contains_key(&ip) && announcers.len() >= MAX_STUN_SOURCES { + announcers.retain(|_, seen| now.saturating_duration_since(*seen) < MEMBER_TTL); + if announcers.len() >= MAX_STUN_SOURCES { + return; + } + } + announcers.insert(ip, now); + } + + /// Whether a native client's signed swarm request, from `source_ip`, is + /// new: signed within [`MAX_REQUEST_SKEW`] of `now_unix`, not more than a + /// few seconds older than the client's newest, and not seen before. A + /// signed request is authentic but not bound to the connection it came + /// on, so a node that saw it could send it again in the client's name. + /// Only an announce or a leave (`starts`) makes a client's history: an + /// answer comes from a member, which announced first, while a leave may + /// overtake the announce it ends. + pub fn first_sight( + &self, + peer: [u8; 32], + source_ip: Option, + request_id: u64, + signed_at: u64, + now_unix: u64, + starts: bool, + ) -> bool { + let skew = MAX_REQUEST_SKEW.as_secs(); + if signed_at.saturating_add(skew) < now_unix || signed_at > now_unix.saturating_add(skew) { + return false; + } + let mut histories = self.histories.lock(); + if !histories.by_peer.contains_key(&peer) + && (!starts || !histories.start(peer, source_ip.map(host_of))) + { + return false; + } + histories + .by_peer + .get_mut(&peer) + .is_some_and(|history| history.note(request_id, signed_at)) + } + + /// Counts for metrics. + pub fn stats(&self) -> SwarmStats { + let state = self.state.lock(); + SwarmStats { + rooms: state.rooms.len(), + members: state.members, + pending_offers: state.pending_offers, + stored_sdp_bytes: state.stored_sdp_bytes, + waiting: self.waiting.load(Ordering::Relaxed), + } + } + + /// Whether swarm request `body` names this node. A native client's request + /// for another node is answered with an error and leaves its history + /// alone: a node that saw a client's request could otherwise send it on + /// here to make the client's later requests here look stale. + pub fn names_this_node(&self, body: &ChunkMessageBody) -> bool { + let node = match body { + ChunkMessageBody::SwarmAnnounceRequest(request) => &request.node, + ChunkMessageBody::SwarmAnswerRequest(request) => &request.node, + ChunkMessageBody::SwarmLeaveRequest(request) => &request.node, + _ => return false, + }; + self.node_id.read().as_ref() == Some(node) + } + + /// Answers one swarm request from `session`, which arrived from + /// `source_ip` at `received`, or returns `None` if `body` is not a swarm + /// request. A native client's announce may wait for a signal; it ends at + /// its deadline, on the first signal, when its member is dropped, or when + /// `shutdown` is cancelled. A request received before its connection + /// closed (or, for a browser, whose `shutdown` is cancelled) joins nothing. + pub async fn handle( + &self, + session: SwarmSession, + source_ip: Option, + body: ChunkMessageBody, + shutdown: &CancellationToken, + received: Instant, + ) -> Option { + let for_this_node = |node: &NodeId| self.node_id.read().as_ref() == Some(node); + match body { + ChunkMessageBody::SwarmAnnounceRequest(request) => { + if !for_this_node(&request.node) { + return Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Error(other_node()), + )); + } + Some(ChunkMessageBody::SwarmAnnounceResponse( + self.announce(session, source_ip, request, shutdown, received) + .await, + )) + } + ChunkMessageBody::SwarmAnswerRequest(request) => { + if !for_this_node(&request.node) { + return Some(ChunkMessageBody::SwarmAnswerResponse( + SwarmAnswerResponse::Error(other_node()), + )); + } + Some(ChunkMessageBody::SwarmAnswerResponse(self.answer( + session, + source_ip.map(host_of), + &request, + received, + shutdown, + ))) + } + ChunkMessageBody::SwarmLeaveRequest(request) => { + if !for_this_node(&request.node) { + return Some(ChunkMessageBody::SwarmLeaveResponse( + SwarmLeaveResponse::Error(other_node()), + )); + } + Some(ChunkMessageBody::SwarmLeaveResponse( + self.leave(session, source_ip, &request, received, shutdown), + )) + } + _ => None, + } + } + + async fn announce( + &self, + session: SwarmSession, + source_ip: Option, + request: SwarmAnnounceRequest, + shutdown: &CancellationToken, + received: Instant, + ) -> SwarmAnnounceResponse { + self.counters.announces.fetch_add(1, Ordering::Relaxed); + if let Err(error) = request.validate() { + return SwarmAnnounceResponse::Error(error); + } + // A node browsers cannot reach (no WebRTC Direct endpoint, so no STUN + // server to name either) is no meeting place: native readers that met + // here would never meet the browsers, which look only for nodes they + // can reach, and would get no STUN from it. Refusing sends every + // client on to the closest nodes that have both, at once. + if self.ice_servers.read().is_empty() { + self.counters.refused.fetch_add(1, Ordering::Relaxed); + return SwarmAnnounceResponse::Refused(SwarmRefusal::NotHosting); + } + let mut request = request; + let host = source_ip.map(host_of); + for offer in &mut request.offers { + offer.sdp = relayable_sdp(&offer.sdp, host); + } + let now = Instant::now(); + let room = request.room; + // A browser's waiting request would hold one of its session's few + // request slots, and its byte reservation, at this node, delaying its + // reads: a browser never waits, and asks again instead. + let wait = match session { + SwarmSession::Browser(_) => Duration::ZERO, + SwarmSession::Native(_) => Duration::from_millis(u64::from(request.wait_ms)), + }; + let joined = match self.join_and_offer(session, source_ip, request, now, received, shutdown) + { + Ok(joined) => joined, + Err(refusal) => { + self.counters.refused.fetch_add(1, Ordering::Relaxed); + return SwarmAnnounceResponse::Refused(refusal); + } + }; + if let Some(ip) = source_ip { + self.note_announcer(ip, now); + } + + // Nothing waiting yet: hold the reply until a signal comes, the + // deadline passes, the member is dropped, or the node stops. One + // announce per member waits; no lock is held meanwhile. + if !wait.is_zero() { + if let Some(_turn) = self.wait_turn(&room, &joined.member) { + let deadline = tokio::time::Instant::now() + wait; + loop { + // Registered before the inbox is looked at, so a signal + // that arrives in between still wakes it. + let notified = joined.wake.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if self.inbox_waits(&room, &joined.member) != Some(true) { + break; + } + tokio::select! { + () = &mut notified => {} + () = tokio::time::sleep_until(deadline) => break, + () = shutdown.cancelled() => break, + } + } + } + } + + let mut state = self.state.lock(); + // A member replaced or dropped while the announce waited is not + // reported as joined. + let current = state + .rooms + .get(&room) + .and_then(|kept| kept.by_session.get(&session)) + == Some(&joined.member); + let drained = + current.then(|| Self::drain(&mut state, &room, &joined.member, Instant::now())); + drop(state); + let Some((members, inbox)) = drained else { + return SwarmAnnounceResponse::Refused(SwarmRefusal::Superseded); + }; + SwarmAnnounceResponse::Joined { + member: joined.member, + members, + offers_delivered: joined.offers_delivered, + inbox, + ice_servers: self.ice_servers.read().clone(), + } + } + + /// Whether the member is still there with nothing for it yet (`None` if + /// it was dropped). + fn inbox_waits(&self, room_key: &RoomKey, member_id: &MemberId) -> Option { + self.state + .lock() + .rooms + .get(room_key) + .and_then(|room| room.members.get(member_id)) + .map(|member| member.inbox.is_empty()) + } + + /// A place among the waiting announces for this member, unless one of + /// its announces waits already or every place is taken. + fn wait_turn<'a>(&'a self, room_key: &RoomKey, member_id: &MemberId) -> Option> { + if !Self::claim_wait(&mut self.state.lock(), room_key, member_id, || { + self.admit_waiter() + }) { + return None; + } + Some(WaitTurn { + service: self, + room: *room_key, + member: *member_id, + }) + } + + /// Marks the member as waiting, if none of its announces is and `admit` + /// gives a place. + fn claim_wait( + state: &mut State, + room_key: &RoomKey, + member_id: &MemberId, + admit: impl FnOnce() -> bool, + ) -> bool { + let Some(member) = state + .rooms + .get_mut(room_key) + .and_then(|room| room.members.get_mut(member_id)) + else { + return false; + }; + if member.waiting || !admit() { + return false; + } + member.waiting = true; + true + } + + /// Joins (or refreshes) `session` in the room, and hands each offer to a + /// different member picked at random. + fn join_and_offer( + &self, + session: SwarmSession, + source_ip: Option, + request: SwarmAnnounceRequest, + now: Instant, + received: Instant, + shutdown: &CancellationToken, + ) -> Result { + let room_key = request.room; + let mut state = self.state.lock(); + // Under the lock that closing takes too: either the close comes after + // and removes this membership, or it came before and is seen here. + if state.stale(session, received, shutdown, now) { + return Err(SwarmRefusal::Busy); + } + let member_id = Self::admit_member( + &mut state, + session, + source_ip, + room_key, + request.membership, + now, + )?; + let wake = Self::refresh_member(&mut state, &room_key, &member_id, now) + .ok_or(SwarmRefusal::Busy)?; + let offers_delivered = self.deliver_offers(&mut state, &room_key, member_id, request, now); + drop(state); + Ok(Joined { + member: member_id, + offers_delivered, + wake, + }) + } + + /// The member `session` already is in the room for `membership`, or a + /// new one if the caps allow. A newer membership takes the place of the + /// member there; an older one is refused, as the client has left it. + fn admit_member( + state: &mut State, + session: SwarmSession, + source_ip: Option, + room_key: RoomKey, + membership: Membership, + now: Instant, + ) -> Result { + let current = state.rooms.get(&room_key).and_then(|room| { + let id = room.by_session.get(&session).copied()?; + room.members.get(&id).map(|member| (id, member.membership)) + }); + if let Some((id, kept)) = current { + match membership.cmp(&kept) { + std::cmp::Ordering::Equal => { + // The same membership over another connection (a native + // client's address changed) counts against its new host. + Self::rehost_member(state, &room_key, &id, source_ip.map(host_of))?; + return Ok(id); + } + std::cmp::Ordering::Less => return Err(SwarmRefusal::Superseded), + std::cmp::Ordering::Greater => { + state.note_ended(session, source_ip.map(host_of), room_key, kept, now); + state.drop_member(&room_key, &id); + } + } + } + // A membership that ended (or an older one) does not come back, however + // late its announce arrives. + if state.ended_covers(session, &room_key, membership) { + return Err(SwarmRefusal::Superseded); + } + if !state.rooms.contains_key(&room_key) && state.rooms.len() >= MAX_ROOMS { + return Err(SwarmRefusal::TooManyRooms); + } + let room_full = state + .rooms + .get(&room_key) + .is_some_and(|room| room.members.len() >= MAX_MEMBERS_PER_ROOM); + if room_full || state.members >= MAX_MEMBERS { + return Err(SwarmRefusal::RoomFull); + } + let host = source_ip.map(host_of); + let host_full = host.is_some_and(|host| { + state.by_host.get(&host).copied().unwrap_or(0) >= MAX_MEMBERS_PER_HOST + }); + if host_full || state.sessions.get(&session).map_or(0, Vec::len) >= MAX_ROOMS_PER_SESSION { + return Err(SwarmRefusal::TooManyRoomsForPeer); + } + // Every older membership of the client's here is replaced by this + // one: none comes back, even after this one goes silent. + if let Some(older) = membership.checked_sub(1) { + state.note_ended(session, host, room_key, older, now); + } + let member_id: MemberId = rand::random(); + let room = state.rooms.entry(room_key).or_default(); + room.by_session.insert(session, member_id); + room.members.insert( + member_id, + Member { + session, + membership, + host, + last_seen: now, + inbox: VecDeque::new(), + pending: VecDeque::new(), + wake: Arc::new(Notify::new()), + waiting: false, + }, + ); + state.sessions.entry(session).or_default().push(room_key); + state.members += 1; + if let Some(host) = host { + *state.by_host.entry(host).or_default() += 1; + } + Ok(member_id) + } + + /// Counts member `id` against `host` from now on, if it moved there and + /// that host has room for one more member; its signals already held stay + /// counted where they were taken. + fn rehost_member( + state: &mut State, + room_key: &RoomKey, + id: &MemberId, + host: Option, + ) -> Result<(), SwarmRefusal> { + let Some(old) = state + .rooms + .get(room_key) + .and_then(|room| room.members.get(id)) + .map(|member| member.host) + else { + return Ok(()); + }; + if old == host { + return Ok(()); + } + if host.is_some_and(|host| { + state.by_host.get(&host).copied().unwrap_or(0) >= MAX_MEMBERS_PER_HOST + }) { + return Err(SwarmRefusal::TooManyRoomsForPeer); + } + if let Some(old) = old { + if let Some(count) = state.by_host.get_mut(&old) { + *count = count.saturating_sub(1); + if *count == 0 { + state.by_host.remove(&old); + } + } + } + if let Some(host) = host { + *state.by_host.entry(host).or_default() += 1; + } + if let Some(member) = state + .rooms + .get_mut(room_key) + .and_then(|room| room.members.get_mut(id)) + { + member.host = host; + } + Ok(()) + } + + /// Marks the member as seen and forgets its offers nobody answered in + /// time. Returns its wake-up handle. + fn refresh_member( + state: &mut State, + room_key: &RoomKey, + member_id: &MemberId, + now: Instant, + ) -> Option> { + let member = state + .rooms + .get_mut(room_key) + .and_then(|room| room.members.get_mut(member_id))?; + member.last_seen = now; + let before = member.pending.len(); + member.pending.retain(|pending| pending.expires > now); + let expired = before - member.pending.len(); + let refreshed = Arc::clone(&member.wake); + state.pending_offers = state.pending_offers.saturating_sub(expired); + Some(refreshed) + } + + /// Hands each offer, in order, to a different other member picked at + /// random, and returns how many went: always the first ones. A member + /// whose inbox is full is passed over; the sender's cap on unanswered + /// offers, or the stored-SDP budget, stops the rest. + fn deliver_offers( + &self, + state: &mut State, + room_key: &RoomKey, + sender_id: MemberId, + request: SwarmAnnounceRequest, + now: Instant, + ) -> u32 { + if request.offers.is_empty() { + return 0; + } + let mut candidates: Vec = state + .rooms + .get(room_key) + .map(|room| { + room.members + .keys() + .filter(|other| **other != sender_id) + .copied() + .collect() + }) + .unwrap_or_default(); + candidates.shuffle(&mut rand::thread_rng()); + let deadline = now.checked_add(SIGNAL_TTL).unwrap_or(now); + let total = request.offers.len(); + let mut offers = request.offers.into_iter().peekable(); + let mut delivered = 0u32; + // Offers go in order, so the client knows which went: the first + // `delivered`. A recipient with a full inbox is passed over; the + // sender's pending cap or the SDP budget stops the rest. + for target in candidates { + // What the stored SDP holds, which its length alone may not say. + let Some(bytes) = offers.peek().map(|offer| offer.sdp.capacity()) else { + break; + }; + let Some(room) = state.rooms.get(room_key) else { + break; + }; + let sender_ok = room + .members + .get(&sender_id) + .is_some_and(|member| member.pending.len() < MAX_PENDING_PER_MEMBER); + if !sender_ok { + break; + } + let target_ok = room + .members + .get(&target) + .is_some_and(|member| member.inbox.len() < MAX_INBOX_PER_MEMBER); + if !target_ok { + continue; + } + let host = room.members.get(&sender_id).and_then(|member| member.host); + if !state.reserve(bytes, host) { + break; + } + let (Some(offer), Some(room)) = (offers.next(), state.rooms.get_mut(room_key)) else { + state.release(bytes, host); + break; + }; + let Some(recipient) = room.members.get_mut(&target) else { + state.release(bytes, host); + break; + }; + recipient.inbox.push_back(Queued { + signal: SwarmSignal::Offer { + from: sender_id, + offer_id: offer.offer_id, + sdp: offer.sdp, + }, + bytes, + host, + expires: deadline, + }); + recipient.wake.notify_waiters(); + if let Some(sender) = room.members.get_mut(&sender_id) { + sender.pending.push_back(Pending { + offer_id: offer.offer_id, + to: target, + expires: deadline, + }); + } + state.pending_offers += 1; + delivered += 1; + } + let undelivered = total.saturating_sub(usize::try_from(delivered).unwrap_or(usize::MAX)); + self.counters.offers_undelivered.fetch_add( + u64::try_from(undelivered).unwrap_or(u64::MAX), + Ordering::Relaxed, + ); + self.counters + .offers_delivered + .fetch_add(u64::from(delivered), Ordering::Relaxed); + delivered + } + + /// Takes the member's signals that fit in one reply, oldest first. + fn drain( + state: &mut State, + room_key: &RoomKey, + member_id: &MemberId, + now: Instant, + ) -> (u32, Vec) { + let mut signals = Vec::new(); + let mut freed = Vec::new(); + let mut reply_bytes = 0usize; + let members = state.rooms.get(room_key).map_or(0, |room| { + u32::try_from(room.members.len()).unwrap_or(u32::MAX) + }); + if let Some(member) = state + .rooms + .get_mut(room_key) + .and_then(|room| room.members.get_mut(member_id)) + { + while let Some(front) = member.inbox.front() { + if front.expires <= now { + freed.push((front.bytes, front.host)); + member.inbox.pop_front(); + continue; + } + let fits = signals.len() < MAX_SWARM_INBOX + && reply_bytes + front.bytes <= MAX_SWARM_REPLY_SDP_BYTES; + // The first signal always goes, so one large SDP cannot wedge the inbox. + if !fits && !signals.is_empty() { + break; + } + let Some(queued) = member.inbox.pop_front() else { + break; + }; + reply_bytes += queued.bytes; + freed.push((queued.bytes, queued.host)); + signals.push(queued.signal); + } + } + state.release_all(freed); + (members, signals) + } + + /// `host` is the one this request came from: the answer's candidates are + /// kept to it, and its bytes counted against it. + fn answer( + &self, + session: SwarmSession, + host: Option, + request: &SwarmAnswerRequest, + received: Instant, + closing: &CancellationToken, + ) -> SwarmAnswerResponse { + self.counters.answers.fetch_add(1, Ordering::Relaxed); + if let Err(error) = request.validate() { + return SwarmAnswerResponse::Error(error); + } + let now = Instant::now(); + let delivered = { + let mut state = self.state.lock(); + !state.stale(session, received, closing, now) + && Self::relay_answer(&mut state, session, host, request, now) + }; + let counter = if delivered { + &self.counters.answers_delivered + } else { + &self.counters.answers_unmatched + }; + counter.fetch_add(1, Ordering::Relaxed); + if delivered { + SwarmAnswerResponse::Delivered + } else { + SwarmAnswerResponse::Unmatched + } + } + + /// Queues the answer for the member that made the offer, if that member + /// delivered the offer to this one and it has not expired or been + /// answered. The offer is consumed whether or not the answer can be kept. + fn relay_answer( + state: &mut State, + session: SwarmSession, + host: Option, + request: &SwarmAnswerRequest, + now: Instant, + ) -> bool { + let Some(answerer) = state + .rooms + .get(&request.room) + .and_then(|room| room.by_session.get(&session).copied()) + else { + return false; + }; + let Some(offerer) = state + .rooms + .get_mut(&request.room) + .and_then(|room| room.members.get_mut(&request.to)) + else { + return false; + }; + let Some(position) = offerer.pending.iter().position(|pending| { + pending.offer_id == request.offer_id && pending.to == answerer && pending.expires > now + }) else { + return false; + }; + offerer.pending.remove(position); + let has_room = offerer.inbox.len() < MAX_INBOX_PER_MEMBER; + state.pending_offers = state.pending_offers.saturating_sub(1); + let sdp = relayable_sdp(&request.sdp, host); + let bytes = sdp.capacity(); + if !has_room || !state.reserve(bytes, host) { + return false; + } + let Some(offerer) = state + .rooms + .get_mut(&request.room) + .and_then(|room| room.members.get_mut(&request.to)) + else { + state.release(bytes, host); + return false; + }; + offerer.inbox.push_back(Queued { + signal: SwarmSignal::Answer { + from: answerer, + offer_id: request.offer_id, + sdp, + }, + bytes, + host, + expires: now.checked_add(SIGNAL_TTL).unwrap_or(now), + }); + offerer.wake.notify_waiters(); + true + } + + fn leave( + &self, + session: SwarmSession, + source_ip: Option, + request: &SwarmLeaveRequest, + received: Instant, + closing: &CancellationToken, + ) -> SwarmLeaveResponse { + let mut state = self.state.lock(); + // A leave from before a close would remove a later connection's + // membership; the close already removed its own. + if state.stale(session, received, closing, Instant::now()) { + return SwarmLeaveResponse::Left; + } + // The membership the leave names ends, and so does an older one in + // its place; a newer one stays. It is remembered as ended even if its + // announce has not arrived yet, so that announce cannot join. + let now = Instant::now(); + let member = state.rooms.get(&request.room).and_then(|room| { + let id = room.by_session.get(&session).copied()?; + room.members + .get(&id) + .is_some_and(|member| member.membership <= request.membership) + .then_some(id) + }); + if let Some(member) = member { + state.drop_member(&request.room, &member); + } + state.note_ended( + session, + source_ip.map(host_of), + request.room, + request.membership, + now, + ); + SwarmLeaveResponse::Left + } + + /// Removes every membership of a connection that closed, and remembers + /// the close a while, so a request of the connection still being handled + /// cannot join again. + pub fn session_closed(&self, session: SwarmSession) { + let mut state = self.state.lock(); + if let SwarmSession::Native(_) = session { + state.note_close(session, Instant::now()); + } + let rooms = state.sessions.get(&session).cloned().unwrap_or_default(); + for room_key in rooms { + let member = state + .rooms + .get(&room_key) + .and_then(|room| room.by_session.get(&session).copied()); + if let Some(member) = member { + state.drop_member(&room_key, &member); + } + } + } + + /// Drops members that stopped announcing, and signals and offers that + /// expired. + pub fn sweep(&self, now: Instant) { + let mut guard = self.state.lock(); + let state = &mut *guard; + let mut silent = Vec::new(); + let mut freed = Vec::new(); + let mut expired_offers = 0usize; + for (room_key, room) in &mut state.rooms { + for (member_id, member) in &mut room.members { + if now.saturating_duration_since(member.last_seen) >= MEMBER_TTL { + silent.push((*room_key, *member_id)); + continue; + } + member.inbox.retain(|queued| { + let keep = queued.expires > now; + if !keep { + freed.push((queued.bytes, queued.host)); + } + keep + }); + let before = member.pending.len(); + member.pending.retain(|pending| pending.expires > now); + expired_offers += before - member.pending.len(); + } + } + state.release_all(freed); + state.pending_offers = state.pending_offers.saturating_sub(expired_offers); + for (room_key, member_id) in &silent { + state.drop_member(room_key, member_id); + } + state.forget_closes(now); + state.forget_ended(now); + drop(guard); + let now_unix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |since| since.as_secs()); + self.histories + .lock() + .forget_older_than(now_unix.saturating_sub(MAX_REQUEST_SKEW.as_secs())); + self.announcers + .write() + .retain(|_, seen| now.saturating_duration_since(*seen) < MEMBER_TTL); + if !silent.is_empty() { + self.counters.members_expired.fetch_add( + u64::try_from(silent.len()).unwrap_or(u64::MAX), + Ordering::Relaxed, + ); + debug!(expired = silent.len(), "swarm members expired"); + } + } +} + +struct Joined { + member: MemberId, + offers_delivered: u32, + wake: Arc, +} + +/// A waiting announce's place, given back however the wait ends. +struct WaitTurn<'a> { + service: &'a SwarmService, + room: RoomKey, + member: MemberId, +} + +impl Drop for WaitTurn<'_> { + fn drop(&mut self) { + self.service.waiting.fetch_sub(1, Ordering::AcqRel); + if let Some(member) = self + .service + .state + .lock() + .rooms + .get_mut(&self.room) + .and_then(|room| room.members.get_mut(&self.member)) + { + member.waiting = false; + } + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +mod tests { + use super::*; + use ant_protocol::swarm::{SwarmOffer, MAX_SWARM_SDP_BYTES}; + + /// The node the tests' requests are for. + const NODE: NodeId = [0xee; 32]; + + /// A node's STUN server, as one with a browser endpoint names it. + const STUN: &str = "stun:203.0.113.9:10000"; + + fn service() -> Arc { + let service = SwarmService::new(); + service.set_node_id(NODE); + service.set_ice_servers(vec![STUN.into()]); + service + } + + fn announce(room: u8, offers: &[u8], wait_ms: u32) -> ChunkMessageBody { + ChunkMessageBody::SwarmAnnounceRequest(SwarmAnnounceRequest { + room: [room; 32], + node: NODE, + membership: 1, + offers: offers + .iter() + .map(|id| SwarmOffer { + offer_id: [*id; 16], + sdp: format!("v=0 offer {id}"), + }) + .collect(), + wait_ms, + }) + } + + struct Reply { + member: MemberId, + members: u32, + delivered: u32, + inbox: Vec, + } + + async fn call(service: &SwarmService, session: SwarmSession, body: ChunkMessageBody) -> Reply { + let token = CancellationToken::new(); + match service + .handle(session, None, body, &token, Instant::now()) + .await + { + Some(ChunkMessageBody::SwarmAnnounceResponse(SwarmAnnounceResponse::Joined { + member, + members, + offers_delivered, + inbox, + .. + })) => Reply { + member, + members, + delivered: offers_delivered, + inbox, + }, + other => panic!("unexpected reply {other:?}"), + } + } + + fn answer(room: u8, to: MemberId, offer: u8) -> ChunkMessageBody { + ChunkMessageBody::SwarmAnswerRequest(SwarmAnswerRequest { + room: [room; 32], + node: NODE, + to, + offer_id: [offer; 16], + sdp: "v=0 answer".into(), + }) + } + + async fn answer_reply( + service: &SwarmService, + session: SwarmSession, + body: ChunkMessageBody, + ) -> SwarmAnswerResponse { + match service + .handle( + session, + None, + body, + &CancellationToken::new(), + Instant::now(), + ) + .await + { + Some(ChunkMessageBody::SwarmAnswerResponse(response)) => response, + other => panic!("unexpected reply {other:?}"), + } + } + + const A: SwarmSession = SwarmSession::Browser(1); + const B: SwarmSession = SwarmSession::Native([2; 32]); + const C: SwarmSession = SwarmSession::Browser(3); + + #[tokio::test] + async fn an_offer_reaches_another_member_and_its_answer_comes_back() { + let service = service(); + let b = call(&service, B, announce(1, &[], 0)).await; + assert_eq!(b.members, 1); + let a = call(&service, A, announce(1, &[30, 31], 0)).await; + assert_eq!((a.members, a.delivered), (2, 1)); + assert_ne!(a.member, b.member); + + let inbox = call(&service, B, announce(1, &[], 0)).await.inbox; + let [SwarmSignal::Offer { from, offer_id, .. }] = inbox.as_slice() else { + panic!("expected one offer: {inbox:?}"); + }; + assert_eq!(*from, a.member); + let offer = offer_id.first().copied().unwrap_or(0); + assert_eq!( + answer_reply(&service, B, answer(1, a.member, offer)).await, + SwarmAnswerResponse::Delivered + ); + let inbox = call(&service, A, announce(1, &[], 0)).await.inbox; + assert!(matches!( + inbox.as_slice(), + [SwarmSignal::Answer { from, .. }] if *from == b.member + )); + // Answered once only. + assert_eq!( + answer_reply(&service, B, answer(1, a.member, offer)).await, + SwarmAnswerResponse::Unmatched + ); + assert_eq!(service.stats().pending_offers, 0); + assert_eq!(service.stats().stored_sdp_bytes, 0); + } + + /// An SDP naming `own` (its sender's public address), a stranger's public + /// address and a local one. + fn pointing(own: &str) -> String { + format!( + "v=0\r\n\ + a=candidate:1 1 udp 1 {own} 5000 typ srflx raddr 0.0.0.0 rport 0\r\n\ + a=candidate:2 1 udp 1 9.9.9.99 5001 typ host\r\n\ + a=candidate:3 1 udp 1 192.168.1.4 5002 typ host\r\n" + ) + } + + /// Sends `body` from `session` at `ip`; the inbox and member id of an + /// announce's reply, if it was one. + async fn from( + service: &SwarmService, + session: SwarmSession, + ip: &str, + body: ChunkMessageBody, + ) -> Option<(MemberId, Vec)> { + let ip: IpAddr = ip.parse().expect("ip"); + match service + .handle( + session, + Some(ip), + body, + &CancellationToken::new(), + Instant::now(), + ) + .await + { + Some(ChunkMessageBody::SwarmAnnounceResponse(SwarmAnnounceResponse::Joined { + member, + inbox, + .. + })) => Some((member, inbox)), + _ => None, + } + } + + #[tokio::test] + async fn a_relayed_sdp_names_no_public_address_but_its_senders() { + let service = service(); + from(&service, B, "1.1.1.5", announce(1, &[], 0)).await; + let mut offer = announce(1, &[30], 0); + if let ChunkMessageBody::SwarmAnnounceRequest(request) = &mut offer { + for sent in &mut request.offers { + sent.sdp = pointing("1.1.1.4"); + } + } + let (a, _) = from(&service, A, "1.1.1.4", offer).await.expect("joined"); + let (_, inbox) = from(&service, B, "1.1.1.5", announce(1, &[], 0)) + .await + .expect("joined"); + let [SwarmSignal::Offer { sdp, .. }] = inbox.as_slice() else { + panic!("expected one offer: {inbox:?}"); + }; + assert!(sdp.contains("1.1.1.4 5000"), "{sdp}"); + // A member reached at a public address keeps no local candidate. + assert!(!sdp.contains("192.168.1.4"), "{sdp}"); + assert!(!sdp.contains("9.9.9.99"), "{sdp}"); + + // The answer, likewise, names its own sender's public address only. + let mut reply = answer(1, a, 30); + if let ChunkMessageBody::SwarmAnswerRequest(request) = &mut reply { + request.sdp = pointing("1.1.1.5"); + } + from(&service, B, "1.1.1.5", reply).await; + let (_, inbox) = from(&service, A, "1.1.1.4", announce(1, &[], 0)) + .await + .expect("joined"); + let [SwarmSignal::Answer { sdp, .. }] = inbox.as_slice() else { + panic!("expected one answer: {inbox:?}"); + }; + assert!(sdp.contains("1.1.1.5 5000"), "{sdp}"); + assert!(!sdp.contains("9.9.9.99"), "{sdp}"); + // What is held is what goes, and all of it is let go. + assert_eq!(service.stats().stored_sdp_bytes, 0); + } + + #[tokio::test] + async fn a_member_that_moves_counts_and_answers_from_its_new_host() { + let service = service(); + let (old, new): (IpAddr, IpAddr) = ( + "1.1.1.5".parse().expect("ip"), + "1.1.1.6".parse().expect("ip"), + ); + from(&service, B, "1.1.1.5", announce(1, &[], 0)).await; + // The same membership, over a connection from another address. + from(&service, B, "1.1.1.6", announce(1, &[], 0)).await; + { + let state = service.state.lock(); + assert_eq!(state.by_host.get(&old), None); + assert_eq!(state.by_host.get(&new), Some(&1)); + } + let (a, _) = from(&service, A, "1.1.1.4", announce(1, &[30], 0)) + .await + .expect("joined"); + from(&service, B, "1.1.1.6", announce(1, &[], 0)).await; + let mut reply = answer(1, a, 30); + if let ChunkMessageBody::SwarmAnswerRequest(request) = &mut reply { + request.sdp = pointing("1.1.1.6"); + } + from(&service, B, "1.1.1.6", reply).await; + let (_, inbox) = from(&service, A, "1.1.1.4", announce(1, &[], 0)) + .await + .expect("joined"); + let [SwarmSignal::Answer { sdp, .. }] = inbox.as_slice() else { + panic!("expected one answer: {inbox:?}"); + }; + // The answer names the address it came from, which is the member's own. + assert!(sdp.contains("1.1.1.6 5000"), "{sdp}"); + } + + #[tokio::test] + async fn an_sdp_of_dropped_candidates_is_charged_only_what_it_keeps() { + let service = service(); + from(&service, B, "1.1.1.5", announce(1, &[], 0)).await; + let mut offer = announce(1, &[31], 0); + if let ChunkMessageBody::SwarmAnnounceRequest(request) = &mut offer { + for sent in &mut request.offers { + // A stranger's address, spelled as a receiver still reads + // it, filling the SDP: none of it is passed on. + sent.sdp = String::from("v=0\r\n"); + while sent.sdp.len() < MAX_SWARM_SDP_BYTES - 64 { + sent.sdp + .push_str("a= candidate:2 1 udp 1 9.9.9.99 5001 typ host\r\n"); + } + } + } + from(&service, A, "1.1.1.4", offer).await; + assert_eq!(service.stats().pending_offers, 1); + let held = service.stats().stored_sdp_bytes; + assert!(held < 64, "{held} bytes charged for a 5-byte SDP"); + let (_, inbox) = from(&service, B, "1.1.1.5", announce(1, &[], 0)) + .await + .expect("joined"); + let [SwarmSignal::Offer { sdp, .. }] = inbox.as_slice() else { + panic!("expected one offer: {inbox:?}"); + }; + assert_eq!(sdp, "v=0\r\n"); + } + + #[test] + fn a_member_on_this_nodes_own_network_keeps_its_local_candidates() { + let sdp = format!( + "{}a=candidate:4 1 udp 1 1f2e3d4c.local 5003 typ host\r\n", + pointing("10.78.3.2") + ); + let kept = relayable_sdp(&sdp, Some(host_of("10.78.3.2".parse().expect("ip")))); + assert!(kept.contains("10.78.3.2 5000"), "{kept}"); + assert!(kept.contains("192.168.1.4 5002"), "{kept}"); + assert!(kept.contains("1f2e3d4c.local"), "{kept}"); + assert!(!kept.contains("9.9.9.99"), "{kept}"); + let public = relayable_sdp(&sdp, Some(host_of("1.1.1.4".parse().expect("ip")))); + assert!( + !public.contains(".local") && !public.contains("192.168.1.4"), + "{public}" + ); + // A member whose address this node cannot tell keeps nothing local. + assert!(!relayable_sdp(&sdp, None).contains("192.168.1.4")); + } + + #[tokio::test] + async fn answers_nobody_asked_for_go_nowhere() { + let service = service(); + let a = call(&service, A, announce(1, &[], 0)).await; + let _b = call(&service, B, announce(1, &[], 0)).await; + let _c = call(&service, C, announce(2, &[], 0)).await; + // No offer 9 was delivered to B; C is not in room 1. + assert_eq!( + answer_reply(&service, B, answer(1, a.member, 9)).await, + SwarmAnswerResponse::Unmatched + ); + assert_eq!( + answer_reply(&service, C, answer(1, a.member, 9)).await, + SwarmAnswerResponse::Unmatched + ); + assert!(call(&service, A, announce(1, &[], 0)) + .await + .inbox + .is_empty()); + } + + #[tokio::test] + async fn a_waiting_announce_wakes_on_the_first_signal() { + let service = service(); + let _b = call(&service, B, announce(1, &[], 0)).await; + let waiting = { + let service = Arc::clone(&service); + tokio::spawn(async move { call(&service, B, announce(1, &[], 10_000)).await }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(service.stats().waiting, 1); + let started = Instant::now(); + let _a = call(&service, A, announce(1, &[40], 0)).await; + let reply = waiting.await.expect("task"); + assert_eq!(reply.inbox.len(), 1); + assert!(started.elapsed() < Duration::from_secs(2)); + assert_eq!(service.stats().waiting, 0); + } + + #[tokio::test] + async fn a_waiting_announce_ends_when_the_node_stops() { + let service = service(); + let token = CancellationToken::new(); + let waiting = { + let service = Arc::clone(&service); + let token = token.clone(); + tokio::spawn(async move { + service + .handle(B, None, announce(1, &[], 20_000), &token, Instant::now()) + .await + }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + token.cancel(); + let reply = tokio::time::timeout(Duration::from_secs(2), waiting).await; + assert!(matches!( + reply, + Ok(Ok(Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Joined { .. } + )))) + )); + } + + #[tokio::test] + async fn caps_refuse_newcomers_and_leaving_frees_the_place() { + let service = service(); + for n in 0..MAX_ROOMS_PER_SESSION { + let room = u8::try_from(n).unwrap_or(0); + let _ = call(&service, A, announce(room, &[], 0)).await; + } + let refused = service + .handle( + A, + None, + announce(200, &[], 0), + &CancellationToken::new(), + Instant::now(), + ) + .await; + assert!(matches!( + refused, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Refused(SwarmRefusal::TooManyRoomsForPeer) + )) + )); + service.session_closed(A); + assert_eq!(service.stats(), SwarmStats::default()); + let _ = call(&service, A, announce(200, &[], 0)).await; + let _ = service + .handle( + A, + None, + ChunkMessageBody::SwarmLeaveRequest(SwarmLeaveRequest { + room: [200; 32], + node: NODE, + membership: 1, + }), + &CancellationToken::new(), + Instant::now(), + ) + .await; + assert_eq!(service.stats().members, 0); + } + + #[tokio::test] + async fn unanswered_offers_and_silent_members_expire() { + let service = service(); + let _b = call(&service, B, announce(1, &[], 0)).await; + let _a = call(&service, A, announce(1, &[50], 0)).await; + assert_eq!(service.stats().pending_offers, 1); + assert!(service.stats().stored_sdp_bytes > 0); + let later = Instant::now() + SIGNAL_TTL + Duration::from_secs(1); + service.sweep(later); + assert_eq!(service.stats().pending_offers, 0); + assert_eq!(service.stats().stored_sdp_bytes, 0); + service.sweep(Instant::now() + MEMBER_TTL + Duration::from_secs(1)); + assert_eq!(service.stats(), SwarmStats::default()); + } + + #[tokio::test] + async fn a_node_browsers_cannot_reach_hosts_no_rooms() { + let service = SwarmService::new(); + service.set_node_id(NODE); + let token = CancellationToken::new(); + let refused = service + .handle(A, None, announce(1, &[], 0), &token, Instant::now()) + .await; + assert!(matches!( + refused, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Refused(SwarmRefusal::NotHosting) + )) + )); + assert_eq!(service.stats().members, 0); + // Once it has a browser endpoint, and so a STUN server to name, it does. + service.set_ice_servers(vec![STUN.into()]); + let joined = service + .handle(A, None, announce(1, &[], 0), &token, Instant::now()) + .await; + assert!(matches!( + joined, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Joined { .. } + )) + )); + } + + #[tokio::test] + async fn stun_answers_only_recent_announcers_and_replies_name_it() { + let service = service(); + let ip: IpAddr = "198.51.100.4".parse().expect("ip"); + let other: IpAddr = "198.51.100.5".parse().expect("ip"); + let now = Instant::now(); + assert!(!service.stun_allows(ip, now)); + let reply = service + .handle( + A, + Some(ip), + announce(1, &[], 0), + &CancellationToken::new(), + Instant::now(), + ) + .await; + assert!(matches!( + reply, + Some(ChunkMessageBody::SwarmAnnounceResponse(SwarmAnnounceResponse::Joined { ice_servers, .. })) + if ice_servers == vec![STUN.to_string()] + )); + assert!(service.stun_allows(ip, Instant::now())); + assert!(!service.stun_allows(other, Instant::now())); + // An IPv4-mapped form of the same address counts as the same. + let mapped: IpAddr = "::ffff:198.51.100.4".parse().expect("ip"); + assert!(service.stun_allows(mapped, Instant::now())); + service.sweep(Instant::now() + MEMBER_TTL + Duration::from_secs(1)); + assert!(!service.stun_allows(ip, Instant::now() + MEMBER_TTL + Duration::from_secs(1))); + } + + #[test] + fn swarm_requests_are_told_apart_from_their_header() { + use ant_protocol::chunk::{ + ChunkGetOrCloserRequest, ChunkGetOrCloserResponse, ChunkGetRequest, ChunkMessage, + }; + let encode = |body: ChunkMessageBody, request_id: u64| { + ChunkMessage { request_id, body }.encode().expect("encode") + }; + for request_id in [0, 1, 300, u64::MAX] { + assert!(is_swarm_request_bytes(&encode( + announce(1, &[1], 0), + request_id + ))); + assert!(is_swarm_request_bytes(&encode( + answer(1, [2; 16], 3), + request_id + ))); + assert!(is_swarm_request_bytes(&encode( + ChunkMessageBody::SwarmLeaveRequest(SwarmLeaveRequest { + room: [1; 32], + node: NODE, + membership: 1, + }), + request_id + ))); + assert!(!is_swarm_request_bytes(&encode( + ChunkMessageBody::GetRequest(ChunkGetRequest::new([1; 32])), + request_id + ))); + // The variants just before the swarm's: a chunk read must never + // be taken for one and left unanswered. + assert!(!is_swarm_request_bytes(&encode( + ChunkMessageBody::GetOrCloserRequest(ChunkGetOrCloserRequest::new([1; 32])), + request_id + ))); + assert!(!is_swarm_request_bytes(&encode( + ChunkMessageBody::GetOrCloserResponse(ChunkGetOrCloserResponse::Error( + ProtocolError::Internal(String::new()) + )), + request_id + ))); + } + assert!(!is_swarm_request_bytes(&[])); + assert!(!is_swarm_request_bytes(&[0xff; 12])); + } + + #[tokio::test] + async fn a_browser_never_waits() { + let service = service(); + let started = Instant::now(); + let _ = call(&service, A, announce(1, &[], 10_000)).await; + assert!(started.elapsed() < Duration::from_secs(1)); + } + + #[tokio::test] + async fn a_member_waits_with_one_announce_at_a_time() { + let service = service(); + let _ = call(&service, B, announce(1, &[], 0)).await; + let waiting = { + let service = Arc::clone(&service); + tokio::spawn(async move { call(&service, B, announce(1, &[], 10_000)).await }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + let started = Instant::now(); + let _ = call(&service, B, announce(1, &[], 10_000)).await; + assert!(started.elapsed() < Duration::from_secs(1)); + assert_eq!(service.stats().waiting, 1); + // Closing the connection ends the wait, and frees the place. + service.session_closed(B); + let ended = tokio::time::timeout(Duration::from_secs(2), waiting).await; + assert!(ended.is_ok()); + assert_eq!(service.stats().waiting, 0); + } + + #[tokio::test] + async fn a_signal_already_taken_does_not_cut_a_later_wait_short() { + let service = service(); + let _ = call(&service, B, announce(1, &[], 0)).await; + // An offer arrives while B is not waiting, and B takes it at once. + let _ = call(&service, A, announce(1, &[60], 0)).await; + assert_eq!(call(&service, B, announce(1, &[], 0)).await.inbox.len(), 1); + let started = Instant::now(); + let reply = call(&service, B, announce(1, &[], 300)).await; + assert!(reply.inbox.is_empty()); + assert!(started.elapsed() >= Duration::from_millis(250)); + } + + #[tokio::test] + async fn a_request_from_before_its_connection_closed_joins_nothing() { + let service = service(); + let before = Instant::now(); + tokio::time::sleep(Duration::from_millis(5)).await; + service.session_closed(B); + let late = service + .handle( + B, + None, + announce(1, &[], 0), + &CancellationToken::new(), + before, + ) + .await; + assert!(matches!( + late, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Refused(SwarmRefusal::Busy) + )) + )); + assert_eq!(service.stats().members, 0); + // A browser whose connection is closing joins nothing either. + let closing = CancellationToken::new(); + closing.cancel(); + let refused = service + .handle(A, None, announce(1, &[], 0), &closing, Instant::now()) + .await; + assert!(matches!( + refused, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Refused(SwarmRefusal::Busy) + )) + )); + // A request after the close is a new connection's. + let _ = call(&service, B, announce(1, &[], 0)).await; + assert_eq!(service.stats().members, 1); + } + + #[tokio::test] + async fn a_request_for_another_node_is_refused() { + let service = service(); + let mut elsewhere = SwarmAnnounceRequest { + room: [1; 32], + node: [1; 32], + membership: 1, + offers: Vec::new(), + wait_ms: 0, + }; + let token = CancellationToken::new(); + let reply = service + .handle( + B, + None, + ChunkMessageBody::SwarmAnnounceRequest(elsewhere.clone()), + &token, + Instant::now(), + ) + .await; + assert!(matches!( + reply, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Error(_) + )) + )); + assert_eq!(service.stats().members, 0); + // A node that does not know its own id yet answers nothing. + let unknown = SwarmService::new(); + elsewhere.node = NODE; + let reply = unknown + .handle( + B, + None, + ChunkMessageBody::SwarmAnnounceRequest(elsewhere), + &token, + Instant::now(), + ) + .await; + assert!(matches!( + reply, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Error(_) + )) + )); + // A leave for another node is told so too, and ends nothing here. + let _ = call(&service, A, announce(1, &[], 0)).await; + let leave = service + .handle( + A, + None, + ChunkMessageBody::SwarmLeaveRequest(SwarmLeaveRequest { + room: [1; 32], + node: [1; 32], + membership: 1, + }), + &token, + Instant::now(), + ) + .await; + assert!(matches!( + leave, + Some(ChunkMessageBody::SwarmLeaveResponse( + SwarmLeaveResponse::Error(_) + )) + )); + assert_eq!(service.stats().members, 1); + } + + #[tokio::test] + async fn a_leave_from_before_a_close_spares_the_next_connection() { + let service = service(); + let leave_received = Instant::now(); + tokio::time::sleep(Duration::from_millis(5)).await; + service.session_closed(B); + // The same client, connected again, joins. + let _ = call(&service, B, announce(1, &[], 0)).await; + let leave = ChunkMessageBody::SwarmLeaveRequest(SwarmLeaveRequest { + room: [1; 32], + node: NODE, + membership: 1, + }); + let _ = service + .handle(B, None, leave, &CancellationToken::new(), leave_received) + .await; + assert_eq!(service.stats().members, 1); + } + + #[tokio::test] + async fn offers_go_in_order_and_the_rest_are_counted() { + let service = service(); + let _b = call(&service, B, announce(1, &[], 0)).await; + // Two offers, one other member: the first goes, the second does not. + let a = call(&service, A, announce(1, &[70, 71], 0)).await; + assert_eq!(a.delivered, 1); + let inbox = call(&service, B, announce(1, &[], 0)).await.inbox; + assert!(matches!( + inbox.as_slice(), + [SwarmSignal::Offer { offer_id, .. }] if *offer_id == [70; 16] + )); + assert_eq!( + service + .counters() + .offers_undelivered + .load(Ordering::Relaxed), + 1 + ); + } + + #[test] + fn closes_of_one_session_keep_the_queue_bounded() { + let mut state = State::default(); + let start = Instant::now(); + let session = SwarmSession::Native([7; 32]); + for _ in 0..(3 * MAX_CLOSED_SESSIONS) { + state.note_close(session, start); + } + assert!(state.closed_order.len() <= MAX_CLOSED_SESSIONS); + assert!(state.closed_since(session, start)); + } + + #[test] + fn closes_are_forgotten_in_order_and_the_newest_is_kept() { + let mut state = State::default(); + let start = Instant::now(); + for n in 0..(MAX_CLOSED_SESSIONS + 10) { + let mut peer = [0u8; 32]; + peer[..8].copy_from_slice(&(n as u64).to_be_bytes()); + state.note_close(SwarmSession::Native(peer), start); + } + assert_eq!(state.closed.len(), MAX_CLOSED_SESSIONS); + // The newest close is always remembered. + let newest = SwarmSession::Native([0xff; 32]); + state.note_close(newest, start); + assert!(state.closed.contains_key(&newest)); + state.forget_closes(start + CLOSED_SESSION_TTL); + assert!(state.closed.is_empty() && state.closed_order.is_empty()); + } + + #[test] + fn a_signed_request_counts_once_and_only_while_fresh() { + let service = service(); + let now = 1_000_000; + let skew = MAX_REQUEST_SKEW.as_secs(); + let seen = + |peer: u8, id: u64, at: u64| service.first_sight([peer; 32], None, id, at, now, true); + assert!(seen(1, 7, now)); + // The same request again, as a node that saw it might send it. + assert!(!seen(1, 7, now)); + // Another client's request with the same id is its own. + assert!(seen(2, 7, now)); + // Too old, or from too far ahead. + assert!(!seen(1, 8, now - skew - 1)); + assert!(!seen(1, 9, now + skew + 1)); + // Overtaken by a newer request by a few seconds: still fine. + assert!(seen(1, 10, now + 4)); + assert!(seen(1, 11, now)); + // Much older than the newest: an old request sent again. + assert!(!seen(1, 12, now - REQUEST_REORDER - 1)); + } + + #[test] + fn only_an_announce_or_a_leave_starts_a_client_s_history() { + assert!(starts_history(&announce(1, &[], 0))); + assert!(starts_history(&ChunkMessageBody::SwarmLeaveRequest( + SwarmLeaveRequest { + room: [1; 32], + node: NODE, + membership: 1, + } + ))); + assert!(!starts_history(&answer(1, [2; 16], 3))); + let service = service(); + let now = 1_000_000; + assert!(!service.first_sight([3; 32], None, 1, now, now, false)); + assert!(service.histories.lock().by_peer.is_empty()); + assert!(service.first_sight([3; 32], None, 2, now, now, true)); + assert!(service.first_sight([3; 32], None, 3, now, now, false)); + } + + #[test] + fn one_host_keeps_only_so_many_histories_and_idle_ones_go() { + let service = service(); + let now = 1_000_000; + let host: IpAddr = "198.51.100.4".parse().expect("ip"); + let peer = |n: u32| { + let mut peer = [0u8; 32]; + peer[..4].copy_from_slice(&n.to_be_bytes()); + peer + }; + for n in 0..u32::try_from(MAX_HISTORIES_PER_HOST).unwrap_or(u32::MAX) { + assert!(service.first_sight(peer(n), Some(host), 1, now, now, true)); + } + assert!(!service.first_sight(peer(100_000), Some(host), 1, now, now, true)); + // Another host is not held back. + let other: IpAddr = "198.51.100.5".parse().expect("ip"); + assert!(service.first_sight(peer(100_001), Some(other), 1, now, now, true)); + // Once nothing they hold could pass the age check, they go. + let later = now + MAX_REQUEST_SKEW.as_secs() + 1; + service + .histories + .lock() + .forget_older_than(later - MAX_REQUEST_SKEW.as_secs()); + assert!(service.histories.lock().by_peer.is_empty()); + assert!(service.histories.lock().order.is_empty()); + assert!(service.first_sight(peer(100_000), Some(host), 1, later, later, true)); + } + + #[test] + fn with_every_history_taken_the_oldest_goes() { + let mut histories = Histories::default(); + let peer = |n: u32| { + let mut peer = [0u8; 32]; + peer[..4].copy_from_slice(&n.to_be_bytes()); + peer + }; + for n in 0..=u32::try_from(MAX_REQUEST_HISTORIES).unwrap_or(u32::MAX) { + assert!(histories.start(peer(n), None)); + } + assert_eq!(histories.by_peer.len(), MAX_REQUEST_HISTORIES); + assert!(!histories.by_peer.contains_key(&peer(0))); + assert!(histories.by_peer.contains_key(&peer(1))); + } + + #[tokio::test] + async fn a_member_can_hold_only_so_many_unanswered_offers() { + let service = service(); + let _b = call(&service, B, announce(1, &[], 0)).await; + let mut delivered = 0; + for offer in 0..40u8 { + delivered += call(&service, A, announce(1, &[offer], 0)).await.delivered; + } + // B's inbox holds 32, A may have 32 offers out: both caps meet at 32. + assert_eq!(delivered, 32); + assert_eq!(service.stats().pending_offers, 32); + } + + #[tokio::test] + async fn a_member_that_goes_takes_the_offers_made_to_it() { + let service = service(); + let _b = call(&service, B, announce(1, &[], 0)).await; + let a = call(&service, A, announce(1, &[80], 0)).await; + assert_eq!(a.delivered, 1); + assert_eq!(service.stats().pending_offers, 1); + // B can no longer answer A's offer: A's place for it is free again. + service.session_closed(B); + assert_eq!(service.stats().pending_offers, 0); + assert_eq!(service.stats().stored_sdp_bytes, 0); + } + + #[tokio::test] + async fn a_newer_membership_takes_the_place_and_an_old_leave_spares_it() { + let service = service(); + let with = |membership: u64, body: ChunkMessageBody| match body { + ChunkMessageBody::SwarmAnnounceRequest(mut request) => { + request.membership = membership; + ChunkMessageBody::SwarmAnnounceRequest(request) + } + other => other, + }; + let leave = |membership: u64| { + ChunkMessageBody::SwarmLeaveRequest(SwarmLeaveRequest { + room: [1; 32], + node: NODE, + membership, + }) + }; + let _b = call(&service, B, announce(1, &[], 0)).await; + let old = call(&service, A, with(5, announce(1, &[90], 0))).await; + assert_eq!(old.delivered, 1); + // The client rejoins: a new member, and the old one's offer is gone. + let new = call(&service, A, with(6, announce(1, &[], 0))).await; + assert_ne!(new.member, old.member); + assert_eq!(service.stats().members, 2); + assert_eq!(service.stats().pending_offers, 0); + // The old membership's announce is refused, and its leave ends nothing. + let stale = service + .handle( + A, + None, + with(5, announce(1, &[], 0)), + &CancellationToken::new(), + Instant::now(), + ) + .await; + assert!(matches!( + stale, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Refused(SwarmRefusal::Superseded) + )) + )); + let _ = service + .handle(A, None, leave(5), &CancellationToken::new(), Instant::now()) + .await; + assert_eq!(service.stats().members, 2); + // Its own leave ends it. + let _ = service + .handle(A, None, leave(6), &CancellationToken::new(), Instant::now()) + .await; + assert_eq!(service.stats().members, 1); + } + + #[tokio::test] + async fn an_ended_membership_does_not_come_back() { + let service = service(); + let token = CancellationToken::new(); + let membership = |membership: u64| { + let ChunkMessageBody::SwarmAnnounceRequest(mut request) = announce(1, &[], 0) else { + unreachable!("an announce"); + }; + request.membership = membership; + ChunkMessageBody::SwarmAnnounceRequest(request) + }; + let leave = |membership: u64| { + ChunkMessageBody::SwarmLeaveRequest(SwarmLeaveRequest { + room: [1; 32], + node: NODE, + membership, + }) + }; + let superseded = |reply: &Option| { + matches!( + reply, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Refused(SwarmRefusal::Superseded) + )) + ) + }; + // A leave that overtook its own announce: the announce joins nothing. + let _ = service + .handle(A, None, leave(6), &token, Instant::now()) + .await; + let late = service + .handle(A, None, membership(6), &token, Instant::now()) + .await; + assert!(superseded(&late)); + // Nor does an older membership's, after the newer one left. + let late = service + .handle(A, None, membership(5), &token, Instant::now()) + .await; + assert!(superseded(&late)); + assert_eq!(service.stats().members, 0); + // A newer membership joins. + let _ = call(&service, A, membership(7)).await; + assert_eq!(service.stats().members, 1); + // A connection that closes ends no membership: the client carries on + // in the same one over its next connection. + service.session_closed(A); + assert_eq!(service.stats().members, 0); + let _ = call(&service, A, membership(7)).await; + assert_eq!(service.stats().members, 1); + } + + #[tokio::test] + async fn an_older_membership_does_not_come_back_after_a_newer_one_goes_silent() { + let service = service(); + let with = |membership: u64| { + let ChunkMessageBody::SwarmAnnounceRequest(mut request) = announce(1, &[], 0) else { + unreachable!("an announce"); + }; + request.membership = membership; + ChunkMessageBody::SwarmAnnounceRequest(request) + }; + // Membership 7 arrives first, then goes silent. + let _ = call(&service, A, with(7)).await; + service.sweep(Instant::now() + MEMBER_TTL + Duration::from_secs(1)); + assert_eq!(service.stats().members, 0); + // The older 6, arriving late, does not come back; 7 itself may. + let late = service + .handle(A, None, with(6), &CancellationToken::new(), Instant::now()) + .await; + assert!(matches!( + late, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Refused(SwarmRefusal::Superseded) + )) + )); + let _ = call(&service, A, with(7)).await; + assert_eq!(service.stats().members, 1); + } + + #[test] + fn a_request_for_another_node_is_told_apart_before_its_history() { + let service = service(); + assert!(service.names_this_node(&announce(1, &[], 0))); + let ChunkMessageBody::SwarmAnnounceRequest(mut elsewhere) = announce(1, &[], 0) else { + unreachable!("an announce"); + }; + elsewhere.node = [1; 32]; + assert!(!service.names_this_node(&ChunkMessageBody::SwarmAnnounceRequest(elsewhere))); + assert!(service.names_this_node(&answer(1, [2; 16], 3))); + } + + #[tokio::test] + async fn one_host_holds_only_its_share_of_the_stored_sdp() { + let service = service(); + let token = CancellationToken::new(); + let sdp = MAX_SWARM_SDP_BYTES; + let offers = |room: u8| { + ChunkMessageBody::SwarmAnnounceRequest(SwarmAnnounceRequest { + room: [room; 32], + node: NODE, + membership: 1, + offers: (0..8u8) + .map(|id| SwarmOffer { + offer_id: [id; 16], + sdp: "v".repeat(sdp), + }) + .collect(), + wait_ms: 0, + }) + }; + // 24 rooms of 8 receivers each, from hosts of their own. + for room in 0..24u8 { + let receivers: IpAddr = format!("203.0.113.{room}").parse().expect("ip"); + for n in 0..8u64 { + let session = SwarmSession::Browser(1000 + u64::from(room) * 8 + n); + let _ = service + .handle( + session, + Some(receivers), + announce(room, &[], 0), + &token, + Instant::now(), + ) + .await; + } + } + // One host offers 8 into each room: more than its share. + let host: IpAddr = "198.51.100.4".parse().expect("ip"); + let mut delivered = 0u32; + for room in 0..24u8 { + if let Some(ChunkMessageBody::SwarmAnnounceResponse(SwarmAnnounceResponse::Joined { + offers_delivered, + .. + })) = service + .handle( + SwarmSession::Browser(u64::from(room)), + Some(host), + offers(room), + &token, + Instant::now(), + ) + .await + { + delivered += offers_delivered; + } + } + assert_eq!( + usize::try_from(delivered).unwrap_or(usize::MAX), + MAX_STORED_SDP_BYTES_PER_HOST / sdp + ); + // The rest of the budget is left for every other host. + assert_eq!( + service.stats().stored_sdp_bytes, + MAX_STORED_SDP_BYTES_PER_HOST + ); + } + + #[tokio::test] + async fn the_stun_allowlist_counts_an_ipv6_network_once() { + let service = service(); + let token = CancellationToken::new(); + for n in 0..32u16 { + let ip = IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 9, 0, 0, 0, n)); + let _ = service + .handle( + SwarmSession::Browser(u64::from(n)), + Some(ip), + announce(1, &[], 0), + &token, + Instant::now(), + ) + .await; + } + assert_eq!(service.announcers.read().len(), 1); + let neighbour = IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 9, 0, 0, 1, 1)); + assert!(service.stun_allows(neighbour, Instant::now())); + } + + #[test] + fn ended_memberships_outlast_every_request_still_taken() { + assert!(ENDED_MEMBERSHIP_TTL > 2 * MAX_REQUEST_SKEW); + } + + #[test] + fn one_connection_cannot_push_out_the_others_ended_memberships() { + let mut state = State::default(); + let now = Instant::now(); + let host: IpAddr = "198.51.100.4".parse().expect("ip"); + let honest = SwarmSession::Browser(1); + state.note_ended(honest, Some(host_of(host)), [1; 32], 5, now); + // A connection ending ever newer memberships of one room holds one + // place, and the queue does not grow past its bound. + let flooder = SwarmSession::Browser(2); + let other: IpAddr = "203.0.113.9".parse().expect("ip"); + for membership in 1..=(3 * MAX_ENDED_MEMBERSHIPS as u64) { + state.note_ended(flooder, Some(host_of(other)), [2; 32], membership, now); + } + assert_eq!(state.ended.len(), 2); + assert!(state.ended_order.len() <= 2 * state.ended.len() + 1024); + // Nor many rooms past its own share. + for room in 0..=u8::MAX { + state.note_ended(flooder, Some(host_of(other)), [room; 32], 1, now); + } + assert!( + state + .ended_by_session + .get(&flooder) + .map_or(0, VecDeque::len) + <= MAX_ENDED_PER_SESSION + ); + // The newest of them are the ones kept. + assert!(state.ended_covers(flooder, &[u8::MAX; 32], 1)); + assert!(state.ended_covers(honest, &[1; 32], 5)); + // A place ended again goes to the back: the oldest other goes first. + let mut rotating = State::default(); + let session = SwarmSession::Browser(9); + for room in 0..MAX_ENDED_PER_SESSION { + let room = u8::try_from(room).unwrap_or(u8::MAX); + rotating.note_ended(session, None, [room; 32], 1, now); + } + rotating.note_ended(session, None, [0; 32], 2, now); + rotating.note_ended(session, None, [200; 32], 1, now); + assert!(rotating.ended_covers(session, &[0; 32], 2)); + assert!(!rotating.ended_covers(session, &[1; 32], 1)); + assert!(rotating.ended_covers(session, &[200; 32], 1)); + // All go once nothing they cover could be taken. + state.forget_ended(now + ENDED_MEMBERSHIP_TTL); + assert!(state.ended.is_empty() && state.ended_order.is_empty()); + assert!(state.ended_by_session.is_empty() && state.ended_by_host.is_empty()); + } + + #[tokio::test] + async fn a_waiting_announce_of_a_replaced_membership_is_not_told_it_joined() { + let service = service(); + let with = |membership: u64, wait_ms: u32| { + let ChunkMessageBody::SwarmAnnounceRequest(mut request) = announce(1, &[], wait_ms) + else { + unreachable!("an announce"); + }; + request.membership = membership; + ChunkMessageBody::SwarmAnnounceRequest(request) + }; + let waiting = { + let service = Arc::clone(&service); + tokio::spawn(async move { + service + .handle( + B, + None, + with(5, 10_000), + &CancellationToken::new(), + Instant::now(), + ) + .await + }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(service.stats().waiting, 1); + let _ = call(&service, B, with(6, 0)).await; + let reply = tokio::time::timeout(Duration::from_secs(2), waiting).await; + assert!(matches!( + reply, + Ok(Ok(Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Refused(SwarmRefusal::Superseded) + )))) + )); + } + + #[tokio::test] + async fn one_host_holds_only_so_many_places() { + let service = service(); + let join = |n: u16, ip: IpAddr| { + let service = Arc::clone(&service); + async move { + let mut peer = [0u8; 32]; + peer[..2].copy_from_slice(&n.to_be_bytes()); + service + .handle( + SwarmSession::Native(peer), + Some(ip), + announce(1, &[], 0), + &CancellationToken::new(), + Instant::now(), + ) + .await + } + }; + let joined = |reply: &Option| { + matches!( + reply, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Joined { .. } + )) + ) + }; + // Identities of one IPv6 /64, each on its own address in it. + let host = |n: u16| IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 7, 0, 0, 0, n)); + for n in 0..u16::try_from(MAX_MEMBERS_PER_HOST).unwrap_or(u16::MAX) { + assert!(joined(&join(n, host(n)).await)); + } + let refused = join(1000, host(1000)).await; + assert!(matches!( + refused, + Some(ChunkMessageBody::SwarmAnnounceResponse( + SwarmAnnounceResponse::Refused(SwarmRefusal::TooManyRoomsForPeer) + )) + )); + // Another host is not held back; one that goes frees its place. + assert!(joined( + &join(2000, "198.51.100.4".parse().expect("ip")).await + )); + let mut first = [0u8; 32]; + first[..2].copy_from_slice(&0u16.to_be_bytes()); + service.session_closed(SwarmSession::Native(first)); + assert!(joined(&join(1000, host(1000)).await)); + } + + #[tokio::test] + async fn a_member_that_goes_leaves_no_offer_in_another_inbox() { + let service = service(); + let _b = call(&service, B, announce(1, &[], 0)).await; + for offer in 0..4u8 { + let _ = call(&service, A, announce(1, &[offer], 0)).await; + service.session_closed(A); + } + assert_eq!(service.stats().pending_offers, 0); + assert_eq!(service.stats().stored_sdp_bytes, 0); + assert!(call(&service, B, announce(1, &[], 0)) + .await + .inbox + .is_empty()); + } +} diff --git a/src/web_rtc.rs b/src/web_rtc.rs index 26df7d96..9deda562 100644 --- a/src/web_rtc.rs +++ b/src/web_rtc.rs @@ -19,6 +19,7 @@ use crate::error::{Error, Result}; use crate::logging::{debug, info, warn}; use crate::payment::{serialize_single_node_proof, PaymentProof}; use crate::storage::AntProtocol; +use crate::swarm::{is_swarm_request, is_swarm_request_bytes, SwarmService, SwarmSession}; use certificate::load_or_generate_certificate; use errors::{error_response, public_error, sanitize_response}; use evmlib::common::{Amount, TxHash}; @@ -61,6 +62,11 @@ const MAX_FIND_NODE_RESULTS: usize = 20; const MAX_CHANNEL_REQUESTS: usize = 4; const RPC_MULTIPLEX_CAPABILITY: &str = "rpc-multiplex-4"; +/// Advertised when `chunk_protocol` admits the swarm messages: a browser +/// sends them only to a node that names this, since a node that predates +/// them closes the channel on an unknown chunk protocol request. +pub const SWARM_CAPABILITY: &str = "swarm_rendezvous"; + /// Advertised when `chunk_protocol` admits pointer reads and paid pointer /// writes (ADR-0016), so a browser client can tell a node that would refuse /// them from one that failed. @@ -85,6 +91,16 @@ const FIRST_DATA_CHANNEL_TIMEOUT_ERROR: &str = "DataChannel opening timed out"; const CHANNEL_CAPACITY_ERROR: &str = "global DataChannel capacity exhausted"; const REQUEST_CAPACITY_ERROR: &str = "global request capacity exhausted"; const REQUEST_RATE_ERROR: &str = "request rate limit exceeded"; +const SWARM_RATE_ERROR: &str = "swarm request rate limit exceeded"; +/// Swarm requests a listener takes in a second, in all and from one source, +/// on budgets of their own: a browser announces about once a second per room +/// it shares a node with, and those announces must never spend the request +/// budget its own and every other browser's reads need. A swarm request the +/// swarm's budget has room for gives the read budget its token back; one past +/// it keeps that charge and is refused, without closing the channel the +/// reads share. +const SWARM_REQUESTS_PER_SECOND: usize = 64; +const SWARM_REQUESTS_PER_SECOND_PER_SOURCE: usize = 8; const GLOBAL_BYTE_CAPACITY_ERROR: &str = "global in-flight byte capacity exhausted"; const SOURCE_BYTE_CAPACITY_ERROR: &str = "source in-flight byte capacity exhausted"; @@ -151,6 +167,13 @@ impl RequestRateBucket { self.token_units -= 1_000_000_000; true } + + /// Gives back a token taken by [`Self::allow`], up to the bucket's + /// capacity. + fn refund(&mut self) { + let capacity = self.rate_per_second.saturating_mul(1_000_000_000); + self.token_units = self.token_units.saturating_add(1_000_000_000).min(capacity); + } } /// Atomic byte budget and an RAII reservation within it. @@ -282,6 +305,8 @@ impl TrackedBytes { struct SourceQuota { bulk_responses: Arc, request_rate: Mutex, + /// See [`SWARM_REQUESTS_PER_SECOND_PER_SOURCE`]. + swarm_rate: Mutex, bytes: Arc, } @@ -306,6 +331,8 @@ fn duration_ms(duration: Duration) -> u64 { /// Admission and accounting shared by every association on one listener. struct ListenerResources { running: AtomicBool, + /// Numbers browser connections for the swarm rendezvous. + next_connection: AtomicU64, connection_rejections: AtomicU64, channel_rejections: AtomicU64, request_rejections: AtomicU64, @@ -337,6 +364,8 @@ struct ListenerResources { bulk_responses: Arc, request_tasks: Mutex, global_request_rate: Mutex, + /// See [`SWARM_REQUESTS_PER_SECOND`]. + global_swarm_rate: Mutex, global_bytes: Arc, source_state: Mutex, max_connections_per_ip: usize, @@ -351,6 +380,7 @@ impl ListenerResources { let byte_rejections = Arc::new(AtomicU64::new(0)); Arc::new(Self { running: AtomicBool::new(false), + next_connection: AtomicU64::new(0), connection_rejections: AtomicU64::new(0), channel_rejections: AtomicU64::new(0), request_rejections: AtomicU64::new(0), @@ -375,6 +405,7 @@ impl ListenerResources { ))), request_tasks: Mutex::new(TaskTracker::new()), global_request_rate: Mutex::new(RequestRateBucket::new(config.max_requests_per_second)), + global_swarm_rate: Mutex::new(RequestRateBucket::new(SWARM_REQUESTS_PER_SECOND)), global_bytes: Arc::new(ByteBudget::with_rejections( config.max_in_flight_bytes, byte_rejections, @@ -437,6 +468,9 @@ impl ListenerResources { request_rate: Mutex::new(RequestRateBucket::new( self.max_requests_per_second_per_ip, )), + swarm_rate: Mutex::new(RequestRateBucket::new( + SWARM_REQUESTS_PER_SECOND_PER_SOURCE, + )), bytes: Arc::new(ByteBudget::with_rejections( self.max_in_flight_bytes_per_ip, Arc::clone(&self.byte_rejections), @@ -456,6 +490,9 @@ impl ListenerResources { request_rate: Mutex::new(RequestRateBucket::new( self.max_requests_per_second_per_connection, )), + swarm: SwarmSession::Browser(self.next_connection.fetch_add(1, Ordering::Relaxed)), + remote_ip: remote_addr.ip(), + closed: CancellationToken::new(), }); Ok(ConnectionAdmission { listener: Arc::clone(self), @@ -518,10 +555,28 @@ impl ListenerResources { } } +/// The budget a request frame was admitted on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum FrameCharge { + /// The request budget the reads share; a swarm request moves its charge + /// onto the swarm's own once it is known for one. + Requests, + /// The swarm's own, with the reads' spent: a frame then goes on only if + /// it is a swarm request, and anything else is refused as over the + /// request rate, its swarm charge given back. + Swarm, +} + struct ConnectionResources { listener: Arc, source: Arc, request_rate: Mutex, + /// This connection, as the swarm rendezvous names it. + swarm: SwarmSession, + /// The browser's address, for the STUN allowlist. + remote_ip: IpAddr, + /// Cancelled when the connection ends, which ends its waiting announces. + closed: CancellationToken, } impl ConnectionResources { @@ -547,6 +602,92 @@ impl ConnectionResources { Ok(permit) } + /// Admits a request frame as [`Self::try_admit_request`] does, or, when + /// the request budget the reads share has no token left, on the swarm's + /// own budget: only once the frame is opened is it known whether it is a + /// swarm request, which that budget is for (see [`FrameCharge`]). + fn try_admit_frame(&self) -> ServerResult<(OwnedSemaphorePermit, FrameCharge)> { + let permit = Arc::clone(&self.listener.request_limit) + .try_acquire_owned() + .map_err(|_| { + self.listener + .request_rejections + .fetch_add(1, Ordering::Relaxed); + REQUEST_CAPACITY_ERROR.to_string() + })?; + let now = Instant::now(); + let charge = if self.take_request_tokens(now) { + FrameCharge::Requests + } else if self.take_swarm_tokens(now) { + FrameCharge::Swarm + } else { + drop(permit); + self.listener + .rate_rejections + .fetch_add(1, Ordering::Relaxed); + return Err(REQUEST_RATE_ERROR.to_string()); + }; + Ok((permit, charge)) + } + + /// Takes a token from each request budget the reads share (the source's, + /// the connection's and the listener's), or from none: a budget with no + /// token left gives the ones before it theirs back, so a frame taken on + /// the swarm's budget instead has spent nothing of the reads'. + fn take_request_tokens(&self, now: Instant) -> bool { + if !self.source.request_rate.lock().allow(now) { + return false; + } + if !self.request_rate.lock().allow(now) { + self.source.request_rate.lock().refund(); + return false; + } + if !self.listener.global_request_rate.lock().allow(now) { + self.request_rate.lock().refund(); + self.source.request_rate.lock().refund(); + return false; + } + true + } + + /// As [`Self::take_request_tokens`], for the swarm's own budgets (the + /// source's and the listener's). + fn take_swarm_tokens(&self, now: Instant) -> bool { + if !self.source.swarm_rate.lock().allow(now) { + return false; + } + if !self.listener.global_swarm_rate.lock().allow(now) { + self.source.swarm_rate.lock().refund(); + return false; + } + true + } + + /// Gives the swarm's budget back a frame's charge: it was no swarm + /// request. + fn refund_swarm(&self) { + self.source.swarm_rate.lock().refund(); + self.listener.global_swarm_rate.lock().refund(); + } + + /// Moves an admitted swarm request's charge from the request budget the + /// reads share onto the swarm's own: true if the swarm's budget had room + /// (the read budget gets its tokens back), false if not (the request + /// keeps its charge, and is refused). + fn charge_to_swarm(&self) -> bool { + let now = Instant::now(); + if !self.take_swarm_tokens(now) { + self.listener + .rate_rejections + .fetch_add(1, Ordering::Relaxed); + return false; + } + self.source.request_rate.lock().refund(); + self.request_rate.lock().refund(); + self.listener.global_request_rate.lock().refund(); + true + } + fn try_reserve_bytes(&self, amount: usize) -> ServerResult { let source = self .source @@ -815,11 +956,13 @@ pub struct WebRtcDirectServer { } /// Start the feature-gated browser listener and return its endpoint and task. +#[allow(clippy::too_many_arguments)] pub async fn spawn( config: &WebRtcDirectConfig, root_dir: &Path, p2p: Arc, ant_protocol: Option>, + swarm: Arc, evm_network: &evmlib::Network, shutdown: CancellationToken, endpoint_catalog: Option>, @@ -841,6 +984,12 @@ pub async fn spawn( ) .await .map_err(|error| Error::Startup(format!("failed to bind WebRTC Direct listener: {error}")))?; + // This socket is also the STUN server for swarm members' WebRTC + // connections to each other, answering clients that announced recently. + let stun = Arc::clone(&swarm); + listener.set_plain_binding_policy(Some(Arc::new(move |ip| { + stun.stun_allows(ip, Instant::now()) + }))); let local_addr = listener.local_addr(); // Like the native QUIC endpoint, ask the gateway to forward the listener's // port. Best effort: without a mapping, publication is unchanged (ADR-0027). @@ -852,6 +1001,7 @@ pub async fn spawn( identity, p2p, ant_protocol, + swarm, payment, publication: RwLock::new(Publication::default()), endpoint_catalog, @@ -994,6 +1144,8 @@ async fn refresh_browser_endpoint( if let Some(catalog) = state.endpoint_catalog.as_ref() { catalog.insert(peer_id, endpoint.clone()); } + // The same UDP socket answers swarm members' STUN requests. + state.swarm.set_ice_servers(vec![format!("stun:{address}")]); *state.publication.write() = Publication::Published { endpoint, source }; state .p2p @@ -1466,6 +1618,10 @@ async fn handle_connection( // listener's request tracker and keep their storage state and admission // until processing finishes, independently of this connection. channel_shutdown.cancel(); + // The browser has gone: its waiting announces end now, and it leaves every + // swarm room it was in. + resources.closed.cancel(); + state.swarm.session_closed(resources.swarm); if let Err(error) = connection.close().await { debug!(remote = %remote_addr, %error, "Failed to close WebRTC Direct connection"); } @@ -1721,6 +1877,7 @@ async fn establish_pq_session( struct AdmittedRequest { request: Request, content: Vec, + charge: FrameCharge, _request_permit: OwnedSemaphorePermit, _in_flight_bytes: InFlightByteReservation, } @@ -1767,6 +1924,7 @@ async fn start_request( let AdmittedRequest { request, content, + charge, _request_permit: request_permit, _in_flight_bytes: in_flight_bytes, } = admitted; @@ -1790,19 +1948,19 @@ async fn start_request( } else { None }; - let (response, content) = match process_request(request, content, &state, &resources).await - { - Ok(processed) => processed, - Err(error) => { - if is_get { - resources - .listener - .get_service_failures - .fetch_add(1, Ordering::Relaxed); + let (response, content) = + match process_request(request, content, charge, &state, &resources).await { + Ok(processed) => processed, + Err(error) => { + if is_get { + resources + .listener + .get_service_failures + .fetch_add(1, Ordering::Relaxed); + } + return Err(error); } - return Err(error); - } - }; + }; if is_get { resources .listener @@ -1841,7 +1999,7 @@ async fn read_webrtc_request( } // Admission happens as soon as a client starts a frame. Idle persistent // channels consume neither request-rate tokens nor request worker slots. - let request_permit = resources.try_admit_request()?; + let (request_permit, charge) = resources.try_admit_frame()?; let max_plaintext_bytes = MAX_BROWSER_FRAME_BYTES; let mut encrypted = read_pq_payload_after_first( first_message, @@ -1877,9 +2035,21 @@ async fn read_webrtc_request( let content = frame[content_offset..].to_vec(); drop(frame); reservation.resize(accounted_request_bytes)?; + // Admitted on the swarm's budget, only a swarm request goes on. + if charge == FrameCharge::Swarm + && !(matches!(request.body, RequestBody::ChunkProtocol) && is_swarm_request_bytes(&content)) + { + resources.refund_swarm(); + resources + .listener + .rate_rejections + .fetch_add(1, Ordering::Relaxed); + return Err(REQUEST_RATE_ERROR.to_string()); + } Ok(AdmittedRequest { request, content, + charge, _request_permit: request_permit, _in_flight_bytes: reservation, }) @@ -2073,9 +2243,38 @@ async fn write_framed_pq_payload(channel: &WebRtcDataChannel, payload: &[u8]) -> Ok(()) } +/// A swarm request's refusal before it is decoded: no legal one is longer +/// than `MAX_SWARM_REQUEST_BYTES`, and swarm requests run on a budget of their +/// own, never the reads' (see [`SWARM_REQUESTS_PER_SECOND`]). `None` lets it +/// go on, its charge moved off the read budget (or there from the start, for +/// a frame `charge` says was admitted on the swarm's). +fn swarm_request_refusal( + request_id: u64, + content_len: usize, + charge: FrameCharge, + resources: &ConnectionResources, +) -> Option { + if content_len > ant_protocol::MAX_SWARM_REQUEST_BYTES { + return Some(Response::error( + request_id, + "invalid_request", + "swarm request too long", + )); + } + if charge == FrameCharge::Requests && !resources.charge_to_swarm() { + return Some(Response::error( + request_id, + "rate_limited", + SWARM_RATE_ERROR, + )); + } + None +} + async fn process_request( request: Request, content: Vec, + charge: FrameCharge, state: &ServerState, resources: &ConnectionResources, ) -> ServerResult<(Response, Option)> { @@ -2095,16 +2294,29 @@ async fn process_request( } match request.body { RequestBody::ChunkProtocol => { - let Some(protocol) = state.ant_protocol.as_ref() else { - return Ok(( + // Every node hosts the swarm rendezvous, storing chunks or not. + let swarm_request = is_swarm_request_bytes(&content); + let storage_disabled = || { + Ok(( Response::error( request.request_id, "storage_disabled", "chunk storage is disabled".to_string(), ), None, - )); + )) }; + let protocol = state.ant_protocol.as_ref(); + if protocol.is_none() && !swarm_request { + return storage_disabled(); + } + if swarm_request { + if let Some(refused) = + swarm_request_refusal(request.request_id, content.len(), charge, resources) + { + return Ok((refused, None)); + } + } // The admitted frame remains charged for the whole request. Charge // decoding before it copies the input, then retain that allowance // for the PUT handler's disk buffer after dropping the wire buffer. @@ -2114,7 +2326,13 @@ async fn process_request( drop(content); let response_limit = binary_response_limit(&message.body)?; reservation.resize(reservation.source.amount.max(2 * response_limit))?; - let response = handle_ant_message(protocol, message).await?; + let response = match protocol { + _ if is_swarm_request(&message.body) => { + handle_swarm_message(&state.swarm, message, resources).await? + } + Some(protocol) => handle_ant_message(protocol, message).await?, + None => return storage_disabled(), + }; let bytes = encode_binary_response(&response, response_limit)?; drop(response); reservation.resize(bytes.len())?; @@ -2182,11 +2400,44 @@ fn binary_response_limit(body: &ChunkMessageBody) -> ServerResult { | ChunkMessageBody::MerkleCandidateQuoteRequest(_) | ChunkMessageBody::MerkleCandidateQuoteRequestV2(_) | ChunkMessageBody::PointerGetRequest(_) - | ChunkMessageBody::PointerPutRequest(_) => Ok(SMALL_BINARY_RESPONSE_BYTES), + | ChunkMessageBody::PointerPutRequest(_) + | ChunkMessageBody::SwarmAnnounceRequest(_) + | ChunkMessageBody::SwarmAnswerRequest(_) + | ChunkMessageBody::SwarmLeaveRequest(_) => Ok(SMALL_BINARY_RESPONSE_BYTES), _ => Err("unsupported chunk protocol request".to_string()), } } +// A swarm reply carries at most MAX_SWARM_REPLY_SDP_BYTES of SDP, plus ids and +// framing for up to MAX_SWARM_INBOX signals: well inside the small bound. +const _: () = assert!( + ant_protocol::swarm::MAX_SWARM_REPLY_SDP_BYTES + 4096 < SMALL_BINARY_RESPONSE_BYTES, + "a swarm reply must fit the small response bound" +); + +/// Answers a swarm request on this browser connection. A waiting announce +/// ends when the connection does. +async fn handle_swarm_message( + swarm: &SwarmService, + message: ChunkMessage, + resources: &ConnectionResources, +) -> ServerResult { + let body = swarm + .handle( + resources.swarm, + Some(resources.remote_ip), + message.body, + &resources.closed, + Instant::now(), + ) + .await + .ok_or_else(|| "unsupported chunk protocol request".to_string())?; + Ok(ChunkMessage { + request_id: message.request_id, + body, + }) +} + fn encode_binary_response(message: &ChunkMessage, limit: usize) -> ServerResult> { // Size without allocating, then encode into exactly the charged space. encode_exact(message, limit).map_err(|error| match error { @@ -2239,6 +2490,8 @@ fn hello_capabilities(pointers: bool) -> Vec { if pointers { capabilities.push(POINTER_PROTOCOL_CAPABILITY.into()); } + // Every node hosts the swarm rendezvous. + capabilities.push(SWARM_CAPABILITY.into()); capabilities } @@ -2695,6 +2948,8 @@ struct ServerState { identity: Arc, p2p: Arc, ant_protocol: Option>, + /// The swarm rendezvous, hosted whether chunks are stored or not. + swarm: Arc, payment: BrowserPaymentNetwork, publication: RwLock, endpoint_catalog: Option>, @@ -2839,6 +3094,7 @@ mod tests { payment_token_address: String::new(), payment_vault_address: String::new(), }, + swarm: SwarmService::new(), publication: RwLock::new(Publication::default()), endpoint_catalog: None, port_mapping: None, @@ -2862,6 +3118,7 @@ mod tests { 0, ), content: Vec::new(), + charge: FrameCharge::Requests, _request_permit: connection .context .try_admit_request() @@ -3134,6 +3391,176 @@ mod tests { assert!(honest.context.try_admit_request().is_ok()); } + #[test] + fn swarm_requests_within_their_budget_give_the_reads_their_tokens_back() { + let config = WebRtcDirectConfig { + max_requests_per_second: 100, + max_requests_per_second_per_ip: 1, + max_requests_per_second_per_connection: 1, + ..WebRtcDirectConfig::default() + }; + let resources = ListenerResources::new(&config); + let address = "198.51.100.1:1000" + .parse() + .unwrap_or_else(|error| panic!("browser address: {error}")); + let browser = resources + .try_admit_connection(address) + .unwrap_or_else(|error| panic!("browser connection: {error}")); + // An announce takes the source's only token as it arrives, and gives + // it back once it is known for a swarm request. + let announce = browser + .context + .try_admit_request() + .unwrap_or_else(|error| panic!("announce admitted: {error}")); + assert!(browser.context.charge_to_swarm()); + drop(announce); + // So the browser's next read still has it. + assert!(browser.context.try_admit_request().is_ok()); + } + + #[test] + fn a_frame_past_the_read_budget_is_admitted_on_the_swarms_own() { + let config = WebRtcDirectConfig { + max_requests_per_second: 100, + max_requests_per_second_per_ip: 1, + max_requests_per_second_per_connection: 1, + ..WebRtcDirectConfig::default() + }; + let resources = ListenerResources::new(&config); + let address = "198.51.100.1:1000" + .parse() + .unwrap_or_else(|error| panic!("browser address: {error}")); + let browser = resources + .try_admit_connection(address) + .unwrap_or_else(|error| panic!("browser connection: {error}")); + // A read takes the source's only token. + let (read, charge) = browser + .context + .try_admit_frame() + .unwrap_or_else(|error| panic!("read admitted: {error}")); + assert_eq!(charge, FrameCharge::Requests); + drop(read); + // An announce right after it is still admitted, on the swarm's + // budget, rather than refused with the channel the reads share. + let (announce, charge) = browser + .context + .try_admit_frame() + .unwrap_or_else(|error| panic!("announce admitted: {error}")); + assert_eq!(charge, FrameCharge::Swarm); + drop(announce); + // A frame that turns out to be no swarm request gives the swarm its + // charge back: the swarm's budget lets as many announces through as + // before, and the reads gain nothing from it. + for _ in 0..SWARM_REQUESTS_PER_SECOND_PER_SOURCE - 1 { + let (frame, charge) = browser + .context + .try_admit_frame() + .unwrap_or_else(|error| panic!("frame admitted: {error}")); + assert_eq!(charge, FrameCharge::Swarm); + browser.context.refund_swarm(); + drop(frame); + } + assert!(browser.context.try_admit_frame().is_ok()); + } + + #[test] + fn a_frame_taken_on_the_swarms_budget_spends_none_of_the_reads() { + // The connection's budget runs out before the source's. + let config = WebRtcDirectConfig { + max_requests_per_second: 100, + max_requests_per_second_per_ip: 2, + max_requests_per_second_per_connection: 1, + ..WebRtcDirectConfig::default() + }; + let resources = ListenerResources::new(&config); + let connect = |address: &str| { + let address = address + .parse() + .unwrap_or_else(|error| panic!("address: {error}")); + resources + .try_admit_connection(address) + .unwrap_or_else(|error| panic!("connection: {error}")) + }; + let browser = connect("198.51.100.1:1000"); + let (_read, charge) = browser + .context + .try_admit_frame() + .unwrap_or_else(|error| panic!("read admitted: {error}")); + assert_eq!(charge, FrameCharge::Requests); + let (_announce, charge) = browser + .context + .try_admit_frame() + .unwrap_or_else(|error| panic!("announce admitted: {error}")); + assert_eq!(charge, FrameCharge::Swarm); + // The source's second token is still there for the reads of another + // of its connections. + let other = connect("198.51.100.1:1001"); + let (_read, charge) = other + .context + .try_admit_frame() + .unwrap_or_else(|error| panic!("other read admitted: {error}")); + assert_eq!(charge, FrameCharge::Requests); + + // The listener's budget runs out first: the source's and the + // connection's tokens come back. + let config = WebRtcDirectConfig { + max_requests_per_second: 1, + max_requests_per_second_per_ip: 10, + max_requests_per_second_per_connection: 10, + ..WebRtcDirectConfig::default() + }; + let resources = ListenerResources::new(&config); + let address = "198.51.100.2:1000" + .parse() + .unwrap_or_else(|error| panic!("address: {error}")); + let browser = resources + .try_admit_connection(address) + .unwrap_or_else(|error| panic!("connection: {error}")); + let (_read, charge) = browser + .context + .try_admit_frame() + .unwrap_or_else(|error| panic!("read admitted: {error}")); + assert_eq!(charge, FrameCharge::Requests); + let (_announce, charge) = browser + .context + .try_admit_frame() + .unwrap_or_else(|error| panic!("announce admitted: {error}")); + assert_eq!(charge, FrameCharge::Swarm); + let now = Instant::now(); + let left = |bucket: &Mutex| { + (0..20).take_while(|_| bucket.lock().allow(now)).count() + }; + assert_eq!(left(&browser.context.request_rate), 9); + assert_eq!(left(&browser.context.source.request_rate), 9); + } + + #[test] + fn swarm_requests_past_their_budget_are_refused_and_keep_their_charge() { + let config = WebRtcDirectConfig { + max_requests_per_second: 1000, + max_requests_per_second_per_ip: 1000, + max_requests_per_second_per_connection: 1000, + ..WebRtcDirectConfig::default() + }; + let resources = ListenerResources::new(&config); + let connect = |address: &str| { + let address = address + .parse() + .unwrap_or_else(|error| panic!("address: {error}")); + resources + .try_admit_connection(address) + .unwrap_or_else(|error| panic!("connection: {error}")) + }; + let busy = connect("198.51.100.1:1000"); + for _ in 0..SWARM_REQUESTS_PER_SECOND_PER_SOURCE { + assert!(busy.context.charge_to_swarm()); + } + assert!(!busy.context.charge_to_swarm()); + // Another source has a budget of its own. + let other = connect("203.0.113.2:2000"); + assert!(other.context.charge_to_swarm()); + } + #[test] fn reconnecting_does_not_reset_the_source_request_bucket() { let config = WebRtcDirectConfig { @@ -3659,6 +4086,7 @@ mod tests { payment_token_address: String::new(), payment_vault_address: String::new(), }, + swarm: SwarmService::new(), publication: RwLock::new(Publication::default()), endpoint_catalog: Some(Arc::clone(&catalog)), port_mapping: Some(Box::new(TestMapping(Arc::clone(&mapping)))), From 2099907a26bacd8a8ff1d6a438fd7886920ca895 Mon Sep 17 00:00:00 2001 From: grumbach Date: Wed, 28 Oct 2026 15:36:19 +0900 Subject: [PATCH 02/15] chore(deps): use the peer-swarm branches of the swarm's dependencies Patch ant-protocol, saorsa-core and saorsa-transport to their feat/peer-swarm branches, which this change needs: the swarm messages (WithAutonomi/ant-protocol#50), the send on one named channel (WithAutonomi/saorsa-core#183) and the plain STUN answers (WithAutonomi/saorsa-transport#231). Once those merge, these point at their main branches, or at the released versions at the release train. --- Cargo.lock | 9 +++------ Cargo.toml | 8 ++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f0ce36b5..68558f7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -906,8 +906,7 @@ dependencies = [ [[package]] name = "ant-protocol" version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28e38869b71a547db27d565d6256c371ec410700d646e0eb7b683684cdaa93de" +source = "git+https://github.com/grumbach/ant-protocol?branch=feat%2Fpeer-swarm#61e3dbf50c3b851e9cd056e01ecc0611a2c31e4e" dependencies = [ "blake3", "bytes", @@ -5345,8 +5344,7 @@ dependencies = [ [[package]] name = "saorsa-core" version = "0.30.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25e3b19c7be5ab56d900ce4c0af1029f4009ef9142d4bb229e0370db0d31b131" +source = "git+https://github.com/grumbach/saorsa-core?branch=feat%2Fpeer-swarm#b7966e2b2f448f665c485adb57b253056d476cf4" dependencies = [ "anyhow", "async-trait", @@ -5414,8 +5412,7 @@ dependencies = [ [[package]] name = "saorsa-transport" version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "beeb14b4a6d7a652dc0433eec1db536901c12c9532f0aaa922b1dd7e0fb58841" +source = "git+https://github.com/grumbach/saorsa-transport?branch=feat%2Fpeer-swarm#8e361de20f10a346762a8177dfda8eea7ebec369" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 7192f204..6396c587 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -248,3 +248,11 @@ cognitive_complexity = "allow" # Allow non-const functions during initial development (may need runtime features later) missing_const_for_fn = "allow" +# The swarm's wire types (ant-protocol #50), the send on one named channel +# (saorsa-core #183) and the plain STUN answers (saorsa-transport #231) are on +# these branches until they merge: drop each entry once a release includes its +# branch. +[patch.crates-io] +ant-protocol = { git = "https://github.com/grumbach/ant-protocol", branch = "feat/peer-swarm" } +saorsa-core = { git = "https://github.com/grumbach/saorsa-core", branch = "feat/peer-swarm" } +saorsa-transport = { git = "https://github.com/grumbach/saorsa-transport", branch = "feat/peer-swarm" } From c08126778d9efee519bec67ef6dbadbc721e89c4 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 6 Nov 2026 10:03:29 +0900 Subject: [PATCH 03/15] fix(notices): match git dependencies on branches whose name has a slash cargo tree percent-encodes the slash in a git branch's name and cargo metadata does not, so the notices generator matched no package on such a branch (this PR's dependency branches are feat/peer-swarm): it now compares both decoded. --- scripts/third_party_notices/generate.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/third_party_notices/generate.py b/scripts/third_party_notices/generate.py index b3bf57ed..15f3ee32 100644 --- a/scripts/third_party_notices/generate.py +++ b/scripts/third_party_notices/generate.py @@ -38,6 +38,7 @@ import time import tomllib import urllib.error +import urllib.parse import urllib.request LICENCE_FILE = re.compile( @@ -152,7 +153,11 @@ def resolve_package(name, version, annotation, by_name_version): elif annotation.startswith("registry "): raise Gap(f"{name} {version} comes from {annotation}; only crates.io is supported") elif "://" in annotation: - matches = [p for p in candidates if (p["source"] or "").startswith("git+" + annotation)] + # cargo tree percent-encodes a git branch's slashes, cargo metadata + # does not: compare both decoded. + wanted = "git+" + urllib.parse.unquote(annotation) + matches = [p for p in candidates + if urllib.parse.unquote(p["source"] or "").startswith(wanted)] else: wanted = os.path.realpath(annotation) matches = [p for p in candidates if p["source"] is None From 96b76e8d42d085af1454c9eb4470c7aa2ed17048 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 30 Oct 2026 20:33:05 +0900 Subject: [PATCH 04/15] test(swarm): compare empty inboxes with assert_eq Rust 1.99's clippy (assert_is_empty, which CI's stable toolchain runs) asks for assert_eq when a test checks that a value is empty, so a failure shows the value. --- src/swarm.rs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/swarm.rs b/src/swarm.rs index 65992ab4..c29f60cd 100644 --- a/src/swarm.rs +++ b/src/swarm.rs @@ -1964,10 +1964,7 @@ mod tests { answer_reply(&service, C, answer(1, a.member, 9)).await, SwarmAnswerResponse::Unmatched ); - assert!(call(&service, A, announce(1, &[], 0)) - .await - .inbox - .is_empty()); + assert_eq!(call(&service, A, announce(1, &[], 0)).await.inbox, []); } #[tokio::test] @@ -2209,7 +2206,7 @@ mod tests { assert_eq!(call(&service, B, announce(1, &[], 0)).await.inbox.len(), 1); let started = Instant::now(); let reply = call(&service, B, announce(1, &[], 300)).await; - assert!(reply.inbox.is_empty()); + assert_eq!(reply.inbox, []); assert!(started.elapsed() >= Duration::from_millis(250)); } @@ -2881,9 +2878,6 @@ mod tests { } assert_eq!(service.stats().pending_offers, 0); assert_eq!(service.stats().stored_sdp_bytes, 0); - assert!(call(&service, B, announce(1, &[], 0)) - .await - .inbox - .is_empty()); + assert_eq!(call(&service, B, announce(1, &[], 0)).await.inbox, []); } } From e486f57b4fee1cf2c7e462b47804fdb3d574dff0 Mon Sep 17 00:00:00 2001 From: grumbach Date: Mon, 2 Nov 2026 17:00:03 +0900 Subject: [PATCH 05/15] test(swarm): test the pending-offer cap and signal expiry on their own The pending-offer test never let B take its offers, so B's 32-signal inbox stopped delivery whatever A's own cap on unanswered offers did. B now collects every offer without answering, and A's 33rd offer is still refused while B's inbox is empty. The expiry test swept past the 70 s signal deadline, which is also past the 60 s a silent member lasts, so the members left and took their offers with them whether or not signals expired. The members are now kept fresh across that deadline, the offer and its SDP must still be there just before it and gone just after it, and silent members expiring is a test of its own. Each test fails with the limit or expiry it covers removed. --- src/swarm.rs | 50 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/src/swarm.rs b/src/swarm.rs index c29f60cd..668686fb 100644 --- a/src/swarm.rs +++ b/src/swarm.rs @@ -2050,18 +2050,51 @@ mod tests { assert_eq!(service.stats().members, 0); } + /// Marks every member of `room` as having announced at `at`, without the + /// announce itself, which would take the member's inbox. + fn seen_at(service: &SwarmService, room: u8, at: Instant) { + if let Some(room) = service.state.lock().rooms.get_mut(&[room; 32]) { + for member in room.members.values_mut() { + member.last_seen = at; + } + } + } + #[tokio::test] - async fn unanswered_offers_and_silent_members_expire() { + async fn unanswered_offers_expire_while_their_members_stay() { let service = service(); + let start = Instant::now(); let _b = call(&service, B, announce(1, &[], 0)).await; let _a = call(&service, A, announce(1, &[50], 0)).await; assert_eq!(service.stats().pending_offers, 1); assert!(service.stats().stored_sdp_bytes > 0); - let later = Instant::now() + SIGNAL_TTL + Duration::from_secs(1); - service.sweep(later); + // Both members keep announcing, so only the signal's own deadline can + // end the offer: it outlives MEMBER_TTL, which would otherwise take + // the members and their offers with them. + let before = start + SIGNAL_TTL.saturating_sub(Duration::from_secs(1)); + seen_at(&service, 1, before); + service.sweep(before); + assert_eq!(service.stats().members, 2); + assert_eq!(service.stats().pending_offers, 1); + assert!(service.stats().stored_sdp_bytes > 0); + let after = start + SIGNAL_TTL + Duration::from_secs(1); + seen_at(&service, 1, after); + service.sweep(after); + assert_eq!(service.stats().members, 2); assert_eq!(service.stats().pending_offers, 0); assert_eq!(service.stats().stored_sdp_bytes, 0); - service.sweep(Instant::now() + MEMBER_TTL + Duration::from_secs(1)); + } + + #[tokio::test] + async fn silent_members_expire_with_their_offers() { + let service = service(); + let start = Instant::now(); + let _b = call(&service, B, announce(1, &[], 0)).await; + let _a = call(&service, A, announce(1, &[50], 0)).await; + service.sweep(start + MEMBER_TTL.saturating_sub(Duration::from_secs(1))); + assert_eq!(service.stats().members, 2); + // Silent for MEMBER_TTL, well before the offer's own deadline. + service.sweep(start + MEMBER_TTL + Duration::from_secs(1)); assert_eq!(service.stats(), SwarmStats::default()); } @@ -2478,12 +2511,19 @@ mod tests { let service = service(); let _b = call(&service, B, announce(1, &[], 0)).await; let mut delivered = 0; + let mut received = 0; for offer in 0..40u8 { delivered += call(&service, A, announce(1, &[offer], 0)).await.delivered; + // B collects every offer and answers none, so its inbox always has + // room: only A's own cap on unanswered offers can stop them. + received += call(&service, B, announce(1, &[], 0)).await.inbox.len(); } - // B's inbox holds 32, A may have 32 offers out: both caps meet at 32. assert_eq!(delivered, 32); + assert_eq!(received, 32); assert_eq!(service.stats().pending_offers, 32); + // A's next offer is still refused while B's inbox is empty. + assert_eq!(call(&service, A, announce(1, &[40], 0)).await.delivered, 0); + assert_eq!(call(&service, B, announce(1, &[], 0)).await.inbox, []); } #[tokio::test] From 068f609b43d307e913bfc6b5bef24b478a7b4fb3 Mon Sep 17 00:00:00 2001 From: grumbach Date: Mon, 2 Nov 2026 17:15:36 +0900 Subject: [PATCH 06/15] test(swarm): only the member an offer went to can answer it The earlier test answered an offer that did not exist, so it passed with the recipient check removed. Now A's live offer goes to B, a third member of the same room answers it with its real id and is refused, the offer stays pending, and B's answer still reaches A. It fails if any member of the room may answer. --- src/swarm.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/swarm.rs b/src/swarm.rs index 668686fb..4e7eef6e 100644 --- a/src/swarm.rs +++ b/src/swarm.rs @@ -1967,6 +1967,30 @@ mod tests { assert_eq!(call(&service, A, announce(1, &[], 0)).await.inbox, []); } + #[tokio::test] + async fn only_the_member_an_offer_went_to_can_answer_it() { + let service = service(); + let _b = call(&service, B, announce(1, &[], 0)).await; + // A's offer 50 goes to B, the only other member then. + let a = call(&service, A, announce(1, &[50], 0)).await; + assert_eq!(a.delivered, 1); + // C joins the same room and answers it with its real id. + let _c = call(&service, C, announce(1, &[], 0)).await; + assert_eq!( + answer_reply(&service, C, answer(1, a.member, 50)).await, + SwarmAnswerResponse::Unmatched + ); + // That took nothing: the offer still waits for B, and A has no answer. + assert_eq!(service.stats().pending_offers, 1); + assert_eq!(call(&service, A, announce(1, &[], 0)).await.inbox, []); + assert_eq!( + answer_reply(&service, B, answer(1, a.member, 50)).await, + SwarmAnswerResponse::Delivered + ); + assert_eq!(service.stats().pending_offers, 0); + assert_eq!(call(&service, A, announce(1, &[], 0)).await.inbox.len(), 1); + } + #[tokio::test] async fn a_waiting_announce_wakes_on_the_first_signal() { let service = service(); From 356382d58e46c1340f6498f5177db5041b282196 Mon Sep 17 00:00:00 2001 From: grumbach Date: Mon, 2 Nov 2026 17:15:36 +0900 Subject: [PATCH 07/15] docs(adr): every fetch of a client's holds its sends to one The client now counts each of its own fetches from the nodes, not only reads of a file's data records, when it limits what it sends other readers: opening a file's map, a raw read or an existence check needs its CPU and line first too. --- .../ADR-0026-clients-share-records-nodes-introduce-them.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md index 1328f83c..ac99b37f 100644 --- a/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md +++ b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md @@ -531,8 +531,9 @@ saorsa-transport's WebRTC Direct already uses). room was joined are offered too. Records a client already holds when it joins a room are told to the room's links as the room learns its file's records, so a link that opened first still hears of them. A client sends at - most 4 records at once, and starts no send past 1 while reads of its own - are running (sends already running when a read starts finish; the count + most 4 records at once, and starts no send past 1 while fetches of its own + are running, whatever they are for (a file's records or its map, a raw + read or an existence check; sends already running when one starts finish; the count is taken in one step, so two links never both take the last place): a record sent costs the sender's CPU and line, which its own download needs first @@ -579,7 +580,7 @@ saorsa-transport's WebRTC Direct already uses). at most 2 offers per announce, each a WebRTC connection with up to 3 s of ICE gathering and every negotiation step bounded, and at most 8 links; per room, at most 8 answers started in 20 s; per client: 4 rooms, 16 - links, 8 answers being made, 4 records sent (1 while any of its own reads + links, 8 answers being made, 4 records sent (1 while any of its own fetches runs) and 8 received at once. In a browser that makes connections take turns to gather ICE (every engine but Chromium), the swarm's own connections take From aa4e5905f33a425abaee7416e0418ebdf350c0a9 Mon Sep 17 00:00:00 2001 From: grumbach Date: Mon, 2 Nov 2026 18:28:21 +0900 Subject: [PATCH 08/15] fix(swarm): hold native swarm requests to a budget of their own Browsers' swarm requests run on a budget of 64 a second per node and 8 per source, but native clients' only had a cap on tasks running at once. A native client can make identities for free, so one host could announce with wait_ms 0 and fresh memberships as fast as the slots turned over, and every replacement does its work under the rendezvous lock browsers use too. Native swarm requests now take a place from a budget of their own with the same figures, 64 a second in all and 8 from one host (an IPv4 address or an IPv6 /64), before they are decoded or given a task; past it they are dropped and counted (native_over_budget), and the client reads from nodes alone. The node's budget is taken first, so once it is spent a flood costs one check a request. At most 4096 hosts' budgets are remembered, those that have refilled forgotten first, which always leaves room for a new host since the node's own budget keeps all but a few full. Tests: a host's identities and an IPv6 /64 share one budget, many hosts together get the node's and no more, the hosts remembered stay bounded and a forgotten one starts afresh, and a flood through the router from one host is held to its budget while another host gets through. They fail with the router's check or the per-host budget removed. ADR-0026 says so. --- ...ents-share-records-nodes-introduce-them.md | 15 +- src/node.rs | 65 +++++- src/swarm.rs | 210 ++++++++++++++++++ 3 files changed, 286 insertions(+), 4 deletions(-) diff --git a/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md index ac99b37f..07180476 100644 --- a/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md +++ b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md @@ -245,7 +245,13 @@ may only be appended. arrives once the reads' budget is spent is taken on the swarm's instead, since only the opened frame says whether it is a swarm request: one that is goes on, and anything else gives the swarm its token back and is - refused as over the request rate, as before. + refused as over the request rate, as before. Native swarm requests have a + budget of their own with the same figures (64 a second in all, 8 from one + host, an IPv4 address or an IPv6 /64, since a native client's identities + cost nothing to make), taken before the request is decoded or given a + task: one past it is dropped, and its client reads from nodes alone. The + node remembers at most 4096 hosts' budgets, forgetting first those that + have refilled, which leaves room for a new host however many came before. - **Closing.** A connection that closes leaves all its rooms at once. No request of it still being handled can act after the close (announce, answer or leave), nor any request older than a minute. A member that goes takes @@ -702,8 +708,11 @@ saorsa-transport's WebRTC Direct already uses). a later wait short, requests from before a close joining nothing, replayed or stale native requests, histories started only by announces and leaves and capped per host, ended memberships refused however late their - announce, kept per connection and host, members capped per host, and a departing member's signals - and the offers made to it going with it. + announce, kept per connection and host, members capped per host, a departing member's signals + and the offers made to it going with it, and the native request budget (a + host's identities and IPv6 /64 sharing one, many hosts held to the node's, + the hosts remembered staying bounded, and a flood through the router held + to it while another host gets through). - ant-client unit tests over an in-memory WebRTC stand-in: a record served to a second and a third client, nothing asked for a record nobody announced, a forged record and out-of-order pieces ending the link, a stalled peer costing diff --git a/src/node.rs b/src/node.rs index 482854d1..4ed4ef07 100644 --- a/src/node.rs +++ b/src/node.rs @@ -151,6 +151,11 @@ impl NativeSwarmRouter { debug!("Swarm request from {source} dropped: no connection to answer on"); return true; }; + let source_ip = transport_source.as_ref().and_then(MultiAddr::ip); + if !self.swarm.admit_native(source_ip, Instant::now()) { + debug!("Swarm request from {source} dropped: over the swarm's request budget"); + return true; + } let Ok(permit) = Arc::clone(&self.slots).try_acquire_owned() else { debug!("Swarm request from {source} dropped: swarm slots full"); return true; @@ -161,7 +166,7 @@ impl NativeSwarmRouter { let request = SwarmRequest { source: *source, channel, - source_ip: transport_source.as_ref().and_then(MultiAddr::ip), + source_ip, signed_at: *timestamp, received: Instant::now(), data: data.clone(), @@ -1386,6 +1391,10 @@ fn jittered_interval(base: std::time::Duration) -> std::time::Duration { #[cfg(test)] #[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] mod tests { + use std::net::SocketAddr; + + use ant_protocol::swarm::SwarmAnnounceRequest; + use ant_protocol::ChunkMessageBody; use tempfile::TempDir; use crate::storage::legacy_artifacts::LEGACY_ENV_DIR; @@ -1451,6 +1460,60 @@ mod tests { node.shutdown.cancel(); } + /// A native host flooding swarm requests, under as many identities as it + /// likes, has its budget's worth routed to the rendezvous and no more, and + /// an honest host beside it still gets through. + #[tokio::test] + async fn a_native_hosts_swarm_flood_is_held_to_its_budget() { + let dir = TempDir::new().expect("temp dir"); + let root = dir.path().join("node"); + std::fs::create_dir_all(&root).expect("mkdir"); + let node = NodeBuilder::new(local_node_config(&root)) + .build() + .await + .expect("a node builds on a clean root"); + let swarm = SwarmService::new(); + let router = NativeSwarmRouter::start( + Arc::clone(&swarm), + Arc::clone(&node.p2p_node), + TaskTracker::new(), + node.shutdown.clone(), + ); + let request = ChunkMessage { + request_id: 1, + body: ChunkMessageBody::SwarmAnnounceRequest(SwarmAnnounceRequest { + room: [1; 32], + node: [2; 32], + membership: 1, + offers: Vec::new(), + wait_ms: 0, + }), + } + .encode() + .expect("encode"); + let from = |ip: &str, identity: u8| P2PEvent::Message { + topic: CHUNK_PROTOCOL_ID.to_string(), + source: Some(PeerId::from_bytes([identity; 32])), + transport_source: Some(MultiAddr::quic(SocketAddr::new( + ip.parse().expect("ip"), + 5000, + ))), + timestamp: 0, + data: request.clone(), + }; + let over_budget = || swarm.counters().native_over_budget.load(Ordering::Relaxed); + let flood: u32 = 100; + for identity in 0..flood { + let identity = u8::try_from(identity).expect("fits"); + assert!(router.route(&from("198.51.100.4", identity))); + } + let dropped = flood - swarm::NATIVE_SWARM_REQUESTS_PER_SECOND_PER_HOST; + assert_eq!(over_budget(), u64::from(dropped)); + assert!(router.route(&from("198.51.100.5", 0))); + assert_eq!(over_budget(), u64::from(dropped)); + node.shutdown.cancel(); + } + /// A node with chunks in a store this build cannot read STARTS, however it is /// configured, and does not lose them. /// diff --git a/src/swarm.rs b/src/swarm.rs index 4e7eef6e..150f1a59 100644 --- a/src/swarm.rs +++ b/src/swarm.rs @@ -248,6 +248,8 @@ pub struct SwarmCounters { pub refused: AtomicU64, /// Members dropped because they stopped announcing. pub members_expired: AtomicU64, + /// Native requests dropped past the node's or their host's budget. + pub native_over_budget: AtomicU64, } /// Counts at one moment. @@ -373,6 +375,8 @@ pub struct SwarmService { /// signed request this node saw cannot be replayed at another, nor one /// another node saw here. Until it is known, every request is refused. node_id: RwLock>, + /// See [`NATIVE_SWARM_REQUESTS_PER_SECOND`]. + native_budget: Mutex, } impl std::fmt::Debug for SwarmService { @@ -386,6 +390,114 @@ impl std::fmt::Debug for SwarmService { /// Swarm requests handled at once on the native path. Above /// [`MAX_WAITING`] so announces that cannot wait still get through. pub const MAX_SWARM_TASKS: usize = MAX_WAITING + 256; +/// Native clients' swarm requests a node takes in a second, in all. +/// +/// With [`NATIVE_SWARM_REQUESTS_PER_SECOND_PER_HOST`], the native path's own +/// budget, as the browser listener's swarm budget is for browsers. An identity +/// costs a native client nothing, so its requests are counted by host (an +/// IPv4 address or an IPv6 /64). Past either, a request is dropped before any +/// work; its client times out and reads from nodes alone. +pub const NATIVE_SWARM_REQUESTS_PER_SECOND: u32 = 64; +/// See [`NATIVE_SWARM_REQUESTS_PER_SECOND`]. +pub const NATIVE_SWARM_REQUESTS_PER_SECOND_PER_HOST: u32 = 8; +/// Hosts whose native swarm budget is remembered at once. A host whose +/// budget has refilled is as good as new, so those go first when it is full; +/// past it, a request from a host not yet counted is dropped. +pub const MAX_BUDGETED_HOSTS: usize = 4096; + +/// A token bucket of `rate` requests a second, holding at most one second's. +#[derive(Debug)] +struct RateBucket { + rate: u128, + /// Tokens, in billionths. + units: u128, + last: Instant, +} + +const TOKEN: u128 = 1_000_000_000; + +impl RateBucket { + fn new(rate: u32, now: Instant) -> Self { + let rate = u128::from(rate); + Self { + rate, + units: rate.saturating_mul(TOKEN), + last: now, + } + } + + fn refill(&mut self, now: Instant) { + let elapsed = now.saturating_duration_since(self.last); + self.last = self.last.max(now); + let capacity = self.rate.saturating_mul(TOKEN); + self.units = self + .units + .saturating_add(elapsed.as_nanos().saturating_mul(self.rate)) + .min(capacity); + } + + fn take(&mut self, now: Instant) -> bool { + self.refill(now); + if self.units < TOKEN { + return false; + } + self.units -= TOKEN; + true + } + + fn refund(&mut self) { + self.units = self + .units + .saturating_add(TOKEN) + .min(self.rate.saturating_mul(TOKEN)); + } + + fn is_full(&mut self, now: Instant) -> bool { + self.refill(now); + self.units >= self.rate.saturating_mul(TOKEN) + } +} + +/// The native path's swarm request budget: the node's and each host's. +#[derive(Debug)] +struct NativeBudget { + all: RateBucket, + hosts: HashMap, +} + +impl NativeBudget { + /// Takes a place from the node's budget and from `ip`'s host's, both or + /// neither. The node's is taken first, so once it is spent a flood costs + /// one check a request. + fn admit(&mut self, ip: IpAddr, now: Instant) -> bool { + if !self.all.take(now) { + return false; + } + let host = host_of(ip); + if !self.hosts.contains_key(&host) && self.hosts.len() >= MAX_BUDGETED_HOSTS { + self.hosts.retain(|_, bucket| !bucket.is_full(now)); + } + let taken = (self.hosts.contains_key(&host) || self.hosts.len() < MAX_BUDGETED_HOSTS) + && self + .hosts + .entry(host) + .or_insert_with(|| RateBucket::new(NATIVE_SWARM_REQUESTS_PER_SECOND_PER_HOST, now)) + .take(now); + if !taken { + self.all.refund(); + } + taken + } +} + +impl Default for NativeBudget { + fn default() -> Self { + Self { + all: RateBucket::new(NATIVE_SWARM_REQUESTS_PER_SECOND, Instant::now()), + hosts: HashMap::new(), + } + } +} // The wire discriminants of the swarm requests in `ChunkMessageBody`, after // the get-or-closer pair at 14 and 15 (ant-protocol pins them in a test). @@ -769,6 +881,20 @@ impl SwarmService { &self.counters } + /// Whether a native swarm request from `ip` may be handled now: it takes + /// a place from the node's budget and from its host's, both or neither + /// (see [`NATIVE_SWARM_REQUESTS_PER_SECOND`]). A request with no source + /// address is refused. + pub fn admit_native(&self, ip: Option, now: Instant) -> bool { + let admitted = ip.is_some_and(|ip| self.native_budget.lock().admit(ip, now)); + if !admitted { + self.counters + .native_over_budget + .fetch_add(1, Ordering::Relaxed); + } + admitted + } + /// Whether a reply to a request of `session` received at `received` /// should be dropped: the connection closed since, so sending it would /// only make the node dial a client that left. @@ -1967,6 +2093,90 @@ mod tests { assert_eq!(call(&service, A, announce(1, &[], 0)).await.inbox, []); } + #[test] + fn a_hosts_native_requests_share_one_budget() { + let service = service(); + let now = Instant::now(); + let host: IpAddr = "198.51.100.4".parse().expect("ip"); + for _ in 0..NATIVE_SWARM_REQUESTS_PER_SECOND_PER_HOST { + assert!(service.admit_native(Some(host), now)); + } + // Whatever identities it uses, the host has spent its second. + assert!(!service.admit_native(Some(host), now)); + assert_eq!( + service + .counters() + .native_over_budget + .load(Ordering::Relaxed), + 1 + ); + // Another host is not held back, and the refused request took + // nothing of the node's budget. + let other: IpAddr = "198.51.100.5".parse().expect("ip"); + assert!(service.admit_native(Some(other), now)); + let all = service.native_budget.lock().all.units / TOKEN; + assert_eq!( + all, + u128::from( + NATIVE_SWARM_REQUESTS_PER_SECOND - NATIVE_SWARM_REQUESTS_PER_SECOND_PER_HOST - 1 + ) + ); + // A second later the host has its budget back. + assert!(service.admit_native(Some(host), now + Duration::from_secs(1))); + // A request with no source address is never admitted. + assert!(!service.admit_native(None, now)); + } + + #[test] + fn an_ipv6_hosts_addresses_share_its_budget() { + let service = service(); + let now = Instant::now(); + for n in 0..NATIVE_SWARM_REQUESTS_PER_SECOND_PER_HOST { + let ip: IpAddr = format!("2001:db8:1:2::{n:x}").parse().expect("ip"); + assert!(service.admit_native(Some(ip), now)); + } + let same_64: IpAddr = "2001:db8:1:2:ffff::1".parse().expect("ip"); + assert!(!service.admit_native(Some(same_64), now)); + let next_64: IpAddr = "2001:db8:1:3::1".parse().expect("ip"); + assert!(service.admit_native(Some(next_64), now)); + } + + #[test] + fn many_hosts_together_get_the_nodes_budget_and_no_more() { + let service = service(); + let now = Instant::now(); + let host = |n: u32| IpAddr::from(std::net::Ipv4Addr::from(0x0a00_0000 + n)); + for n in 0..NATIVE_SWARM_REQUESTS_PER_SECOND { + assert!(service.admit_native(Some(host(n)), now)); + } + // A host that has sent nothing is refused too: the node's second is spent. + assert!(!service.admit_native(Some(host(10_000)), now)); + // It refills at its rate. + let later = now + Duration::from_secs(1) / NATIVE_SWARM_REQUESTS_PER_SECOND; + assert!(service.admit_native(Some(host(10_000)), later)); + assert!(!service.admit_native(Some(host(10_001)), later)); + } + + #[test] + fn the_hosts_remembered_stay_bounded_and_a_new_host_still_gets_in() { + let service = service(); + let start = Instant::now(); + // More hosts than are remembered, each once, slower than the node's + // budget refills. + let spacing = Duration::from_millis(20); + let total = u32::try_from(MAX_BUDGETED_HOSTS).expect("fits") + 100; + let mut at = start; + for n in 0..total { + at = start + spacing * n; + let ip = IpAddr::from(std::net::Ipv4Addr::from(0x0a00_0000 + n)); + assert!(service.admit_native(Some(ip), at), "host {n}"); + assert!(service.native_budget.lock().hosts.len() <= MAX_BUDGETED_HOSTS); + } + // A host it forgot (its budget had refilled) starts afresh. + let first = IpAddr::from(std::net::Ipv4Addr::from(0x0a00_0000)); + assert!(service.admit_native(Some(first), at)); + } + #[tokio::test] async fn only_the_member_an_offer_went_to_can_answer_it() { let service = service(); From 676c0afad5f9d82327852d0e9ae6d3d16661724c Mon Sep 17 00:00:00 2001 From: grumbach Date: Mon, 2 Nov 2026 20:29:50 +0900 Subject: [PATCH 09/15] docs(adr): a waiting room join is made when its file is joined again or read The client now tries a waiting room join again when the file is joined again or a read of one of its records starts, not only when a record of it comes in, so a file whose reads the nodes fail still joins once a room has gone unread. --- .../ADR-0026-clients-share-records-nodes-introduce-them.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md index 07180476..1e1f05c7 100644 --- a/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md +++ b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md @@ -332,8 +332,9 @@ saorsa-transport's WebRTC Direct already uses). file's record keeps the file's room in use for as long as it runs, however long the nodes take. Joining past 4 rooms leaves the room unread longest, once no read has used it for 30 s; with every room still read, the join waits (at - most 2 do) and is made at a later read of one of the file's records once a - room has gone unread, the file meanwhile read from the nodes. A client + most 2 do) and is made once a room has gone unread, when the file is + joined again or a read of one of its records starts or ends, the file + meanwhile read from the nodes. A client reading more files than that at once so cannot make every read a new membership (a key, a lookup, links that never open), and a long download's room is never left for a file opened beside it; at most 8 rooms left at a time tell their nodes so (each From 8cb14657997167845c19d15309cdedf010350b4e Mon Sep 17 00:00:00 2001 From: grumbach Date: Tue, 3 Nov 2026 22:39:36 +0900 Subject: [PATCH 10/15] fix(swarm): host rooms only under the endpoint a refresh published The swarm names the browser endpoint's address as its rooms' STUN server and hosts rooms only while it has one. Publishing a changed endpoint writes the endpoint file first, which can fail and keep the old publication: if that was a gateway mapping that has ended or moved, the swarm went on naming an address the node no longer has. Every refresh of the endpoint now ends by setting the swarm's STUN server from its outcome: the published endpoint's address after a refresh that succeeded, none after one that failed. A refresh that changes or withdraws the endpoint also stops the swarm hosting before its first await (the file write, or the DHT record's withdrawal, which can wait on peers), so the swarm never names the old address meanwhile. This covers an old endpoint published again before a changed one could be, and a first endpoint that cannot be published. The refresh's former body is `publish_browser_endpoint`, unchanged but for that clearing. Four tests block the endpoint file: a moved mapping (the old endpoint stays published, the swarm hosts no rooms, then names the moved mapping once the file can be written), the old mapping coming back before the moved one is published (the swarm names it again), a first publication, and, on Unix, a FIFO that holds the moved mapping's write while the test checks the swarm hosts no rooms. The first two fail if the swarm takes the published endpoint even after a failed refresh, they and main's withdrawal test fail if it is never updated, and the FIFO test fails without the clearing before the write. ADR-0026 says when a node hosts rooms. --- ...ents-share-records-nodes-introduce-them.md | 11 +- src/swarm.rs | 8 +- src/web_rtc.rs | 200 +++++++++++++++++- 3 files changed, 207 insertions(+), 12 deletions(-) diff --git a/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md index 1e1f05c7..22236095 100644 --- a/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md +++ b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md @@ -94,10 +94,13 @@ announces itself at the `SWARM_RENDEZVOUS_NODES` (2) nodes closest to the key that host rooms, so two readers whose views of the close group differ slightly still share one, and readers still meet while some of the closest nodes predate rooms: it looks up the 8 nodes closest to the key and passes -over those that host none (see Older nodes). A node hosts rooms only once -browsers can reach it: until its browser listener has an endpoint, and so a -STUN server to name, it refuses every announce as `NotHosting`, and a client -passes it over at once, with no pause. Native readers so pass over the same +over those that host none (see Older nodes). A node hosts rooms only while +browsers can reach it, under the endpoint its browser listener has published, +whose address it names as their STUN server: until one is published, while a +changed one is being published and after a refresh that failed to (the old one +may be a gateway mapping that has ended), and from the start of the withdrawal +of an endpoint only a mapping reached (ADR-0027), it refuses every announce as +`NotHosting`, and a client passes it over at once, with no pause. Native readers so pass over the same nodes browsers never see (a relay-only node, or one built without the listener), and meet the browsers at the closest nodes both can reach, rather than at nodes no browser comes to and that give them diff --git a/src/swarm.rs b/src/swarm.rs index 150f1a59..0bb231e4 100644 --- a/src/swarm.rs +++ b/src/swarm.rs @@ -907,8 +907,14 @@ impl SwarmService { *self.node_id.write() = Some(node_id); } + /// The STUN servers handed to members: none while the node hosts no rooms. + #[cfg(test)] + pub(crate) fn ice_servers(&self) -> Vec { + self.ice_servers.read().clone() + } + /// Sets the STUN servers handed to members (this node's own, once its - /// public address is known). + /// public address is known; none once browsers can no longer reach it). pub fn set_ice_servers(&self, servers: Vec) { *self.ice_servers.write() = servers .into_iter() diff --git a/src/web_rtc.rs b/src/web_rtc.rs index 9deda562..bb37d18f 100644 --- a/src/web_rtc.rs +++ b/src/web_rtc.rs @@ -1108,11 +1108,39 @@ async fn log_listener_summaries(diagnostics: WebRtcServerDiagnostics, shutdown: debug!("WebRTC Direct listener summary loop shut down"); } +/// Refresh the browser endpoint (see [`publish_browser_endpoint`]), then let +/// the swarm host rooms only under the endpoint left published, naming its +/// address as their STUN server: none after a refresh that failed, which can +/// leave an old endpoint published (a gateway mapping that has ended or moved). +/// A refresh that changes or withdraws the endpoint stops the swarm hosting +/// before its first await, so it never names the old address meanwhile. async fn refresh_browser_endpoint( state: &ServerState, root_dir: &Path, local_addr: SocketAddr, certificate_sha256: [u8; 32], +) -> Result<()> { + let result = publish_browser_endpoint(state, root_dir, local_addr, certificate_sha256).await; + let stun = match &result { + Ok(()) => state.publication.read().endpoint().and_then(stun_server), + Err(_) => None, + }; + state.swarm.set_ice_servers(stun.into_iter().collect()); + result +} + +/// The STUN server a published endpoint's socket answers as: the same UDP +/// socket answers swarm members' STUN requests. +fn stun_server(endpoint: &BrowserEndpoint) -> Option { + let address = endpoint.parse().ok()?.socket_addr().ok()?; + Some(format!("stun:{address}")) +} + +async fn publish_browser_endpoint( + state: &ServerState, + root_dir: &Path, + local_addr: SocketAddr, + certificate_sha256: [u8; 32], ) -> Result<()> { let native = state .p2p @@ -1140,12 +1168,13 @@ async fn refresh_browser_endpoint( .multiaddr .parse() .map_err(|error| Error::Startup(format!("invalid WebRTC transport address: {error}")))?; + // The published endpoint changes, possibly from a mapping that has ended + // or moved: no rooms until the refresh ends (see `refresh_browser_endpoint`). + state.swarm.set_ice_servers(Vec::new()); persist_browser_endpoint(root_dir, &endpoint).await?; if let Some(catalog) = state.endpoint_catalog.as_ref() { catalog.insert(peer_id, endpoint.clone()); } - // The same UDP socket answers swarm members' STUN requests. - state.swarm.set_ice_servers(vec![format!("stun:{address}")]); *state.publication.write() = Publication::Published { endpoint, source }; state .p2p @@ -1156,11 +1185,11 @@ async fn refresh_browser_endpoint( } /// Withdraw an endpoint whose address only a gateway mapping that has ended -/// provided, everywhere it was published: HELLO, the in-process catalog and -/// the DHT address record first, once, then the endpoint file. Any other -/// publication is kept. +/// provided, everywhere it was published: HELLO, the in-process catalog, the +/// swarm's STUN server and the DHT address record first, once, then the +/// endpoint file. Any other publication is kept. /// -/// The first three cannot fail, so a file error never keeps the endpoint +/// The first four cannot fail, so a file error never keeps the endpoint /// advertised. It leaves the publication [`Publication::Withdrawing`], and the /// next call retries only the file. async fn withdraw_mapped_endpoint(state: &ServerState, root_dir: &Path) -> Result<()> { @@ -1171,6 +1200,8 @@ async fn withdraw_mapped_endpoint(state: &ServerState, root_dir: &Path) -> Resul if let Some(catalog) = state.endpoint_catalog.as_ref() { catalog.remove(state.p2p.peer_id()); } + // No rooms while the DHT record is withdrawn, which can wait on peers. + state.swarm.set_ice_servers(Vec::new()); state .p2p .dht_manager() @@ -4130,11 +4161,13 @@ mod tests { .await } - /// Assert that peers and browsers no longer see the endpoint. + /// Assert that peers and browsers no longer see the endpoint, and that + /// the swarm hosts no rooms without it. async fn assert_unpublished(&self) { assert_eq!(self.publication().endpoint(), None); assert_eq!(self.catalog.get(&self.peer_id()), None); assert_eq!(self.supplemental().await, Vec::::new()); + assert_eq!(self.state.swarm.ice_servers(), Vec::::new()); } } @@ -4162,6 +4195,10 @@ mod tests { .contains(&published.multiaddr)); assert_eq!(listener.catalog.get(&listener.peer_id()), Some(published)); assert_eq!(listener.supplemental().await.len(), 1); + assert_eq!( + listener.state.swarm.ice_servers(), + vec!["stun:198.51.100.2:51000".to_string()] + ); *listener.mapping.lock() = None; listener.refresh().await.expect("withdraw the mapping"); @@ -4206,6 +4243,155 @@ mod tests { assert_eq!(listener.publication(), Publication::Unpublished); } + #[tokio::test] + async fn the_swarm_hosts_no_rooms_while_a_changed_endpoint_cannot_be_published() { + let listener = MappedOnlyListener::new().await; + listener.refresh().await.expect("publish the mapping"); + + // A directory in the file's place makes writing it fail. + tokio::fs::remove_file(listener.file()) + .await + .expect("remove the endpoint file"); + tokio::fs::create_dir(listener.file()) + .await + .expect("block the endpoint file"); + *listener.mapping.lock() = Some("198.51.100.3:52000".parse().expect("moved mapping")); + listener + .refresh() + .await + .expect_err("the endpoint file cannot be written"); + assert!(listener + .publication() + .endpoint() + .expect("the old endpoint stays published") + .multiaddr + .contains("/198.51.100.2/udp/51000/")); + assert_eq!(listener.state.swarm.ice_servers(), Vec::::new()); + + // Once the file can be written, the moved mapping is published and + // the swarm names it. + tokio::fs::remove_dir(listener.file()) + .await + .expect("unblock the endpoint file"); + listener.refresh().await.expect("publish the moved mapping"); + assert!(listener + .publication() + .endpoint() + .expect("the moved endpoint is published") + .multiaddr + .contains("/198.51.100.3/udp/52000/")); + assert_eq!( + listener.state.swarm.ice_servers(), + vec!["stun:198.51.100.3:52000".to_string()] + ); + } + + #[tokio::test] + async fn the_swarm_hosts_rooms_again_when_its_published_endpoint_comes_back() { + let listener = MappedOnlyListener::new().await; + listener.refresh().await.expect("publish the mapping"); + tokio::fs::remove_file(listener.file()) + .await + .expect("remove the endpoint file"); + tokio::fs::create_dir(listener.file()) + .await + .expect("block the endpoint file"); + *listener.mapping.lock() = Some("198.51.100.3:52000".parse().expect("moved mapping")); + listener + .refresh() + .await + .expect_err("the endpoint file cannot be written"); + assert_eq!(listener.state.swarm.ice_servers(), Vec::::new()); + + // The mapping returns to the published endpoint before the moved one + // could be published: nothing is written, and the swarm names it again. + *listener.mapping.lock() = Some("198.51.100.2:51000".parse().expect("mapping")); + listener + .refresh() + .await + .expect("reconfirm the published endpoint"); + assert_eq!( + listener.state.swarm.ice_servers(), + vec!["stun:198.51.100.2:51000".to_string()] + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn the_swarm_hosts_no_rooms_while_a_changed_endpoint_is_written() { + let listener = MappedOnlyListener::new().await; + listener.refresh().await.expect("publish the mapping"); + + // A FIFO in the file's place holds the write until a reader opens it. + tokio::fs::remove_file(listener.file()) + .await + .expect("remove the endpoint file"); + let path = std::ffi::CString::new(listener.file().as_os_str().as_encoded_bytes()) + .expect("endpoint file path"); + #[allow(unsafe_code)] + let rc = unsafe { libc::mkfifo(path.as_ptr(), 0o600) }; + assert_eq!(rc, 0, "mkfifo failed: {}", std::io::Error::last_os_error()); + *listener.mapping.lock() = Some("198.51.100.3:52000".parse().expect("moved mapping")); + + let refresh = listener.refresh(); + tokio::pin!(refresh); + let mut finished = None; + for _ in 0..250 { + if listener.state.swarm.ice_servers().is_empty() { + break; + } + if let Ok(result) = tokio::time::timeout(Duration::from_millis(20), &mut refresh).await + { + finished = Some(result); + break; + } + } + let while_written = listener.state.swarm.ice_servers(); + if finished.is_none() { + if let Ok(result) = tokio::time::timeout(Duration::from_millis(20), &mut refresh).await + { + finished = Some(result); + } + } + let finished_unread = finished.is_some(); + + // A reader lets a write still waiting finish, so a failed check never + // leaves the write blocked. + let result = if let Some(result) = finished { + result + } else { + let fifo = listener.file(); + let reader = tokio::task::spawn_blocking(move || std::fs::read(fifo)); + let result = refresh.await; + reader + .await + .expect("reader task") + .expect("read the endpoint"); + result + }; + assert!(!finished_unread, "the endpoint write waits for a reader"); + assert_eq!(while_written, Vec::::new()); + result.expect("publish the moved mapping"); + assert_eq!( + listener.state.swarm.ice_servers(), + vec!["stun:198.51.100.3:52000".to_string()] + ); + } + + #[tokio::test] + async fn the_swarm_hosts_no_rooms_when_the_first_endpoint_cannot_be_published() { + let listener = MappedOnlyListener::new().await; + tokio::fs::create_dir(listener.file()) + .await + .expect("block the endpoint file"); + listener + .refresh() + .await + .expect_err("the endpoint file cannot be written"); + assert_eq!(listener.publication(), Publication::Unpublished); + assert_eq!(listener.state.swarm.ice_servers(), Vec::::new()); + } + #[test] fn only_a_mapped_publication_is_withdrawn() { let endpoint = BrowserEndpoint::new( From 94b48e435bef41529b220220cbce75960ff5deb4 Mon Sep 17 00:00:00 2001 From: grumbach Date: Wed, 4 Nov 2026 01:46:32 +0900 Subject: [PATCH 11/15] fix(swarm): compile the STUN server accessor only with the browser listener `SwarmService::ice_servers` is read only by the browser listener's tests in src/web_rtc.rs, a module that exists only with the `webrtc-direct` feature. Without that feature the accessor went unused, so the CI job that runs `cargo test --lib --no-default-features` under `-D warnings` failed to compile the tests. Gate it on `webrtc-direct` as well as `test`. Without default features, the library's tests build with `-D warnings` and 1190 pass; with default features 1251 pass. The previous gate fails the same build with "method `ice_servers` is never used". --- src/swarm.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/swarm.rs b/src/swarm.rs index 0bb231e4..b7ad8f72 100644 --- a/src/swarm.rs +++ b/src/swarm.rs @@ -908,7 +908,8 @@ impl SwarmService { } /// The STUN servers handed to members: none while the node hosts no rooms. - #[cfg(test)] + /// Only the browser listener's tests read it, so it exists only with them. + #[cfg(all(test, feature = "webrtc-direct"))] pub(crate) fn ice_servers(&self) -> Vec { self.ice_servers.read().clone() } From b287e01b0f2c31bd2612f660c425a94efc97c5b4 Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 5 Nov 2026 13:10:45 +0900 Subject: [PATCH 12/15] fix(swarm): give a frame's swarm charge back on every early exit A request frame that arrives once the reads' budget is spent is admitted on the swarm's own budget, since only the opened frame says whether it is a swarm request. The charge was given back only when the opened frame turned out to be some other request. Every earlier exit kept it: a frame that failed to arrive (bytes after its declared end, the channel closing partway), that did not open under the PQ session, whose header did not parse, or whose byte reservation was refused, and a read the connection abandoned partway through a frame. A connected browser could so spend its source's and the listener's swarm tokens with malformed frames, and hold them from the swarm requests of other connections until they refilled. The charge is now provisional: a guard taken at admission gives it back when dropped, on every exit and on cancellation, and only a frame known for a swarm request keeps it. The request reader takes its messages through a small trait, implemented for the DataChannel, so tests can feed it frames. Tests exhaust the reads' budget, then send twice the source's swarm budget of each kind of failing frame, with a real PQ session for those that open, and check that both swarm buckets are full after each and that a swarm request is still admitted on the swarm's budget and keeps its charge there. They fail with the refund removed. --- ...ents-share-records-nodes-introduce-them.md | 5 +- src/web_rtc.rs | 298 +++++++++++++++++- 2 files changed, 297 insertions(+), 6 deletions(-) diff --git a/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md index 22236095..9dbb40e3 100644 --- a/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md +++ b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md @@ -248,7 +248,10 @@ may only be appended. arrives once the reads' budget is spent is taken on the swarm's instead, since only the opened frame says whether it is a swarm request: one that is goes on, and anything else gives the swarm its token back and is - refused as over the request rate, as before. Native swarm requests have a + refused as over the request rate, as before. So does a frame that fails to + arrive, open or parse, whose buffers are refused, or whose read is + abandoned: the swarm keeps a token only from a frame known for a swarm + request. Native swarm requests have a budget of their own with the same figures (64 a second in all, 8 from one host, an IPv4 address or an IPv6 /64, since a native client's identities cost nothing to make), taken before the request is decoded or given a diff --git a/src/web_rtc.rs b/src/web_rtc.rs index bb37d18f..bf9a8109 100644 --- a/src/web_rtc.rs +++ b/src/web_rtc.rs @@ -41,7 +41,7 @@ use saorsa_transport::webrtc::{ }; use saorsa_transport::webrtc_direct::{ WebRtcAdmissionLimits, WebRtcDataChannel, WebRtcDiagnostics, WebRtcDiagnosticsSnapshot, - WebRtcDirectConnection, WebRtcDirectListener, + WebRtcDirectConnection, WebRtcDirectError, WebRtcDirectListener, }; use std::collections::{HashMap, HashSet, VecDeque}; use std::future::Future; @@ -567,6 +567,39 @@ enum FrameCharge { Swarm, } +/// A frame's charge to the swarm's own budget while the frame is not yet +/// known for a swarm request. Dropping it gives the charge back, so a frame +/// that fails to arrive, open or parse, whose buffers are refused, that turns +/// out to be no swarm request, or whose read is abandoned spends none of the +/// swarm's budget; [`Self::keep`] holds the charge once the frame is a swarm +/// request. +struct ProvisionalSwarmCharge<'a> { + /// The connection charged, or `None` once kept or for a frame the reads' + /// budget took. + resources: Option<&'a ConnectionResources>, +} + +impl<'a> ProvisionalSwarmCharge<'a> { + fn new(resources: &'a ConnectionResources, charge: FrameCharge) -> Self { + Self { + resources: (charge == FrameCharge::Swarm).then_some(resources), + } + } + + /// The frame is a swarm request: it keeps its charge. + fn keep(mut self) { + self.resources = None; + } +} + +impl Drop for ProvisionalSwarmCharge<'_> { + fn drop(&mut self) { + if let Some(resources) = self.resources.take() { + resources.refund_swarm(); + } + } +} + struct ConnectionResources { listener: Arc, source: Arc, @@ -664,7 +697,7 @@ impl ConnectionResources { } /// Gives the swarm's budget back a frame's charge: it was no swarm - /// request. + /// request (see [`ProvisionalSwarmCharge`]). fn refund_swarm(&self) { self.source.swarm_rate.lock().refund(); self.listener.global_swarm_rate.lock().refund(); @@ -2012,8 +2045,25 @@ async fn start_request( })? } +/// The messages a request frame arrives in: a browser's `DataChannel`, or a +/// test's. Shared by the request futures, which must be `Send`. +trait FrameChannel: Sync { + /// One binary message; an empty one means the channel closed. + fn receive( + &self, + ) -> impl Future, WebRtcDirectError>> + Send; +} + +impl FrameChannel for WebRtcDataChannel { + fn receive( + &self, + ) -> impl Future, WebRtcDirectError>> + Send { + Self::receive(self) + } +} + async fn read_webrtc_request( - channel: &WebRtcDataChannel, + channel: &impl FrameChannel, pq_session: &Mutex, resources: &ConnectionResources, active_requests: &watch::Receiver, @@ -2031,6 +2081,9 @@ async fn read_webrtc_request( // Admission happens as soon as a client starts a frame. Idle persistent // channels consume neither request-rate tokens nor request worker slots. let (request_permit, charge) = resources.try_admit_frame()?; + // Every exit before the frame is known for a swarm request, cancellation + // included, gives a swarm charge back. + let swarm_charge = ProvisionalSwarmCharge::new(resources, charge); let max_plaintext_bytes = MAX_BROWSER_FRAME_BYTES; let mut encrypted = read_pq_payload_after_first( first_message, @@ -2070,13 +2123,13 @@ async fn read_webrtc_request( if charge == FrameCharge::Swarm && !(matches!(request.body, RequestBody::ChunkProtocol) && is_swarm_request_bytes(&content)) { - resources.refund_swarm(); resources .listener .rate_rejections .fetch_add(1, Ordering::Relaxed); return Err(REQUEST_RATE_ERROR.to_string()); } + swarm_charge.keep(); Ok(AdmittedRequest { request, content, @@ -2125,7 +2178,7 @@ async fn receive_first_message( async fn read_pq_payload_after_first( first_message: Vec, - channel: &WebRtcDataChannel, + channel: &impl FrameChannel, max_payload_bytes: usize, frame_timeout_message: &str, resources: &ConnectionResources, @@ -3001,7 +3054,9 @@ mod tests { PointerGetRequest, PointerGetResponse, PointerPutRequest, PointerPutResponse, }; use ant_protocol::pointer::POINTER_WIRE_LEN; + use futures::FutureExt; use saorsa_core::NodeConfig as CoreNodeConfig; + use saorsa_transport::webrtc::{encode_pq_frame, PqClientHandshake}; use saorsa_transport::webrtc_direct::WebRtcCertificate; use std::net::Ipv4Addr; @@ -3565,6 +3620,239 @@ mod tests { assert_eq!(left(&browser.context.source.request_rate), 9); } + /// A request frame's `DataChannel` messages, after which the channel stays + /// open with nothing more to read. + struct QueuedMessages(Mutex>>); + + impl QueuedMessages { + fn new(messages: Vec>) -> Self { + Self(Mutex::new(messages.into())) + } + } + + impl FrameChannel for QueuedMessages { + fn receive( + &self, + ) -> impl Future, WebRtcDirectError>> + Send { + let next = self.0.lock().pop_front(); + async move { + match next { + Some(message) => Ok(message), + None => std::future::pending().await, + } + } + } + } + + /// Both ends of a PQ session, the browser's and the node's. + fn pq_session_pair() -> (PqSession, Mutex) { + let identity = NodeIdentity::generate().expect("identity"); + let peer_id = *identity.peer_id().to_bytes(); + let (handshake, hello) = PqClientHandshake::start().expect("client hello"); + let (accept, server) = accept_pq_session( + &hello, + &peer_id, + identity.public_key().as_bytes(), + |transcript| { + identity + .sign(transcript) + .map(|signature| signature.as_bytes().to_vec()) + }, + ) + .expect("server accept"); + let client = handshake.finish(&accept, &peer_id).expect("client session"); + (client, Mutex::new(server)) + } + + /// A browser request frame: its header names `content`'s length. + fn sealed_request(client: &mut PqSession, kind: &str, content: &[u8]) -> Vec { + let header = serde_json::json!({ + "version": BROWSER_PROTOCOL_VERSION, + "request_id": 1, + "type": kind, + "content_length": content.len(), + }); + let mut plaintext = serde_json::to_vec(&header).expect("header"); + plaintext.extend_from_slice(content); + sealed(client, &plaintext) + } + + fn sealed(client: &mut PqSession, plaintext: &[u8]) -> Vec { + encode_pq_frame(&client.seal(plaintext).expect("seal")).expect("frame") + } + + /// A well-formed frame of `payload`, sealed by no session: the frames that + /// never reach the session must not spend the browser's sequence numbers. + fn unsealed(payload: &[u8]) -> Vec { + encode_pq_frame(payload).expect("frame") + } + + fn chunk_message(body: ChunkMessageBody) -> Vec { + ChunkMessage { + request_id: 7, + body, + } + .encode() + .expect("chunk message") + } + + fn swarm_leave() -> Vec { + chunk_message(ChunkMessageBody::SwarmLeaveRequest( + ant_protocol::swarm::SwarmLeaveRequest { + room: [1; 32], + node: [2; 32], + membership: 1, + }, + )) + } + + /// The swarm tokens left in a bucket, as of its last charge or refund. + fn swarm_tokens(bucket: &Mutex) -> u128 { + bucket.lock().token_units / 1_000_000_000 + } + + #[tokio::test] + async fn a_frame_on_the_swarms_budget_gives_it_back_on_every_early_exit() { + let config = WebRtcDirectConfig { + max_requests_per_second: 1000, + max_requests_per_second_per_ip: 1, + max_requests_per_second_per_connection: 1, + ..WebRtcDirectConfig::default() + }; + let resources = ListenerResources::new(&config); + let address = "198.51.100.1:1000".parse().expect("browser address"); + let browser = resources + .try_admit_connection(address) + .expect("browser connection"); + let context = &browser.context; + let (mut client, server) = pq_session_pair(); + let (_activity, active_requests) = watch::channel(0_usize); + // The reads' budget is spent, so every frame below is taken on the + // swarm's (the reads' token comes back only a second later). + let spend_reads = || { + while context.take_request_tokens(Instant::now()) {} + }; + let read = |messages: Vec>| { + spend_reads(); + let channel = QueuedMessages::new(messages); + let server = &server; + let active_requests = &active_requests; + async move { read_webrtc_request(&channel, server, context, active_requests).await } + }; + let full_source = swarm_tokens(&context.source.swarm_rate); + let full_listener = swarm_tokens(&resources.global_swarm_rate); + assert_eq!(full_source, SWARM_REQUESTS_PER_SECOND_PER_SOURCE as u128); + assert_eq!(full_listener, SWARM_REQUESTS_PER_SECOND as u128); + let assert_restored = |exit: &str| { + assert_eq!( + swarm_tokens(&context.source.swarm_rate), + full_source, + "{exit}" + ); + assert_eq!( + swarm_tokens(&resources.global_swarm_rate), + full_listener, + "{exit}" + ); + }; + + // Twice the source's swarm budget of each, so a charge kept by any of + // them would leave the later ones refused as over the request rate. + for _ in 0..2 * SWARM_REQUESTS_PER_SECOND_PER_SOURCE { + // Bytes after the frame's declared end. + let mut frame = unsealed(&[0; 64]); + frame.push(0); + let error = read(vec![frame]).await.err().expect("trailing bytes"); + assert!(error.contains("after its declared payload"), "{error}"); + assert_restored("trailing bytes"); + + // The channel closes partway through a frame. + let frame = unsealed(&[0; 64]); + let (start, _) = frame.split_at(frame.len() / 2); + let error = read(vec![start.to_vec(), Vec::new()]) + .await + .err() + .expect("closed mid-frame"); + assert_eq!(error, "DataChannel closed"); + assert_restored("closed mid-frame"); + + // A frame that does not open under the session. + let frame = unsealed(&[0xa5; PQ_ENCRYPTED_OVERHEAD_BYTES + 16]); + let error = read(vec![frame]).await.err().expect("unopenable frame"); + assert!(error.starts_with("PQ session:"), "{error}"); + assert_restored("unopenable frame"); + + // A frame that opens to no request header. + let frame = sealed(&mut client, b"not a request header"); + let error = read(vec![frame]).await.err().expect("malformed header"); + assert_ne!(error, REQUEST_RATE_ERROR); + assert_restored("malformed header"); + + // A frame that is no swarm request. + let frame = sealed_request( + &mut client, + "chunk_protocol", + &chunk_message(ChunkMessageBody::GetRequest( + crate::ant_protocol::ChunkGetRequest::new([3; 32]), + )), + ); + let error = read(vec![frame]).await.err().expect("chunk read"); + assert_eq!(error, REQUEST_RATE_ERROR); + assert_restored("no swarm request"); + + // A read abandoned partway through a frame. + let frame = unsealed(&[0; 64]); + let (start, _) = frame.split_at(frame.len() / 2); + assert!(read(vec![start.to_vec()]).now_or_never().is_none()); + assert_restored("abandoned read"); + } + + // A swarm request is still admitted on the swarm's budget, and keeps + // its charge there. + let frame = sealed_request(&mut client, "chunk_protocol", &swarm_leave()); + let admitted = read(vec![frame]).await.expect("swarm request admitted"); + assert_eq!(admitted.charge, FrameCharge::Swarm); + assert_eq!(swarm_tokens(&context.source.swarm_rate), full_source - 1); + assert_eq!( + swarm_tokens(&resources.global_swarm_rate), + full_listener - 1 + ); + } + + #[tokio::test] + async fn a_frame_whose_buffers_are_refused_gives_the_swarm_its_charge_back() { + let config = WebRtcDirectConfig { + max_requests_per_second: 1000, + max_requests_per_second_per_ip: 1, + max_requests_per_second_per_connection: 1, + max_in_flight_bytes_per_ip: 64, + ..WebRtcDirectConfig::default() + }; + let resources = ListenerResources::new(&config); + let address = "198.51.100.1:1000".parse().expect("browser address"); + let browser = resources + .try_admit_connection(address) + .expect("browser connection"); + let context = &browser.context; + let (_client, server) = pq_session_pair(); + let (_activity, active_requests) = watch::channel(0_usize); + let full_source = swarm_tokens(&context.source.swarm_rate); + let full_listener = swarm_tokens(&resources.global_swarm_rate); + for _ in 0..2 * SWARM_REQUESTS_PER_SECOND_PER_SOURCE { + while context.take_request_tokens(Instant::now()) {} + // A frame larger than the source's byte budget. + let frame = unsealed(&[0; 256]); + let channel = QueuedMessages::new(vec![frame]); + let error = read_webrtc_request(&channel, &server, context, &active_requests) + .await + .err() + .expect("over the byte budget"); + assert_eq!(error, SOURCE_BYTE_CAPACITY_ERROR); + assert_eq!(swarm_tokens(&context.source.swarm_rate), full_source); + assert_eq!(swarm_tokens(&resources.global_swarm_rate), full_listener); + } + } + #[test] fn swarm_requests_past_their_budget_are_refused_and_keep_their_charge() { let config = WebRtcDirectConfig { From 546028bab6f2f116ea10498c7151ca8b0b2b734b Mon Sep 17 00:00:00 2001 From: grumbach Date: Thu, 5 Nov 2026 13:55:04 +0900 Subject: [PATCH 13/15] docs(adr): a client's downloads of one file share their record order A browser download asked for a file's records in an order picked at random for that download. Two downloads of one file in the same client then asked in different orders, so they no longer overlapped record by record and shared fewer fetches, and main's test that a joined fetch charges both downloads its GETs could fail. The client now picks one order per client (ant-client #244): readers in different clients still ask in different orders, which is what the order is for, while one client's downloads of a file keep sharing each record's fetch. The ADR says so. --- ...ADR-0026-clients-share-records-nodes-introduce-them.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md index 9dbb40e3..786fd416 100644 --- a/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md +++ b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md @@ -507,11 +507,13 @@ saorsa-transport's WebRTC Direct already uses). the read in all, from its first ask; a peer that stalls or runs past its record's time is passed over for 30 s (the read's own time running out does not count against it). A - browser download asks for its records in an order of its own, picked at - random, as BitTorrent picks pieces: a download keeps up to 8 records in + browser client asks for a file's records in an order of its own, picked at + random once, as BitTorrent picks pieces: a download keeps up to 8 records in flight (4 at first outside Chromium), and readers of one file who start together and ask in the same order would hold the same records at every moment, and want the same next - ones, so none would have a record another is still waiting for. A + ones, so none would have a record another is still waiting for. Two + downloads of a file in one client ask in the same order, so they still + share each record's fetch. A download delivers its chunks in any order, so its order costs it nothing; a stream or a range read asks in the order its reader reads. A client asks one link for one record at a time: in Chromium, two records sent at once on one data channel, a few pieces of From 48e2761f58b1a52ce8cb2a27fa627b408b2f34a8 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 6 Nov 2026 13:36:34 +0900 Subject: [PATCH 14/15] fix(swarm): keep only local addresses for a member on this node's network `relayable_sdp` kept every candidate that ant-protocol called non-global for a member this node reaches at a non-global address. As the classifier called some globally reachable addresses non-global (`2001:4:112::/48`, NAT64's `64:ff9b::/96`), a member reached at a local address could have this node relay a public destination such as `2001:4:112::1`, or 8.8.8.8 as `64:ff9b::808:808`, to other readers. Lock ant-protocol at 4bf8dc9, which classes addresses as the IANA registries do, drops NAT64 candidates and adds `is_local_ip`, and use that allowlist on both sides of the local branch: a member counts as local only at a private, shared, link-local, loopback or unique local address, and then passes on only such candidates and mDNS names besides its own public host. An address neither public nor local (documentation, benchmarking, reserved, Teredo) goes to no one. IPv6 loopback stays itself as a host, since its /64 (`::`) is not local. ADR-0026's candidate rule now says the same. New tests: a remote announce from a member on the node's own network passes on its local candidates, mDNS name and own address, and drops `64:ff9b::808:808`, `2001:4:112::1`, `192.0.0.9`, `2001:db8::1`, `198.18.0.1`, `240.0.0.1`, `64:ff9b:1::808:808` and another host's public address; a member at an address neither public nor local keeps nothing local; one over loopback in either family, link-local or unique local keeps its own. --- Cargo.lock | 2 +- ...ents-share-records-nodes-introduce-them.md | 22 ++-- src/swarm.rs | 103 ++++++++++++++++-- 3 files changed, 108 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 68558f7d..3125ae32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -906,7 +906,7 @@ dependencies = [ [[package]] name = "ant-protocol" version = "6.0.0" -source = "git+https://github.com/grumbach/ant-protocol?branch=feat%2Fpeer-swarm#61e3dbf50c3b851e9cd056e01ecc0611a2c31e4e" +source = "git+https://github.com/grumbach/ant-protocol?branch=feat%2Fpeer-swarm#4bf8dc937bd3cb0049ae45fb01491b1c99d5bc0f" dependencies = [ "blake3", "bytes", diff --git a/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md index 786fd416..0f560976 100644 --- a/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md +++ b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md @@ -159,14 +159,20 @@ bytes). sends ICE connectivity checks to every candidate in it before anything proves who made it. Both sides therefore pass each SDP through the protocol's `constrain_swarm_sdp`: UDP candidates only, naming an IP address that is not -unspecified, multicast or broadcast, or an mDNS `.local` name, at most 16. A -node relaying an SDP also drops every candidate whose address is public and -not the sender's own host, as the node reaches it on that request (an IPv4 -address, or an IPv6 /64). Local addresses and mDNS names stay only where the -node and the client reach each other at local addresses (a network of their -own, such as a test network or a LAN deployment): the node keeps them only -for a sender it reaches at a local address, and a client applies them only -from a node it reached at one, loopback ones only over loopback. On a network +unspecified, multicast, broadcast or in a NAT64 prefix (`64:ff9b::/96`, +`64:ff9b:1::/48`, which stand for IPv4 addresses a translator on the +receiver's network sends to), or an mDNS `.local` name, at most 16. A node +relaying an SDP also drops every candidate whose address is public (globally +reachable, as the IANA special-purpose address registries mark it: +`is_global_ip`) and not the sender's own host, as the node reaches it on that +request (an IPv4 address, or an IPv6 /64). Local addresses (private use, +shared, link-local, loopback and unique local: `is_local_ip`) and mDNS names +stay only where the node and the client reach each other at local addresses +(a network of their own, such as a test network or a LAN deployment): the +node keeps them only for a sender it reaches at a local address, and a client +applies them only from a node it reached at one, loopback ones only over +loopback. An address that is neither public nor local (documentation, +benchmarking, reserved, Teredo) stays on no network. On a network reached at public addresses, readers meet by their public (server-reflexive) addresses alone, so two readers behind one NAT need it to hairpin. A stranger in a room therefore cannot point other readers' checks at a third party, nor diff --git a/src/swarm.rs b/src/swarm.rs index b7ad8f72..ca19ecc2 100644 --- a/src/swarm.rs +++ b/src/swarm.rs @@ -30,7 +30,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use ant_protocol::chunk::{ChunkMessageBody, ProtocolError}; use ant_protocol::swarm::{ - constrain_swarm_sdp, is_global_ip, MemberId, Membership, NodeId, OfferId, RoomKey, + constrain_swarm_sdp, is_global_ip, is_local_ip, MemberId, Membership, NodeId, OfferId, RoomKey, SwarmAnnounceRequest, SwarmAnnounceResponse, SwarmAnswerRequest, SwarmAnswerResponse, SwarmLeaveRequest, SwarmLeaveResponse, SwarmRefusal, SwarmSignal, MAX_SWARM_ICE_SERVERS, MAX_SWARM_ICE_SERVER_BYTES, MAX_SWARM_INBOX, MAX_SWARM_REPLY_SDP_BYTES, SWARM_SIGNAL_TTL_MS, @@ -533,24 +533,29 @@ pub fn is_swarm_request_bytes(data: &[u8]) -> bool { /// An SDP from a member, as this node passes it on: its candidates may name /// the member's own host, as this node reaches it, and no other public -/// address, at most `MAX_SWARM_CANDIDATES` of them. Local addresses and mDNS -/// names stay only for a member this node reaches at a local address itself -/// (a network of its own, such as a test network): elsewhere they would let a -/// stranger point readers at their own home networks. The member an SDP goes -/// to sends connectivity checks to each candidate before anything proves who -/// made it. +/// address, at most `MAX_SWARM_CANDIDATES` of them. Local addresses +/// ([`is_local_ip`]: private use, shared, link-local, loopback, unique local) +/// and mDNS names stay only for a member this node reaches at a local address +/// itself (a network of its own, such as a test network): elsewhere they +/// would let a stranger point readers at their own home networks. Any other +/// address, neither public nor local (documentation, benchmarking, reserved, +/// Teredo), stays for no member. The member an SDP goes to sends +/// connectivity checks to each candidate before anything proves who made it. fn relayable_sdp(sdp: &str, host: Option) -> String { - let local = host.is_some_and(|host| !is_global_ip(host)); + let local = host.is_some_and(is_local_ip); constrain_swarm_sdp(sdp, |ip| match ip { Some(ip) if is_global_ip(ip) => Some(host_of(ip)) == host, - _ => local, + Some(ip) => local && is_local_ip(ip), + None => local, }) } /// The host `ip` counts as: an IPv4 address, or an IPv6 /64, which one -/// client is usually given whole. +/// client is usually given whole. IPv6 loopback stays itself: its /64 (`::`) +/// would not read as local. fn host_of(ip: IpAddr) -> IpAddr { match ip.to_canonical() { + IpAddr::V6(v6) if v6.is_loopback() => IpAddr::V6(v6), IpAddr::V6(v6) => { let mut octets = v6.octets(); if let Some(interface) = octets.get_mut(8..) { @@ -2080,6 +2085,84 @@ mod tests { ); // A member whose address this node cannot tell keeps nothing local. assert!(!relayable_sdp(&sdp, None).contains("192.168.1.4")); + // Nor one at an address neither public nor local. + for host in [ + "192.0.2.7", + "198.18.0.1", + "2001:db8::7", + "64:ff9b:1::a0a:302", + ] { + let kept = relayable_sdp(&sdp, Some(host_of(host.parse().expect("ip")))); + assert!( + !kept.contains("192.168.1.4") && !kept.contains(".local"), + "{host}: {kept}" + ); + } + // A member over loopback, in either family, is on this node's own + // network too. + for host in ["127.0.0.1", "::1", "::ffff:127.0.0.1", "fe80::1", "fd12::4"] { + let kept = relayable_sdp(&sdp, Some(host_of(host.parse().expect("ip")))); + assert!( + kept.contains("192.168.1.4") && kept.contains("1f2e3d4c.local"), + "{host}: {kept}" + ); + } + } + + #[tokio::test] + async fn a_member_on_this_nodes_own_network_passes_on_only_local_and_its_own_candidates() { + let service = service(); + from(&service, B, "10.78.3.3", announce(1, &[], 0)).await; + let mut offer = announce(1, &[32], 0); + if let ChunkMessageBody::SwarmAnnounceRequest(request) = &mut offer { + for sent in &mut request.offers { + sent.sdp = String::from( + "v=0\r\n\ + a=candidate:1 1 udp 1 10.78.3.2 5000 typ host\r\n\ + a=candidate:2 1 udp 1 192.168.1.4 5001 typ host\r\n\ + a=candidate:3 1 udp 1 1f2e3d4c.local 5002 typ host\r\n\ + a=candidate:4 1 udp 1 fd12::4 5003 typ host\r\n\ + a=candidate:5 1 udp 1 64:ff9b::808:808 5004 typ host\r\n\ + a=candidate:6 1 udp 1 2001:4:112::1 5005 typ host\r\n\ + a=candidate:7 1 udp 1 192.0.0.9 5006 typ host\r\n\ + a=candidate:8 1 udp 1 2001:db8::1 5007 typ host\r\n\ + a=candidate:9 1 udp 1 198.18.0.1 5008 typ host\r\n\ + a=candidate:10 1 udp 1 240.0.0.1 5009 typ host\r\n\ + a=candidate:11 1 udp 1 64:ff9b:1::808:808 5010 typ host\r\n\ + a=candidate:12 1 udp 1 9.9.9.99 5011 typ srflx raddr 0.0.0.0 rport 0\r\n", + ); + } + } + from(&service, A, "10.78.3.2", offer).await.expect("joined"); + let (_, inbox) = from(&service, B, "10.78.3.3", announce(1, &[], 0)) + .await + .expect("joined"); + let [SwarmSignal::Offer { sdp, .. }] = inbox.as_slice() else { + panic!("expected one offer: {inbox:?}"); + }; + // Its local candidates, mDNS name and own address go on... + for kept in [ + "10.78.3.2 5000", + "192.168.1.4 5001", + "1f2e3d4c.local 5002", + "fd12::4 5003", + ] { + assert!(sdp.contains(kept), "{kept}: {sdp}"); + } + // ...and nothing else: no public address of another host, nothing + // through NAT64, nothing neither public nor local. + for dropped in [ + "64:ff9b::808:808", + "2001:4:112::1", + "192.0.0.9", + "2001:db8::1", + "198.18.0.1", + "240.0.0.1", + "64:ff9b:1::808:808", + "9.9.9.99", + ] { + assert!(!sdp.contains(dropped), "{dropped}: {sdp}"); + } } #[tokio::test] From ad27073f5d8a6170fb641403320c41fbde186c96 Mon Sep 17 00:00:00 2001 From: grumbach Date: Fri, 6 Nov 2026 16:22:09 +0900 Subject: [PATCH 15/15] fix(swarm): a member reached through shared CGNAT space keeps nothing local ant-protocol's `is_local_ip` counted shared address space (`100.64.0.0/10`, RFC 6598) as local, so this node took a member it reaches at `100.64.0.1` for one on its own network and relayed that member's private, unique local, link-local and mDNS candidates to other readers. A provider numbers the subscribers behind its carrier-grade NAT in that space; they share no network. Lock ant-protocol at f79e2b2, where shared space is neither local nor global: such a member now passes on no local candidate and no shared address, only a public one that is its own host. The relay rule's comment and ADR-0026's candidate paragraph say so. New tests: a remote announce from a member at `100.64.0.1` reaches another member with none of its `192.168.1.4`, `fd12::4`, `fe80::1`, mDNS or shared candidates nor a stranger's public one; hosts at either end of shared space and IPv4-mapped keep nothing local; the hosts just outside it keep only their own public address. --- Cargo.lock | 2 +- ...ents-share-records-nodes-introduce-them.md | 28 +++---- src/swarm.rs | 73 ++++++++++++++++++- 3 files changed, 85 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3125ae32..b3e4f749 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -906,7 +906,7 @@ dependencies = [ [[package]] name = "ant-protocol" version = "6.0.0" -source = "git+https://github.com/grumbach/ant-protocol?branch=feat%2Fpeer-swarm#4bf8dc937bd3cb0049ae45fb01491b1c99d5bc0f" +source = "git+https://github.com/grumbach/ant-protocol?branch=feat%2Fpeer-swarm#f79e2b271ab2a0ae2231d5d4ee5c127210c6c150" dependencies = [ "blake3", "bytes", diff --git a/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md index 0f560976..ac797f37 100644 --- a/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md +++ b/docs/adr/ADR-0026-clients-share-records-nodes-introduce-them.md @@ -166,20 +166,22 @@ relaying an SDP also drops every candidate whose address is public (globally reachable, as the IANA special-purpose address registries mark it: `is_global_ip`) and not the sender's own host, as the node reaches it on that request (an IPv4 address, or an IPv6 /64). Local addresses (private use, -shared, link-local, loopback and unique local: `is_local_ip`) and mDNS names -stay only where the node and the client reach each other at local addresses -(a network of their own, such as a test network or a LAN deployment): the -node keeps them only for a sender it reaches at a local address, and a client +link-local, loopback and unique local: `is_local_ip`) and mDNS names stay +only where the node and the client reach each other at local addresses (a +network of their own, such as a test network or a LAN deployment): the node +keeps them only for a sender it reaches at a local address, and a client applies them only from a node it reached at one, loopback ones only over -loopback. An address that is neither public nor local (documentation, -benchmarking, reserved, Teredo) stays on no network. On a network -reached at public addresses, readers meet by their public (server-reflexive) -addresses alone, so two readers behind one NAT need it to hairpin. A stranger -in a room therefore cannot point other readers' checks at a third party, nor -at the readers' own home networks, and a hostile node can point them at 16 -addresses at most per SDP. Receivers read SDP leniently (webrtc-rs trims an -attribute's value and skips a carriage return before a line, Chrome takes an -attribute name followed by a space, Firefox names attributes in any case), so +loopback. An address that is neither public nor local (shared carrier-grade +NAT space, whose subscribers share no network, documentation, benchmarking, +reserved, Teredo) stays on no network, and a sender or node reached at one +gives no local candidates. On a network reached at public addresses, readers +meet by their public (server-reflexive) addresses alone, so two readers +behind one NAT need it to hairpin. A stranger in a room therefore cannot +point other readers' checks at a third party, nor at the readers' own home +networks, and a hostile node can point them at 16 addresses at most per SDP. +Receivers read SDP leniently (webrtc-rs trims an attribute's value and skips +a carriage return before a line, Chrome takes an attribute name followed by a +space, Firefox names attributes in any case), so a line any of them may read as a candidate or a DTLS fingerprint stays only in its one standard form (`a=candidate:` or `a=fingerprint:`, then a value with no whitespace around it), and a line with a carriage return anywhere but diff --git a/src/swarm.rs b/src/swarm.rs index ca19ecc2..77f82cb1 100644 --- a/src/swarm.rs +++ b/src/swarm.rs @@ -534,12 +534,14 @@ pub fn is_swarm_request_bytes(data: &[u8]) -> bool { /// An SDP from a member, as this node passes it on: its candidates may name /// the member's own host, as this node reaches it, and no other public /// address, at most `MAX_SWARM_CANDIDATES` of them. Local addresses -/// ([`is_local_ip`]: private use, shared, link-local, loopback, unique local) -/// and mDNS names stay only for a member this node reaches at a local address +/// ([`is_local_ip`]: private use, link-local, loopback, unique local) and +/// mDNS names stay only for a member this node reaches at a local address /// itself (a network of its own, such as a test network): elsewhere they /// would let a stranger point readers at their own home networks. Any other -/// address, neither public nor local (documentation, benchmarking, reserved, -/// Teredo), stays for no member. The member an SDP goes to sends +/// address, neither public nor local (shared carrier-grade NAT space, whose +/// subscribers share no network, documentation, benchmarking, reserved, +/// Teredo), stays for no member, and a member reached at one keeps nothing +/// local. The member an SDP goes to sends /// connectivity checks to each candidate before anything proves who made it. fn relayable_sdp(sdp: &str, host: Option) -> String { let local = host.is_some_and(is_local_ip); @@ -2098,6 +2100,35 @@ mod tests { "{host}: {kept}" ); } + // Nor one reached through shared (carrier-grade NAT) space, at either + // end of it or IPv4-mapped: its subscribers share no network. + for host in [ + "100.64.0.0", + "100.64.0.1", + "100.127.255.255", + "::ffff:100.64.0.1", + ] { + let kept = relayable_sdp(&sdp, Some(host_of(host.parse().expect("ip")))); + assert!( + !kept.contains("192.168.1.4") && !kept.contains(".local"), + "{host}: {kept}" + ); + } + // Just outside it, a member is public: its own address only. + for host in ["100.63.255.255", "100.128.0.0"] { + let theirs = format!( + "{}a=candidate:4 1 udp 1 1f2e3d4c.local 5003 typ host\r\n", + pointing(host) + ); + let kept = relayable_sdp(&theirs, Some(host_of(host.parse().expect("ip")))); + assert!(kept.contains(&format!("{host} 5000")), "{host}: {kept}"); + assert!( + !kept.contains("192.168.1.4") + && !kept.contains(".local") + && !kept.contains("9.9.9.99"), + "{host}: {kept}" + ); + } // A member over loopback, in either family, is on this node's own // network too. for host in ["127.0.0.1", "::1", "::ffff:127.0.0.1", "fe80::1", "fd12::4"] { @@ -2109,6 +2140,40 @@ mod tests { } } + #[tokio::test] + async fn a_member_reached_through_shared_cgnat_space_passes_on_nothing_local() { + let service = service(); + from(&service, B, "100.64.0.2", announce(1, &[], 0)).await; + let mut offer = announce(1, &[33], 0); + if let ChunkMessageBody::SwarmAnnounceRequest(request) = &mut offer { + for sent in &mut request.offers { + sent.sdp = String::from( + "v=0\r\n\ + a=candidate:1 1 udp 1 100.64.0.1 5000 typ host\r\n\ + a=candidate:2 1 udp 1 192.168.1.4 5001 typ host\r\n\ + a=candidate:3 1 udp 1 fd12::4 5002 typ host\r\n\ + a=candidate:4 1 udp 1 fe80::1 5003 typ host\r\n\ + a=candidate:5 1 udp 1 1f2e3d4c.local 5004 typ host\r\n\ + a=candidate:6 1 udp 1 9.9.9.99 5005 typ srflx raddr 0.0.0.0 rport 0\r\n", + ); + } + } + // Two subscribers behind one carrier-grade NAT share no network: + // none of the sender's private, unique local, link-local or mDNS + // candidates, nor its shared address, nor a stranger's public one, + // goes on. + from(&service, A, "100.64.0.1", offer) + .await + .expect("joined"); + let (_, inbox) = from(&service, B, "100.64.0.2", announce(1, &[], 0)) + .await + .expect("joined"); + let [SwarmSignal::Offer { sdp, .. }] = inbox.as_slice() else { + panic!("expected one offer: {inbox:?}"); + }; + assert_eq!(sdp, "v=0\r\n"); + } + #[tokio::test] async fn a_member_on_this_nodes_own_network_passes_on_only_local_and_its_own_candidates() { let service = service();