diff --git a/crates/sandlock-core/src/network.rs b/crates/sandlock-core/src/network.rs index dca5f8c..8d06dfd 100644 --- a/crates/sandlock-core/src/network.rs +++ b/crates/sandlock-core/src/network.rs @@ -20,6 +20,13 @@ use crate::sys::structs::{SeccompNotif, AF_INET, AF_INET6, ECONNREFUSED}; /// Prevents a sandboxed process from triggering OOM in the supervisor. const MAX_SEND_BUF: usize = 64 << 20; +/// Maximum ancillary (control) buffer we copy for an on-behalf `sendmsg`. +/// A control buffer larger than this fails closed with `EMSGSIZE` rather than +/// being silently truncated into a partial cmsg chain (`SCM_MAX_FD` is 253 fds +/// ≈ 1 KiB, so 16 KiB is far above any legitimate use while bounding supervisor +/// memory per trapped send). +const MAX_CONTROL_BUF: usize = 16 << 10; + /// An IPv4 or IPv6 address with a prefix length, used by `--net-deny` /// to match destination IPs by exact address (`/32`, `/128`) or by range. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -459,6 +466,124 @@ pub(crate) fn query_socket_protocol(fd: RawFd) -> Option { } } +/// Rewrite the `SCM_RIGHTS` file descriptors in a copied control buffer from +/// the child's fd numbers to supervisor fd numbers, rejecting identity cmsgs. +/// +/// On-behalf sends run in the supervisor, so a control buffer copied verbatim +/// from the child carries fd numbers that are meaningless (or, worse, alias +/// unrelated files) in the supervisor. For every `SOL_SOCKET`/`SCM_RIGHTS` +/// message we `pidfd_getfd` each child fd into the supervisor and patch its +/// number in place. The returned `OwnedFd`s must stay alive until after +/// `sendmsg` (the kernel installs its own copies into the socket buffer during +/// the send), then drop to close the supervisor's copies. +/// +/// `SCM_CREDENTIALS` is rejected with `EPERM`: on the on-behalf path the +/// *supervisor* is the syscall's sender, so forwarding the child's crafted +/// `pid/uid/gid` would either fail `EPERM` anyway (an unprivileged supervisor +/// can't assert them) or, for a privileged supervisor, let the child forge +/// credentials it could never send itself. Failing closed is the safe choice. +/// +/// Errors: `EBADF` if a child fd can't be fetched (matching the kernel's own +/// error for a bad fd), `EPERM` for a credential cmsg, `EINVAL` for a malformed +/// cmsg header (too short or extending past the buffer) — all fail closed, none +/// sends a partial or forged control chain. +fn translate_scm_rights(child_pid: u32, control: &[u8]) -> Result<(Vec, Vec), i32> { + // sizeof(struct cmsghdr) on LP64: cmsg_len(8) + cmsg_level(4) + cmsg_type(4). + // CMSG_ALIGN(16) == 16, so cmsg data begins right after the header. + const CMSG_HDR: usize = 16; + const FD: usize = std::mem::size_of::(); + let mut out = control.to_vec(); + let mut held: Vec = Vec::new(); + let mut off = 0usize; + while off + CMSG_HDR <= out.len() { + let cmsg_len = usize::from_ne_bytes(out[off..off + 8].try_into().unwrap()); + let level = i32::from_ne_bytes(out[off + 8..off + 12].try_into().unwrap()); + let ctype = i32::from_ne_bytes(out[off + 12..off + 16].try_into().unwrap()); + // `cmsg_len` is child-controlled. Compare against the *remaining* space + // (never `off + cmsg_len`, which could overflow `usize`). A header that + // is too short or claims to run past the buffer is malformed — fail + // closed, as the kernel would `EINVAL` it. `off <= out.len() - CMSG_HDR` + // holds from the loop guard, so `out.len() - off` cannot underflow. + if cmsg_len < CMSG_HDR || cmsg_len > out.len() - off { + return Err(libc::EINVAL); + } + if level == libc::SOL_SOCKET { + if ctype == libc::SCM_RIGHTS { + let data_off = off + CMSG_HDR; + let nfds = (cmsg_len - CMSG_HDR) / FD; + for i in 0..nfds { + let p = data_off + i * FD; + let child_fd = i32::from_ne_bytes(out[p..p + FD].try_into().unwrap()); + let sup_fd = crate::seccomp::notif::dup_fd_from_pid(child_pid, child_fd) + .map_err(|e| e.raw_os_error().unwrap_or(libc::EBADF))?; + out[p..p + FD].copy_from_slice(&sup_fd.as_raw_fd().to_ne_bytes()); + held.push(sup_fd); + } + } else if ctype == libc::SCM_CREDENTIALS { + return Err(libc::EPERM); + } + } + // Advance to CMSG_ALIGN(cmsg_len). `cmsg_len <= out.len() - off` bounds + // the aligned add well under `usize::MAX`; each step is >= CMSG_HDR so + // the loop always makes progress. + off += (cmsg_len + 7) & !7; + } + Ok((out, held)) +} + +/// True iff `fd` is an `AF_UNIX` socket, probed via `SO_DOMAIN`. `SCM_RIGHTS` +/// and `SCM_CREDENTIALS` are unix-only, so control rewriting/gating is applied +/// only to unix sockets — an IP socket's control (e.g. `IP_PKTINFO`) carries no +/// fds or credentials and passes through untouched. +fn socket_is_unix(fd: RawFd) -> bool { + let mut domain: libc::c_int = 0; + let mut len = std::mem::size_of::() as libc::socklen_t; + let rc = unsafe { + libc::getsockopt( + fd, + libc::SOL_SOCKET, + libc::SO_DOMAIN, + &mut domain as *mut _ as *mut libc::c_void, + &mut len, + ) + }; + rc == 0 && domain == libc::AF_UNIX +} + +/// Copy (and, for a unix socket, translate) the control buffer of an on-behalf +/// send. Shared by `send_msghdr_on_behalf` and `send_named_unix_msghdr`. +/// +/// Returns `(control_bytes, held_fds)` — the fds keep the translated +/// `SCM_RIGHTS` files open across the send. Fails closed: oversized control → +/// `EMSGSIZE`; unreadable control → `EIO` (never a silent send that drops the +/// caller's cmsgs). For a unix socket, `SCM_RIGHTS` fds are translated and +/// `SCM_CREDENTIALS` is rejected; a non-unix socket's control passes through +/// verbatim. +fn materialize_control( + notif: &SeccompNotif, + notif_fd: RawFd, + msg_control_ptr: u64, + msg_controllen: u64, + is_unix: bool, +) -> Result<(Option>, Vec), i32> { + if msg_control_ptr == 0 || msg_controllen == 0 { + return Ok((None, Vec::new())); + } + // Fail closed on an oversized control buffer instead of silently truncating + // (which could drop SCM_RIGHTS fds or send a malformed tail). + if msg_controllen as usize > MAX_CONTROL_BUF { + return Err(libc::EMSGSIZE); + } + let raw = read_child_mem(notif_fd, notif.id, notif.pid, msg_control_ptr, msg_controllen as usize) + .map_err(|_| libc::EIO)?; + if is_unix { + let (buf, fds) = translate_scm_rights(notif.pid, &raw)?; + Ok((Some(buf), fds)) + } else { + Ok((Some(raw), Vec::new())) + } +} + // ============================================================ // connect_on_behalf — perform connect() on behalf of the child (TOCTOU-safe) // ============================================================ @@ -957,25 +1082,29 @@ fn sendmsg_named_unix_on_behalf( sun_path: &std::path::Path, writable: &[std::path::PathBuf], ) -> NotifAction { - match send_named_unix_msghdr(notif, notif_fd, sockfd, msghdr_ptr, flags, sun_path, writable) { - Ok(n) => NotifAction::ReturnValue(n as i64), + match send_named_unix_msghdr(notif, notif_fd, sockfd, msghdr_ptr, sun_path, writable) { + Ok((dup_fd, m)) => { + let blocking = wants_blocking(dup_fd.as_raw_fd(), flags); + resolve_send(dup_fd, m, flags, blocking) + } Err(errno) => NotifAction::Errno(errno), } } -/// Core of the named-unix on-behalf `sendmsg`: resolve+verify `sun_path`, copy -/// the message's iovecs/control from the child, and send to the pinned inode -/// via `/proc/self/fd`. Returns the bytes sent or an errno. Shared by the -/// single-message `sendmsg` path and the per-entry `sendmmsg` path. +/// Core of the named-unix on-behalf `sendmsg`: resolve+verify `sun_path` and +/// copy the message's iovecs/control from the child, addressed to the pinned +/// inode via `/proc/self/fd`. Returns the dup'd socket and a [`MaterializedMsg`] +/// (which keeps the inode pin alive) for the caller to send — inline, and +/// deferred if it would block. Shared by the single-message `sendmsg` path and +/// the per-entry `sendmmsg` path. fn send_named_unix_msghdr( notif: &SeccompNotif, notif_fd: RawFd, sockfd: i32, msghdr_ptr: u64, - flags: i32, sun_path: &std::path::Path, writable: &[std::path::PathBuf], -) -> Result { +) -> Result<(OwnedFd, MaterializedMsg), i32> { let pinned = match resolve_named_unix_target(notif.pid, sun_path, writable) { Ok(fd) => fd, Err(NotifAction::Errno(e)) => return Err(e), @@ -995,56 +1124,32 @@ fn send_named_unix_msghdr( let iovlen = (msg_iovlen as usize).min(1024); let iov_bytes = read_child_mem(notif_fd, notif.id, notif.pid, msg_iov_ptr, iovlen * 16) .map_err(|_| libc::EIO)?; - let mut data_bufs: Vec> = Vec::with_capacity(iovlen); - for i in 0..iovlen { - let off = i * 16; - if off + 16 > iov_bytes.len() { - break; - } - let base = u64::from_ne_bytes(iov_bytes[off..off + 8].try_into().unwrap()); - let len = u64::from_ne_bytes(iov_bytes[off + 8..off + 16].try_into().unwrap()) as usize; - if len > MAX_SEND_BUF { - return Err(libc::EMSGSIZE); - } - if base == 0 || len == 0 { - data_bufs.push(Vec::new()); - continue; - } - data_bufs.push(read_child_mem(notif_fd, notif.id, notif.pid, base, len).map_err(|_| libc::EIO)?); - } - let mut local_iovs: Vec = data_bufs - .iter() - .map(|b| libc::iovec { - iov_base: b.as_ptr() as *mut libc::c_void, - iov_len: b.len(), - }) - .collect(); - - let control_buf = if msg_control_ptr != 0 && msg_controllen > 0 { - let len = (msg_controllen as usize).min(4096); - read_child_mem(notif_fd, notif.id, notif.pid, msg_control_ptr, len).ok() - } else { - None - }; + let data = flatten_iovecs(notif, notif_fd, &iov_bytes, iovlen)?; + // Named target is always AF_UNIX, so translate SCM_RIGHTS / reject creds. + let (control_buf, scm_fds) = + materialize_control(notif, notif_fd, msg_control_ptr, msg_controllen, true)?; let dup_fd = crate::seccomp::notif::dup_fd_from_pid(notif.pid, sockfd) .map_err(|e| e.raw_os_error().unwrap_or(libc::EBADF))?; - let mut msg: libc::msghdr = unsafe { std::mem::zeroed() }; - msg.msg_name = &sun as *const libc::sockaddr_un as *mut libc::c_void; - msg.msg_namelen = sun_len; - msg.msg_iov = local_iovs.as_mut_ptr(); - msg.msg_iovlen = local_iovs.len(); - if let Some(ref ctrl) = control_buf { - msg.msg_control = ctrl.as_ptr() as *mut libc::c_void; - msg.msg_controllen = ctrl.len(); - } - let ret = unsafe { libc::sendmsg(dup_fd.as_raw_fd(), &msg, flags) }; - if ret >= 0 { - Ok(ret) - } else { - Err(unsafe { *libc::__errno_location() }) - } + // The destination is the `/proc/self/fd/` sockaddr; `pinned` must + // stay open (and at the same fd number) for that path to resolve, so the + // message keeps it alive. Copy the sockaddr bytes it currently encodes. + let addr = unsafe { + std::slice::from_raw_parts(&sun as *const libc::sockaddr_un as *const u8, sun_len as usize) + } + .to_vec(); + + Ok(( + dup_fd, + MaterializedMsg { + data, + control: control_buf, + addr, + _scm_fds: scm_fds, + _pinned: Some(pinned), + }, + )) } /// Read a `sendmmsg` entry's `msg_name` and return its NAMED `AF_UNIX` path, or @@ -1094,22 +1199,52 @@ fn sendmmsg_named_unix_on_behalf( // stop here and report a short send rather than passing it through. None => break, }; - match send_named_unix_msghdr(notif, notif_fd, sockfd, entry_ptr, flags, &path, writable) { - Ok(n) => { - let bytes = (n as u32).to_ne_bytes(); - let _ = write_child_mem( - notif_fd, - notif.id, - notif.pid, - entry_ptr + MSG_LEN_OFFSET as u64, - &bytes, - ); - sent += 1; - } + let (dup_fd, m) = match send_named_unix_msghdr(notif, notif_fd, sockfd, entry_ptr, &path, writable) { + Ok(pair) => pair, Err(errno) => { first_errno = Some(errno); break; } + }; + // Non-blocking per entry so a batch never occupies the notification loop; + // the two cases that must leave the loop are deferred (see below). + let blocking = wants_blocking(dup_fd.as_raw_fd(), flags); + let ret = send_materialized_at(dup_fd.as_raw_fd(), &m, 0, flags | libc::MSG_DONTWAIT); + if ret >= 0 { + if blocking && (ret as usize) < m.data.len() { + // Partial stream on a blocking socket: complete this entry off the + // loop and report it, so a caller ignoring per-entry msg_len isn't + // silently truncated. + return complete_batch_entry( + dup_fd, m, flags, ret as usize, notif_fd, notif.id, notif.pid, + entry_ptr + MSG_LEN_OFFSET as u64, sent, + ); + } + let bytes = (ret as u32).to_ne_bytes(); + let _ = write_child_mem( + notif_fd, notif.id, notif.pid, entry_ptr + MSG_LEN_OFFSET as u64, &bytes, + ); + sent += 1; + } else { + let err = unsafe { *libc::__errno_location() }; + if err == libc::EAGAIN || err == libc::EWOULDBLOCK { + if sent == 0 && blocking { + // Entry 0 would block entirely: a blocking socket never returns + // EAGAIN, so complete it off the loop. + return complete_batch_entry( + dup_fd, m, flags, 0, notif_fd, notif.id, notif.pid, + entry_ptr + MSG_LEN_OFFSET as u64, 0, + ); + } + // i>0 with nothing sent is a contract-legal short count; a + // non-blocking child at entry 0 gets EAGAIN. + if sent == 0 { + first_errno = Some(libc::EAGAIN); + } + break; + } + first_errno = Some(err); + break; } } if sent > 0 { @@ -1231,27 +1366,19 @@ async fn sendto_on_behalf( Err(_) => return NotifAction::Errno(libc::EIO), }; - // 4. (dup_fd from step 2 is reused for the supervisor sendto.) - - // 5. Perform sendto in supervisor with validated sockaddr + copied data - let ret = unsafe { - libc::sendto( - dup_fd.as_raw_fd(), - data.as_ptr() as *const libc::c_void, - data.len(), - flags, - addr_bytes.as_ptr() as *const libc::sockaddr, - addr_len as libc::socklen_t, - ) + // 4. Send on-behalf (deferred if it would block), like sendmsg — a + // sendto is a sendmsg with a single iovec and an explicit destination. + // The first attempt is non-blocking on the loop; a blocking child whose + // send buffer is full defers off the loop instead of wedging it. + let m = MaterializedMsg { + data, + control: None, + addr: addr_bytes, + _scm_fds: Vec::new(), + _pinned: None, }; - - // 6. Return result - if ret >= 0 { - NotifAction::ReturnValue(ret as i64) - } else { - let errno = unsafe { *libc::__errno_location() }; - NotifAction::Errno(errno) - } + let blocking = wants_blocking(dup_fd.as_raw_fd(), flags); + resolve_send(dup_fd, m, flags, blocking) } else { // Non-IP family. Gate a NAMED AF_UNIX datagram the same way as connect: // sendto to a named socket is a WRITE on its inode, so deny unless the @@ -1332,13 +1459,25 @@ async fn sendmsg_on_behalf( Ok(fd) => fd, Err(e) => return NotifAction::Errno(e.raw_os_error().unwrap_or(libc::EBADF)), }; - let protocol = match query_socket_protocol(dup_fd.as_raw_fd()) { - Some(p) => p, - None => return NotifAction::Errno(ECONNREFUSED), - }; - - match send_msghdr_on_behalf(notif, ctx, notif_fd, &dup_fd, protocol, msghdr_ptr, flags).await { - Ok(n) => NotifAction::ReturnValue(n as i64), + // Resolve the protocol as `Option`: it is only consumed to validate a + // non-connected IP destination. `query_socket_protocol` returns `None` for + // an AF_UNIX socket (no IP protocol), and a connected send (every AF_UNIX + // send that reaches here — its connection was gated at connect time) never + // consumes it, so the send goes through the TOCTOU-safe on-behalf path on + // our immune `dup_fd` rather than being refused. A non-connected send with + // no resolvable protocol fails closed inside `send_msghdr_on_behalf`. + // + // On-behalf (not Continue) is load-bearing under a destination policy: a + // Continue would let the kernel re-resolve `sockfd`/`msg_name` against the + // live child, so a racing `dup2(inet_sock, sockfd)` after a domain check + // could redirect the send onto an IP socket to a denied destination. + let protocol = query_socket_protocol(dup_fd.as_raw_fd()); + + match send_msghdr_on_behalf(notif, ctx, notif_fd, &dup_fd, protocol, msghdr_ptr).await { + Ok(m) => { + let blocking = wants_blocking(dup_fd.as_raw_fd(), flags); + resolve_send(dup_fd, m, flags, blocking) + } Err(errno) => NotifAction::Errno(errno), } } @@ -1391,27 +1530,237 @@ fn prescan_msghdr( PrescanResult::OnBehalf } +/// A fully-materialized on-behalf send. Owns the (flattened) iovec payload, the +/// translated control buffer (with its `SCM_RIGHTS` fds and any named-unix inode +/// pin kept alive), and the destination sockaddr (empty = connected). Owning +/// everything lets the send be retried — from a byte offset, on a deferred +/// worker — without borrowing supervisor state, so a blocked send can leave the +/// sequential notification loop while still delivering the whole message. +struct MaterializedMsg { + data: Vec, + control: Option>, + addr: Vec, + _scm_fds: Vec, + _pinned: Option, +} + +/// True iff this send should block until it completes: the socket is in blocking +/// mode (`O_NONBLOCK` clear — the dup shares the child's file description, so it +/// reflects the child's own mode) *and* the per-call `send_flags` did not request +/// non-blocking with `MSG_DONTWAIT`. A child that passes `MSG_DONTWAIT` on a +/// blocking socket wants the immediate short-count/`EAGAIN`, not a deferred +/// block-to-completion, so it must not be deferred. +fn wants_blocking(fd: RawFd, send_flags: i32) -> bool { + if send_flags & libc::MSG_DONTWAIT != 0 { + return false; + } + let fl = unsafe { libc::fcntl(fd, libc::F_GETFL) }; + fl >= 0 && (fl & libc::O_NONBLOCK) == 0 +} + +/// One `sendmsg` of `m` starting at byte `offset`. The destination address and +/// control ancillary are attached only at `offset == 0`: `SCM_RIGHTS` transmits +/// exactly once, and a stream continuation carries no new address. Returns the +/// kernel result (>= 0 bytes, or -1 with errno in `*__errno_location`). +fn send_materialized_at(fd: RawFd, m: &MaterializedMsg, offset: usize, flags: i32) -> isize { + let iov = libc::iovec { + iov_base: unsafe { m.data.as_ptr().add(offset) } as *mut libc::c_void, + iov_len: m.data.len() - offset, + }; + let mut msg: libc::msghdr = unsafe { std::mem::zeroed() }; + if offset == 0 { + if !m.addr.is_empty() { + msg.msg_name = m.addr.as_ptr() as *mut libc::c_void; + msg.msg_namelen = m.addr.len() as u32; + } + if let Some(ref c) = m.control { + msg.msg_control = c.as_ptr() as *mut libc::c_void; + msg.msg_controllen = c.len(); + } + } + msg.msg_iov = &iov as *const libc::iovec as *mut libc::iovec; + msg.msg_iovlen = 1; + unsafe { libc::sendmsg(fd, &msg, flags) } +} + +/// Read a child's iovec array (`iov_bytes` = the raw `struct iovec[]`) and +/// concatenate the referenced buffers into one owned payload, capped at +/// `MAX_SEND_BUF` total so a hostile child can't OOM the supervisor. A null +/// base or zero length contributes nothing (matching the prior per-iovec copy). +fn flatten_iovecs( + notif: &SeccompNotif, + notif_fd: RawFd, + iov_bytes: &[u8], + iovlen: usize, +) -> Result, i32> { + let mut data: Vec = Vec::new(); + for i in 0..iovlen { + let off = i * 16; + if off + 16 > iov_bytes.len() { + break; + } + let base = u64::from_ne_bytes(iov_bytes[off..off + 8].try_into().unwrap()); + let len = u64::from_ne_bytes(iov_bytes[off + 8..off + 16].try_into().unwrap()) as usize; + if base == 0 || len == 0 { + continue; + } + if len > MAX_SEND_BUF || data.len().saturating_add(len) > MAX_SEND_BUF { + return Err(libc::EMSGSIZE); + } + data.extend_from_slice( + &read_child_mem(notif_fd, notif.id, notif.pid, base, len).map_err(|_| libc::EIO)?, + ); + } + Ok(data) +} + +/// Resolve a materialized send to a terminal action. The first attempt is +/// non-blocking (`MSG_DONTWAIT`) on the seccomp loop, so it never blocks there. +/// A non-blocking child gets whatever that one attempt returns (short count or +/// `EAGAIN`), exactly as the kernel would give it. A blocking child whose whole +/// message didn't fit is completed off the loop (`defer_send`), preserving the +/// kernel's "a blocking send of N returns N" contract without occupying the +/// loop or a worker thread — a stream send that partially fit continues from +/// the sent offset; a full send buffer defers from offset 0. +fn resolve_send(dup_fd: OwnedFd, m: MaterializedMsg, flags: i32, child_blocking: bool) -> NotifAction { + let ret = send_materialized_at(dup_fd.as_raw_fd(), &m, 0, flags | libc::MSG_DONTWAIT); + if ret >= 0 { + let sent = ret as usize; + if !child_blocking || sent >= m.data.len() { + return NotifAction::ReturnValue(ret as i64); + } + // Blocking stream socket, partial fit: finish the remainder off the loop. + return NotifAction::defer(defer_send(dup_fd, m, flags, sent)); + } + let err = unsafe { *libc::__errno_location() }; + if err == libc::EAGAIN || err == libc::EWOULDBLOCK { + if child_blocking { + return NotifAction::defer(defer_send(dup_fd, m, flags, 0)); + } + return NotifAction::Errno(libc::EAGAIN); + } + NotifAction::Errno(err) +} + +/// Byte-level completion core: await writability on the dup'd fd through the +/// Tokio IO driver's epoll (never blocking a worker thread) and push the rest of +/// the message, advancing `offset` past each partial send, until the whole +/// message is delivered or a real error occurs. Bounded by the supervisor's +/// deferred-work timeout, so a peer that never drains can wedge only this one +/// send for that bound — never the notification loop. +/// +/// `Ok(n)` is the total bytes sent: the full length on success, or the bytes +/// queued before a hard error interrupted a partially-sent stream. `Err(e)` is +/// returned only when nothing at all was sent. Shared by the single-message +/// deferral ([`defer_send`]) and the batch tail ([`complete_batch_entry`]). +async fn push_until_done( + dup_fd: OwnedFd, + m: MaterializedMsg, + flags: i32, + mut offset: usize, +) -> Result { + let afd = tokio::io::unix::AsyncFd::with_interest(dup_fd, tokio::io::Interest::WRITABLE) + .map_err(|_| libc::EIO)?; + loop { + let mut guard = afd.writable().await.map_err(|_| libc::EIO)?; + let ret = send_materialized_at(afd.get_ref().as_raw_fd(), &m, offset, flags | libc::MSG_DONTWAIT); + if ret >= 0 { + offset += ret as usize; + if offset >= m.data.len() { + return Ok(m.data.len()); + } + // More to send; the next writability edge (or an EAGAIN below) gates it. + continue; + } + let err = unsafe { *libc::__errno_location() }; + if err == libc::EAGAIN || err == libc::EWOULDBLOCK { + guard.clear_ready(); + continue; + } + return if offset > 0 { Ok(offset) } else { Err(err) }; + } +} + +/// Deferred tail of [`resolve_send`] for a single message: complete the send and +/// return the byte count (matching a blocking send of N returning N; a partial +/// stream then error returns the partial count). +async fn defer_send(dup_fd: OwnedFd, m: MaterializedMsg, flags: i32, offset: usize) -> NotifAction { + match push_until_done(dup_fd, m, flags, offset).await { + Ok(n) => NotifAction::ReturnValue(n as i64), + Err(e) => NotifAction::Errno(e), + } +} + +/// Deferred tail shared by the three `sendmmsg` batch loops. Completes entry +/// `prior_count` (which either would-block entirely, offset 0, or partially sent +/// a stream, offset > 0) off the loop, then reports the *message* count — not a +/// byte count — as `sendmmsg` requires: `prior_count + 1` once the entry is +/// fully delivered (writing its full `msg_len` back so a blocking caller that +/// ignores per-entry `msg_len` isn't silently truncated). On error it reports +/// `prior_count` (the entries already fully sent), or the errno only when +/// nothing at all has been sent (`prior_count == 0` and this entry made no +/// progress). Entries beyond this one are left for the child to retry, so the +/// batch is never materialized whole. +fn complete_batch_entry( + dup_fd: OwnedFd, + m: MaterializedMsg, + flags: i32, + offset: usize, + notif_fd: RawFd, + notif_id: u64, + notif_pid: u32, + msglen_addr: u64, + prior_count: usize, +) -> NotifAction { + let m_len = m.data.len(); + NotifAction::defer(async move { + match push_until_done(dup_fd, m, flags, offset).await { + Ok(n) if n == m_len => { + let bytes = (m_len as u32).to_ne_bytes(); + let _ = write_child_mem(notif_fd, notif_id, notif_pid, msglen_addr, &bytes); + NotifAction::ReturnValue((prior_count + 1) as i64) + } + // Partial stream then hard error: entry not fully delivered, so it is + // not counted; the child retries it (and the rest). + Ok(_) => NotifAction::ReturnValue(prior_count as i64), + Err(e) => { + if prior_count == 0 { + NotifAction::Errno(e) + } else { + NotifAction::ReturnValue(prior_count as i64) + } + } + } + }) +} + /// Validate, materialize, and send one `struct msghdr` on behalf of /// the child. Caller is responsible for: /// - dup'ing the child fd (`dup_fd`), /// - resolving the socket protocol (`protocol`) via -/// `query_socket_protocol` on that dup, -/// - having confirmed via `prescan_msghdr` that `msghdr_ptr` points -/// at an IP-family destination (non-NULL `msg_name`). +/// `query_socket_protocol` on that dup. +/// +/// `protocol` is `Option` because it is only consumed to validate a +/// *non-connected* IP destination against the allowlist. A connected send +/// (`msg_name == NULL`) — which is every send that reaches here on an AF_UNIX +/// socket, since its connection was already gated at connect time — carries no +/// destination and needs no protocol, so `None` is passed through unused. When +/// the message *is* non-connected, a missing protocol fails closed +/// (`ECONNREFUSED`), so an IP send whose protocol can't be resolved is refused +/// rather than escaping the allowlist. /// -/// Returns the byte count returned by `sendmsg`, or an errno suitable -/// for `NotifAction::Errno`. ECONNREFUSED is used both for "destination -/// blocked by policy" and for "couldn't parse a port from the -/// sockaddr"; EIO for sub-buffer read failures (iovec / control). +/// Returns a [`MaterializedMsg`] the caller sends (inline and, if it would +/// block, deferred) via [`resolve_send`] / [`send_materialized`]; or an errno. +/// ECONNREFUSED is used both for "destination blocked by policy" and for +/// "couldn't parse a port from the sockaddr"; EIO for sub-buffer read failures. async fn send_msghdr_on_behalf( notif: &SeccompNotif, ctx: &Arc, notif_fd: RawFd, dup_fd: &std::os::unix::io::OwnedFd, - protocol: Protocol, + protocol: Option, msghdr_ptr: u64, - flags: i32, -) -> Result { +) -> Result { let msghdr_bytes = match read_child_mem(notif_fd, notif.id, notif.pid, msghdr_ptr, 56) { Ok(b) if b.len() >= 56 => b, _ => return Err(libc::EFAULT), @@ -1447,6 +1796,9 @@ async fn send_msghdr_on_behalf( None => return Err(libc::EAFNOSUPPORT), }; let dest_port = parse_port_from_sockaddr(&addr_bytes); + // A non-connected IP send must have a resolved protocol to key the + // per-protocol allowlist. If it couldn't be resolved, fail closed. + let protocol = protocol.ok_or(ECONNREFUSED)?; let ns = ctx.network.lock().await; let live_policy = { @@ -1465,63 +1817,29 @@ async fn send_msghdr_on_behalf( } let iovlen = (msg_iovlen as usize).min(1024); - let iov_size = iovlen * 16; - let iov_bytes = match read_child_mem(notif_fd, notif.id, notif.pid, msg_iov_ptr, iov_size) { + let iov_bytes = match read_child_mem(notif_fd, notif.id, notif.pid, msg_iov_ptr, iovlen * 16) { Ok(b) => b, Err(_) => return Err(libc::EIO), }; - let mut data_bufs: Vec> = Vec::with_capacity(iovlen); - let mut local_iovs: Vec = Vec::with_capacity(iovlen); - for i in 0..iovlen { - let off = i * 16; - if off + 16 > iov_bytes.len() { break; } - let iov_base = u64::from_ne_bytes(iov_bytes[off..off + 8].try_into().unwrap()); - let iov_len = u64::from_ne_bytes(iov_bytes[off + 8..off + 16].try_into().unwrap()) as usize; - if iov_len > MAX_SEND_BUF { - return Err(libc::EMSGSIZE); - } - if iov_base == 0 || iov_len == 0 { - data_bufs.push(Vec::new()); - continue; - } - let buf = match read_child_mem(notif_fd, notif.id, notif.pid, iov_base, iov_len) { - Ok(b) => b, - Err(_) => return Err(libc::EIO), - }; - data_bufs.push(buf); - } - for buf in &data_bufs { - local_iovs.push(libc::iovec { - iov_base: buf.as_ptr() as *mut libc::c_void, - iov_len: buf.len(), - }); - } - - let control_buf = if msg_control_ptr != 0 && msg_controllen > 0 { - let len = (msg_controllen as usize).min(4096); - read_child_mem(notif_fd, notif.id, notif.pid, msg_control_ptr, len).ok() - } else { - None - }; + let data = flatten_iovecs(notif, notif_fd, &iov_bytes, iovlen)?; - let mut msg: libc::msghdr = unsafe { std::mem::zeroed() }; - if !connected { - msg.msg_name = addr_bytes.as_ptr() as *mut libc::c_void; - msg.msg_namelen = addr_bytes.len() as u32; - } - msg.msg_iov = local_iovs.as_mut_ptr(); - msg.msg_iovlen = local_iovs.len(); - if let Some(ref ctrl) = control_buf { - msg.msg_control = ctrl.as_ptr() as *mut libc::c_void; - msg.msg_controllen = ctrl.len(); - } - - let ret = unsafe { libc::sendmsg(dup_fd.as_raw_fd(), &msg, flags) }; - if ret >= 0 { - Ok(ret) - } else { - Err(unsafe { *libc::__errno_location() }) - } + // Translate SCM_RIGHTS / reject creds only for a unix socket; an IP socket's + // control carries no fds or credentials and passes through untouched. + let (control_buf, scm_fds) = materialize_control( + notif, + notif_fd, + msg_control_ptr, + msg_controllen, + socket_is_unix(dup_fd.as_raw_fd()), + )?; + + Ok(MaterializedMsg { + data, + control: control_buf, + addr: if connected { Vec::new() } else { addr_bytes }, + _scm_fds: scm_fds, + _pinned: None, + }) } // ============================================================ @@ -1617,32 +1935,60 @@ async fn sendmmsg_on_behalf( Ok(fd) => fd, Err(e) => return NotifAction::Errno(e.raw_os_error().unwrap_or(libc::EBADF)), }; - let protocol = match query_socket_protocol(dup_fd.as_raw_fd()) { - Some(p) => p, - None => return NotifAction::Errno(ECONNREFUSED), - }; + // Protocol is resolved as `Option` and consumed only by a non-connected + // IP entry (see `send_msghdr_on_behalf`). It is `None` for an AF_UNIX + // socket — whose connected entries send through the immune `dup_fd` + // without a destination check — so the batch is handled on-behalf here + // rather than refused with ECONNREFUSED. On-behalf (not Continue) keeps + // it TOCTOU-safe against a racing fd swap. + let protocol = query_socket_protocol(dup_fd.as_raw_fd()); let mut sent: usize = 0; let mut first_errno: Option = None; for i in 0..vlen { let entry_ptr = msgvec_ptr + (i * MMSGHDR_SIZE) as u64; - match send_msghdr_on_behalf(notif, ctx, notif_fd, &dup_fd, protocol, entry_ptr, flags) + let m = match send_msghdr_on_behalf(notif, ctx, notif_fd, &dup_fd, protocol, entry_ptr) .await { - Ok(n) => { - let bytes = (n as u32).to_ne_bytes(); - let _ = write_child_mem( - notif_fd, - notif.id, - notif.pid, - entry_ptr + MSG_LEN_OFFSET as u64, - &bytes, - ); - sent += 1; - } + Ok(m) => m, Err(errno) => { first_errno = Some(errno); break; } + }; + // A batch sends each entry non-blocking so it never occupies the loop; + // the two cases that must not (entry 0 fully blocked, or a partial + // stream entry) are completed off the loop instead of the child + // seeing a spurious EAGAIN or a silently-truncated msg_len. + let blocking = wants_blocking(dup_fd.as_raw_fd(), flags); + let ret = send_materialized_at(dup_fd.as_raw_fd(), &m, 0, flags | libc::MSG_DONTWAIT); + if ret >= 0 { + if blocking && (ret as usize) < m.data.len() { + return complete_batch_entry( + dup_fd, m, flags, ret as usize, notif_fd, notif.id, notif.pid, + entry_ptr + MSG_LEN_OFFSET as u64, sent, + ); + } + let bytes = (ret as u32).to_ne_bytes(); + let _ = write_child_mem( + notif_fd, notif.id, notif.pid, entry_ptr + MSG_LEN_OFFSET as u64, &bytes, + ); + sent += 1; + } else { + let err = unsafe { *libc::__errno_location() }; + if err == libc::EAGAIN || err == libc::EWOULDBLOCK { + if sent == 0 && blocking { + return complete_batch_entry( + dup_fd, m, flags, 0, notif_fd, notif.id, notif.pid, + entry_ptr + MSG_LEN_OFFSET as u64, 0, + ); + } + if sent == 0 { + first_errno = Some(libc::EAGAIN); + } + break; + } + first_errno = Some(err); + break; } } return if sent > 0 { @@ -1680,20 +2026,47 @@ async fn sendmmsg_on_behalf( for i in 0..vlen { let entry_ptr = msgvec_ptr + (i * MMSGHDR_SIZE) as u64; - match send_msghdr_on_behalf(notif, ctx, notif_fd, &dup_fd, protocol, entry_ptr, flags).await { - Ok(n) => { - let bytes = (n as u32).to_ne_bytes(); - let _ = write_child_mem( - notif_fd, notif.id, notif.pid, - entry_ptr + MSG_LEN_OFFSET as u64, - &bytes, - ); - sent += 1; - } + // Every entry is OnBehalf (IP, non-connected) per the prescan above, so + // the resolved protocol is always required and present here. + let m = match send_msghdr_on_behalf(notif, ctx, notif_fd, &dup_fd, Some(protocol), entry_ptr).await { + Ok(m) => m, Err(errno) => { first_errno = Some(errno); break; } + }; + // Non-blocking per entry (see the destination-policy batch above); the + // entry-0-fully-blocked and partial-stream cases complete off the loop. + let blocking = wants_blocking(dup_fd.as_raw_fd(), flags); + let ret = send_materialized_at(dup_fd.as_raw_fd(), &m, 0, flags | libc::MSG_DONTWAIT); + if ret >= 0 { + if blocking && (ret as usize) < m.data.len() { + return complete_batch_entry( + dup_fd, m, flags, ret as usize, notif_fd, notif.id, notif.pid, + entry_ptr + MSG_LEN_OFFSET as u64, sent, + ); + } + let bytes = (ret as u32).to_ne_bytes(); + let _ = write_child_mem( + notif_fd, notif.id, notif.pid, entry_ptr + MSG_LEN_OFFSET as u64, &bytes, + ); + sent += 1; + } else { + let err = unsafe { *libc::__errno_location() }; + if err == libc::EAGAIN || err == libc::EWOULDBLOCK { + if sent == 0 && blocking { + return complete_batch_entry( + dup_fd, m, flags, 0, notif_fd, notif.id, notif.pid, + entry_ptr + MSG_LEN_OFFSET as u64, 0, + ); + } + if sent == 0 { + first_errno = Some(libc::EAGAIN); + } + break; + } + first_errno = Some(err); + break; } } @@ -2024,6 +2397,57 @@ pub fn compose_virtual_etc_hosts( mod tests { use super::*; + // --- translate_scm_rights tests (control-buffer parsing, child-controlled) --- + + fn cmsg_hdr(cmsg_len: usize, level: i32, ctype: i32) -> Vec { + let mut b = vec![0u8; 16]; + b[0..8].copy_from_slice(&cmsg_len.to_ne_bytes()); + b[8..12].copy_from_slice(&level.to_ne_bytes()); + b[12..16].copy_from_slice(&ctype.to_ne_bytes()); + b + } + + #[test] + fn scm_rights_rejects_overflowing_cmsg_len() { + // A child-crafted cmsg_len near usize::MAX must fail closed (EINVAL), + // never overflow-panic (debug) or wrap past the bounds check (release). + let buf = cmsg_hdr(usize::MAX - 7, libc::SOL_SOCKET, libc::SCM_RIGHTS); + assert_eq!(translate_scm_rights(0, &buf).map(drop), Err(libc::EINVAL)); + } + + #[test] + fn scm_rights_rejects_short_header() { + let buf = cmsg_hdr(8, libc::SOL_SOCKET, libc::SCM_RIGHTS); // < CMSG_HDR (16) + assert_eq!(translate_scm_rights(0, &buf).map(drop), Err(libc::EINVAL)); + } + + #[test] + fn scm_rights_rejects_cmsg_running_past_buffer() { + let buf = cmsg_hdr(17, libc::SOL_SOCKET, libc::SCM_RIGHTS); // claims 17, has 16 + assert_eq!(translate_scm_rights(0, &buf).map(drop), Err(libc::EINVAL)); + } + + #[test] + fn scm_rights_rejects_credentials() { + // Identity ancillary data on the on-behalf path must fail closed, not be + // forwarded with the child's crafted pid/uid/gid. + let buf = cmsg_hdr(16, libc::SOL_SOCKET, libc::SCM_CREDENTIALS); + assert_eq!(translate_scm_rights(0, &buf).map(drop), Err(libc::EPERM)); + } + + #[test] + fn scm_rights_passes_through_empty_and_non_socket_cmsg() { + // Empty control → no-op. A non-SOL_SOCKET cmsg (e.g. an IP-level control) + // passes through byte-for-byte with no fetched fds. + let (out, fds) = translate_scm_rights(0, &[]).unwrap(); + assert!(out.is_empty() && fds.is_empty()); + + let buf = cmsg_hdr(16, libc::IPPROTO_IP, 2 /* IP_TTL-ish */); + let (out, fds) = translate_scm_rights(0, &buf).unwrap(); + assert_eq!(out, buf); + assert!(fds.is_empty()); + } + // --- NetAllow::parse tests --- #[test] diff --git a/crates/sandlock-core/tests/integration/test_network.rs b/crates/sandlock-core/tests/integration/test_network.rs index 45409fe..2e47e3d 100644 --- a/crates/sandlock-core/tests/integration/test_network.rs +++ b/crates/sandlock-core/tests/integration/test_network.rs @@ -793,3 +793,303 @@ async fn test_empty_net_allow_denies_tcp_despite_fs_grants() { "empty net_allow must deny TCP connect via Landlock (EACCES); got {got:?}" ); } + +/// Regression: a `sendmsg()` on a *connected* `AF_UNIX` stream socket must pass +/// through when a `net_allow` destination policy is active. The IP destination +/// policy governs IP sockets only; a bug routed every connected send through +/// the IP on-behalf path, where `query_socket_protocol` returns `None` for a +/// unix socket and the send was refused with `ECONNREFUSED`. That broke every +/// connected-socket `sendmsg` user (Wayland, D-Bus — they use `sendmsg` for its +/// fd-passing capability) the moment any `--net-allow` rule was set, even +/// though the send targets no network destination. `send()`/`sendto` were +/// unaffected (they Continue connected sockets), so only `sendmsg` regressed. +/// +/// A live `AF_UNIX` listener is used so the child's `connect()` succeeds and the +/// `sendmsg()` reaches a real peer: the send must return the byte count, not an +/// errno. +#[tokio::test] +async fn test_connected_unix_sendmsg_passes_through_under_net_policy() { + use std::io::Read; + use std::os::unix::net::UnixListener; + + let sock_path = temp_file("unix-sendmsg.sock"); + let _ = std::fs::remove_file(&sock_path); + let listener = UnixListener::bind(&sock_path).unwrap(); + + // Accept + drain in a background thread so the child's send has a peer. + let accepter = std::thread::spawn(move || { + if let Ok((mut conn, _)) = listener.accept() { + let mut buf = [0u8; 16]; + let _ = conn.read(&mut buf); + } + }); + + let out = temp_file("unix-sendmsg-out"); + + // `net_allow` activates `has_net_destination_policy`; `fs_write("/tmp")` + // covers the socket path so `has_unix_fs_gate` is on too — the realistic + // shape of the report (a `--net-allow` rule plus fs grants). + let policy = base_policy().net_allow("127.0.0.1:80").build().unwrap(); + + let script = format!(concat!( + "import socket\n", + "s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)\n", + "s.connect('{sock}')\n", + "try:\n", + " n = s.sendmsg([b'hello'])\n", + " open('{out}', 'w').write('SENT:%d' % n)\n", + "except OSError as e:\n", + " open('{out}', 'w').write('ERR:%d' % e.errno)\n", + "s.close()\n", + ), sock = sock_path.display(), out = out.display()); + + let result = policy.with_name("test") + .run_interactive(&["python3", "-c", &script]).await.unwrap(); + assert!(result.success(), "exit={:?}", result.code()); + + let got = std::fs::read_to_string(&out).unwrap_or_default(); + let _ = std::fs::remove_file(&out); + let _ = accepter.join(); + let _ = std::fs::remove_file(&sock_path); + + assert_eq!( + got, "SENT:5", + "connected AF_UNIX sendmsg under net_allow must pass through (got {got:?})" + ); +} + +/// Regression: a `sendmsg()` that passes a file descriptor via `SCM_RIGHTS` on a +/// connected `AF_UNIX` socket must deliver a *working* fd when a `net_allow` +/// policy routes the send on-behalf. The on-behalf path copies the control +/// buffer, so without translation the child's fd numbers reach the supervisor's +/// `sendmsg` verbatim — passing the wrong file or failing `EBADF`. The fix +/// `pidfd_getfd`s each `SCM_RIGHTS` fd into the supervisor before the send. +/// +/// The child opens a payload file and passes its fd; the receiver reads the +/// passed fd back and must see the payload's exact bytes — which only holds if +/// the fd was translated to the real open file, not an unrelated supervisor fd. +#[tokio::test] +async fn test_connected_unix_sendmsg_translates_scm_rights_under_net_policy() { + use std::os::unix::io::{AsRawFd, FromRawFd, RawFd}; + + let payload = temp_file("scm-payload"); + std::fs::write(&payload, b"SCMOK").unwrap(); + + let sock_path = temp_file("scm-sendmsg.sock"); + let _ = std::fs::remove_file(&sock_path); + let listener = std::os::unix::net::UnixListener::bind(&sock_path).unwrap(); + + // Accept, recvmsg the SCM_RIGHTS fd, read it, and report the bytes. + let received = std::sync::Arc::new(std::sync::Mutex::new(String::new())); + let received_w = received.clone(); + let accepter = std::thread::spawn(move || { + let (conn, _) = match listener.accept() { Ok(c) => c, Err(_) => return }; + let cfd = conn.as_raw_fd(); + let mut databuf = [0u8; 8]; + let mut cmsgbuf = [0u8; 64]; + let mut iov = libc::iovec { + iov_base: databuf.as_mut_ptr() as *mut libc::c_void, + iov_len: databuf.len(), + }; + let mut msg: libc::msghdr = unsafe { std::mem::zeroed() }; + msg.msg_iov = &mut iov; + msg.msg_iovlen = 1; + msg.msg_control = cmsgbuf.as_mut_ptr() as *mut libc::c_void; + msg.msg_controllen = cmsgbuf.len() as _; + let n = unsafe { libc::recvmsg(cfd, &mut msg, 0) }; + if n < 0 { return; } + let cmsg = unsafe { libc::CMSG_FIRSTHDR(&msg) }; + if cmsg.is_null() { return; } + let c = unsafe { &*cmsg }; + if c.cmsg_level == libc::SOL_SOCKET && c.cmsg_type == libc::SCM_RIGHTS { + let data = unsafe { libc::CMSG_DATA(cmsg) } as *const RawFd; + let passed = unsafe { std::ptr::read_unaligned(data) }; + let f = unsafe { std::fs::File::from_raw_fd(passed) }; + use std::io::Read; + let mut s = String::new(); + let _ = (&f).take(16).read_to_string(&mut s); + *received_w.lock().unwrap() = s; + } + }); + + let out = temp_file("scm-out"); + // fs_read("/tmp") lets the child open the payload O_RDONLY; net_allow forces + // the on-behalf path where the SCM_RIGHTS translation matters. + let policy = base_policy().fs_read("/tmp").net_allow("127.0.0.1:80").build().unwrap(); + + let script = format!(concat!( + "import socket, array, os\n", + "s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)\n", + "s.connect('{sock}')\n", + "fd = os.open('{payload}', os.O_RDONLY)\n", + "try:\n", + " n = s.sendmsg([b'F'], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array('i', [fd]))])\n", + " open('{out}', 'w').write('SENT:%d' % n)\n", + "except OSError as e:\n", + " open('{out}', 'w').write('ERR:%d' % e.errno)\n", + "s.close()\n", + ), sock = sock_path.display(), payload = payload.display(), out = out.display()); + + let result = policy.with_name("test") + .run_interactive(&["python3", "-c", &script]).await.unwrap(); + assert!(result.success(), "exit={:?}", result.code()); + + let sent = std::fs::read_to_string(&out).unwrap_or_default(); + let _ = accepter.join(); + let got_fd = received.lock().unwrap().clone(); + let _ = std::fs::remove_file(&out); + let _ = std::fs::remove_file(&sock_path); + let _ = std::fs::remove_file(&payload); + + assert_eq!(sent, "SENT:1", "SCM_RIGHTS sendmsg under net_allow must succeed (got {sent:?})"); + assert_eq!( + got_fd, "SCMOK", + "receiver must read the passed fd's file — SCM_RIGHTS translated to the real open file (got {got_fd:?})" + ); +} + +/// A large blocking on-behalf send to a peer that stalls before draining must +/// fill the socket buffer, would-block, and *defer* off the notification loop — +/// then resume on writability and deliver every byte. This exercises the +/// `MSG_DONTWAIT` + `AsyncFd` deferred path; a regression would either truncate +/// the transfer, spuriously fail with `EAGAIN`, or (before the fix) block the +/// whole supervisor loop on the send. +#[tokio::test] +async fn test_large_blocking_send_defers_and_delivers_under_net_policy() { + use std::io::Read; + use std::os::unix::net::UnixListener; + use std::sync::atomic::{AtomicUsize, Ordering}; + + const N: usize = 4 * 1024 * 1024; // well past any socket send buffer + + let sock_path = temp_file("defer-send.sock"); + let _ = std::fs::remove_file(&sock_path); + let listener = UnixListener::bind(&sock_path).unwrap(); + + // Accept, stall briefly (so the sender's buffer fills and the on-behalf send + // must defer), then drain everything and count the bytes. + let total = std::sync::Arc::new(AtomicUsize::new(0)); + let total_w = total.clone(); + let accepter = std::thread::spawn(move || { + if let Ok((mut conn, _)) = listener.accept() { + std::thread::sleep(std::time::Duration::from_millis(500)); + let mut buf = [0u8; 65536]; + loop { + match conn.read(&mut buf) { + Ok(0) | Err(_) => break, + Ok(n) => { + total_w.fetch_add(n, Ordering::Relaxed); + } + } + } + } + }); + + let out = temp_file("defer-send-out"); + let policy = base_policy().net_allow("127.0.0.1:80").build().unwrap(); + // A SINGLE blocking send() of N bytes must return N — the kernel's contract + // for a blocking stream socket. A regression that returned on the first + // partial (one SO_SNDBUF worth) would write a short count here. + let script = format!(concat!( + "import socket\n", + "s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)\n", + "s.connect('{sock}')\n", + "n = s.send(b'z' * {n})\n", // one syscall; blocking → must return all N + "s.close()\n", + "open('{out}', 'w').write('SENT:%d' % n)\n", + ), sock = sock_path.display(), n = N, out = out.display()); + + let result = policy.with_name("test") + .run_interactive(&["python3", "-c", &script]).await.unwrap(); + assert!(result.success(), "exit={:?}", result.code()); + + let sent = std::fs::read_to_string(&out).unwrap_or_default(); + let _ = accepter.join(); + let got = total.load(Ordering::Relaxed); + let _ = std::fs::remove_file(&out); + let _ = std::fs::remove_file(&sock_path); + + assert_eq!( + sent, format!("SENT:{N}"), + "one blocking send() must return the full {N} bytes (deferred to completion, not truncated)" + ); + assert_eq!(got, N, "peer must receive all {N} bytes — no truncation on the deferred path"); +} + +/// `sendmmsg` of one large message on a blocking connected stream socket, to a +/// peer that stalls before draining: the entry can't complete on the first +/// non-blocking attempt, so it must be finished off the loop (not reported as a +/// spurious EAGAIN or a short `msg_len`). The child must see `ret == 1` with the +/// entry's `msg_len == N`, and the peer must receive all `N` bytes. +#[tokio::test] +async fn test_sendmmsg_blocking_entry_defers_and_delivers_under_net_policy() { + use std::io::Read; + use std::os::unix::net::UnixListener; + use std::sync::atomic::{AtomicUsize, Ordering}; + + const N: usize = 4 * 1024 * 1024; + + let sock_path = temp_file("mmsg-defer.sock"); + let _ = std::fs::remove_file(&sock_path); + let listener = UnixListener::bind(&sock_path).unwrap(); + + let total = std::sync::Arc::new(AtomicUsize::new(0)); + let total_w = total.clone(); + let accepter = std::thread::spawn(move || { + if let Ok((mut conn, _)) = listener.accept() { + std::thread::sleep(std::time::Duration::from_millis(500)); + let mut buf = [0u8; 65536]; + loop { + match conn.read(&mut buf) { + Ok(0) | Err(_) => break, + Ok(n) => { + total_w.fetch_add(n, Ordering::Relaxed); + } + } + } + } + }); + + let out = temp_file("mmsg-defer-out"); + let policy = base_policy().net_allow("127.0.0.1:80").build().unwrap(); + let script = format!(concat!( + "import ctypes, socket\n", + "libc = ctypes.CDLL('libc.so.6', use_errno=True)\n", + "libc.sendmmsg.restype = ctypes.c_int\n", + "class iovec(ctypes.Structure):\n", + " _fields_ = [('iov_base', ctypes.c_void_p), ('iov_len', ctypes.c_size_t)]\n", + "class msghdr(ctypes.Structure):\n", + " _fields_ = [('msg_name', ctypes.c_void_p), ('msg_namelen', ctypes.c_uint),\n", + " ('_p1', ctypes.c_uint), ('msg_iov', ctypes.c_void_p), ('msg_iovlen', ctypes.c_size_t),\n", + " ('msg_control', ctypes.c_void_p), ('msg_controllen', ctypes.c_size_t),\n", + " ('msg_flags', ctypes.c_int), ('_p2', ctypes.c_uint)]\n", + "class mmsghdr(ctypes.Structure):\n", + " _fields_ = [('msg_hdr', msghdr), ('msg_len', ctypes.c_uint), ('_p', ctypes.c_uint)]\n", + "s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)\n", // blocking, connected + "s.connect('{sock}')\n", + "data = ctypes.create_string_buffer(b'z' * {n}, {n})\n", + "iov = iovec(); iov.iov_base = ctypes.cast(data, ctypes.c_void_p).value; iov.iov_len = {n}\n", + "vec = (mmsghdr * 1)()\n", + "vec[0].msg_hdr.msg_iov = ctypes.cast(ctypes.pointer(iov), ctypes.c_void_p).value\n", + "vec[0].msg_hdr.msg_iovlen = 1\n", // msg_name NULL => connected + "ret = libc.sendmmsg(s.fileno(), vec, 1, 0)\n", + "open('{out}', 'w').write('ret=%d msg_len=%d' % (ret, vec[0].msg_len))\n", + "s.close()\n", + ), sock = sock_path.display(), n = N, out = out.display()); + + let result = policy.with_name("test") + .run_interactive(&["python3", "-c", &script]).await.unwrap(); + assert!(result.success(), "exit={:?}", result.code()); + + let content = std::fs::read_to_string(&out).unwrap_or_default(); + let _ = accepter.join(); + let got = total.load(Ordering::Relaxed); + let _ = std::fs::remove_file(&out); + let _ = std::fs::remove_file(&sock_path); + + assert_eq!( + content, format!("ret=1 msg_len={N}"), + "blocking sendmmsg entry must complete off-loop: ret=1 with full msg_len (got {content:?})" + ); + assert_eq!(got, N, "peer must receive all {N} bytes of the deferred batch entry"); +}