From 5bb51bc126e7719f1dec59ead5b5cffc6e542bd4 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:03:22 -0300 Subject: [PATCH 001/208] feat: deferred replies and pipe-to-self (rank 11), WIP --- CHANGELOG.md | 6 ++++++ docs/MIGRATION.md | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e997ea44..5a487a6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +- **Breaking (0.8), in progress:** deferred replies and pipe-to-self + (ROADMAP rank 11), implementing + [the design](docs/design/deferred-replies.md). Every request now has a reply + slot: one packed atomic word (state, token, registry and running-job count) + whose transitions are checked with Loom (`reply_slot_loom`). See + [migration](docs/MIGRATION.md#unreleased). - Design (not implementation) for rank 11, deferred typed replies and pipe-to-self, revised over four passes after independent reviews: `docs/design/deferred-replies.md`, client mapping for ethrex and ethlambda, diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 6f3e8102..816f791f 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -3,6 +3,15 @@ Upgrade notes per release, newest first. The [CHANGELOG](../CHANGELOG.md) lists every change; this guide covers what you need to edit when upgrading. +## Unreleased + +### Deferred replies (rank 11, targets 0.8, in progress) + +This is a breaking release. The notes below grow with the implementation: + +- `ActorError` is `#[non_exhaustive]` and gains `ReplyDropped` and + `Overloaded`; add a wildcard arm to exhaustive matches. + ## 0.7.3 **Behaviour change (supervision).** A Transient child's replacement (any generation after the child was first From 847cfe81bd624caca57276726fb0caf560732295 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:08:40 -0300 Subject: [PATCH 002/208] feat(reply): packed-word reply slot state machine with Loom model --- concurrency/src/lib.rs | 2 + concurrency/src/reply/mod.rs | 35 ++ concurrency/src/reply/slot.rs | 545 +++++++++++++++++++++++++++ concurrency/tests/reply_slot_loom.rs | 417 ++++++++++++++++++++ rt/src/tasks/sim.rs | 7 + rt/src/time.rs | 67 ++++ 6 files changed, 1073 insertions(+) create mode 100644 concurrency/src/reply/mod.rs create mode 100644 concurrency/src/reply/slot.rs create mode 100644 concurrency/tests/reply_slot_loom.rs diff --git a/concurrency/src/lib.rs b/concurrency/src/lib.rs index d623935d..29696c02 100644 --- a/concurrency/src/lib.rs +++ b/concurrency/src/lib.rs @@ -121,6 +121,8 @@ mod native_link_regression_tests; mod actor_cleanup; mod mailbox; +#[allow(dead_code)] // wired into the request path in the next steps +mod reply; mod instrumentation; #[cfg(feature = "metrics")] diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs new file mode 100644 index 00000000..f59daaaa --- /dev/null +++ b/concurrency/src/reply/mod.rs @@ -0,0 +1,35 @@ +//! Reply transport for requests: reply slots, tokens and receivers +//! (docs/design/deferred-replies.md). + +pub(crate) mod slot; + +/// Production synchronization types for [`slot`]; the Loom model supplies its +/// own (`tests/reply_slot_loom.rs`). +mod sync { + pub(crate) use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + pub(crate) use std::sync::{Arc, Condvar, Mutex, MutexGuard}; + pub(crate) use std::thread::yield_now; + + /// Wait on `condvar` for at most `ticks` clock ticks (nanoseconds), or + /// until notified when `None`. + pub(crate) fn wait_ticks<'a, T>( + condvar: &Condvar, + guard: MutexGuard<'a, T>, + ticks: Option, + ) -> MutexGuard<'a, T> { + match ticks { + None => condvar.wait(guard).unwrap_or_else(|p| p.into_inner()), + Some(ticks) => { + condvar + .wait_timeout(guard, std::time::Duration::from_nanos(ticks)) + .unwrap_or_else(|p| p.into_inner()) + .0 + } + } + } +} + +/// Clock ticks now, on the `spawned_rt` clock (virtual under simulation). +pub(crate) fn now_ticks() -> u64 { + spawned_rt::time::now_ticks() +} diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs new file mode 100644 index 00000000..ac1c3bba --- /dev/null +++ b/concurrency/src/reply/slot.rs @@ -0,0 +1,545 @@ +//! The reply slot: one packed atomic word per request +//! (docs/design/deferred-replies.md §1, "Slot synchronization protocol"). +//! +//! ```text +//! word: state:4 | TOKEN:1 | REG:1 | JOBS:16 +//! credits(word) = max(JOBS, TOKEN and state is live ? 1 : 0) +//! ``` +//! +//! Every change of the state, `TOKEN`, `REG` or `JOBS` is one compare-and-swap +//! on the word. The owner's credit pool changes only by +//! `credits(new) - credits(old)` of a successful CAS: a positive delta is +//! reserved before the CAS and rolled back if the CAS fails; a negative delta +//! is returned after it, before the receiver is woken. +//! +//! `WRITING` counts as live for `credits`: it is the transient state of a +//! reply between its linearization point (`OPEN -> WRITING`) and the +//! publication of the value (`WRITING -> REPLIED`) or the deadline check +//! (`WRITING -> TIMED_OUT`), so the token's credit returns at that second CAS. +//! +//! This file is shared with the Loom model (`tests/reply_slot_loom.rs`), which +//! supplies the synchronization types through `super::sync`. Time is passed in +//! as a `now` closure returning clock ticks, so the model controls the clock. +//! No user code runs under the slot mutex: values and wakers are moved out and +//! dropped or woken after it is released. + +use super::sync::{wait_ticks, yield_now, Arc, AtomicU64, AtomicUsize, Condvar, Mutex, Ordering}; +use std::task::{Poll, Waker}; + +pub(crate) const OPEN: u64 = 0; +pub(crate) const WRITING: u64 = 1; +pub(crate) const REPLIED: u64 = 2; +pub(crate) const DROPPED: u64 = 3; +pub(crate) const OVERLOADED: u64 = 4; +pub(crate) const TIMED_OUT: u64 = 5; +pub(crate) const STOPPED: u64 = 6; +pub(crate) const ABANDONED: u64 = 7; + +const STATE_MASK: u64 = 0xF; +pub(crate) const TOKEN: u64 = 1 << 4; +pub(crate) const REG: u64 = 1 << 5; +const JOBS_SHIFT: u32 = 6; +const JOB_ONE: u64 = 1 << JOBS_SHIFT; +/// Largest running linked-job count per slot; `MAX_OUTSTANDING` is capped at +/// this value at configuration so the field cannot overflow. +pub(crate) const MAX_JOBS: u64 = 0xFFFF; +const JOBS_MASK: u64 = MAX_JOBS << JOBS_SHIFT; + +/// Deadline value meaning "never" (`Deadline::Never`). +pub(crate) const NEVER: u64 = u64::MAX; + +pub(crate) fn state(word: u64) -> u64 { + word & STATE_MASK +} +pub(crate) fn jobs(word: u64) -> u64 { + (word & JOBS_MASK) >> JOBS_SHIFT +} +fn live(state: u64) -> bool { + state == OPEN || state == WRITING +} +/// Credits a slot holds in its owner's pool. +pub(crate) fn credits(word: u64) -> u64 { + let token = (word & TOKEN != 0 && live(state(word))) as u64; + jobs(word).max(token) +} +fn with_state(word: u64, to: u64) -> u64 { + (word & !STATE_MASK) | to +} + +/// Terminal outcome of a request, as the caller sees it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Terminal { + Replied, + Dropped, + Overloaded, + TimedOut, + Stopped, + Abandoned, +} + +impl Terminal { + pub(crate) fn of(state: u64) -> Option { + Some(match state { + REPLIED => Self::Replied, + DROPPED => Self::Dropped, + OVERLOADED => Self::Overloaded, + TIMED_OUT => Self::TimedOut, + STOPPED => Self::Stopped, + ABANDONED => Self::Abandoned, + _ => return None, + }) + } + fn state(self) -> u64 { + match self { + Self::Replied => REPLIED, + Self::Dropped => DROPPED, + Self::Overloaded => OVERLOADED, + Self::TimedOut => TIMED_OUT, + Self::Stopped => STOPPED, + Self::Abandoned => ABANDONED, + } + } +} + +/// Outcome of a reply, as the replier sees it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Delivery { + /// The value was published to the caller. + Delivered, + /// The caller left (abandoned the receiver) or the slot was already + /// answered (dropped, overloaded). + CallerGone, + /// The request's deadline had passed; the value was dropped. + Expired, + /// The owner died and its registry closed the slot (`ActorStopped`). + OwnerExited, +} + +/// Why `defer()` or a linked-job attach failed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Refused { + /// The slot is no longer open (answered, expired, abandoned, closed). + Closed, + /// The owner's credit pool is exhausted. For `defer()` the caller has + /// already been answered `Overloaded`. + Overloaded, +} + +/// Owner-wide credit counter (`MAX_OUTSTANDING`). +pub(crate) struct CreditPool { + used: AtomicUsize, + max: usize, +} + +impl CreditPool { + pub(crate) fn new(max: usize) -> Self { + Self { + used: AtomicUsize::new(0), + max, + } + } + /// Reserve `n` credits, failing at the bound without waiting. + pub(crate) fn reserve(&self, n: u64) -> bool { + let n = n as usize; + self.used + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |used| { + used.checked_add(n).filter(|total| *total <= self.max) + }) + .is_ok() + } + pub(crate) fn release(&self, n: u64) { + if n != 0 { + let previous = self.used.fetch_sub(n as usize, Ordering::AcqRel); + debug_assert!(previous >= n as usize, "credit pool underflow"); + } + } + pub(crate) fn used(&self) -> usize { + self.used.load(Ordering::Acquire) + } + pub(crate) fn max(&self) -> usize { + self.max + } +} + +struct Inner { + /// Written only by the reply that moved the word to `WRITING`, and + /// published by `WRITING -> REPLIED` under this lock. + value: Option, + waker: Option, + /// The owner's pool, set by `defer()` before its CAS installs `TOKEN`. + pool: Option>, + /// Threads parked in `recv_blocking`. + parked: usize, +} + +pub(crate) struct Slot { + word: AtomicU64, + deadline: AtomicU64, + inner: Mutex>, + parked: Condvar, +} + +fn lock(mutex: &Mutex) -> super::sync::MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(|p| p.into_inner()) +} + +/// What a winning transition must do after its CAS: return credits, then wake. +struct Wake { + pool: Option>, + release: u64, + waker: Option, + notify: bool, +} + +impl Wake { + fn run(self, slot_parked: &Condvar) { + if let Some(pool) = self.pool { + pool.release(self.release); + } + if let Some(waker) = self.waker { + waker.wake(); + } + if self.notify { + slot_parked.notify_all(); + } + } +} + +impl Slot { + /// A fresh `OPEN` slot whose deadline is `deadline` ticks (`NEVER` for none). + pub(crate) fn new(deadline: u64) -> Self { + Self { + word: AtomicU64::new(OPEN), + deadline: AtomicU64::new(deadline), + inner: Mutex::new(Inner { + value: None, + waker: None, + pool: None, + parked: 0, + }), + parked: Condvar::new(), + } + } + + pub(crate) fn word(&self) -> u64 { + self.word.load(Ordering::Acquire) + } + + pub(crate) fn deadline(&self) -> u64 { + self.deadline.load(Ordering::Acquire) + } + + pub(crate) fn is_terminal(&self) -> bool { + !live(state(self.word())) + } + + /// One CAS with credit accounting. `step` maps the current word to the new + /// one or aborts. A positive delta is reserved from `pool` first and rolled + /// back on CAS failure; `Err(None)` reports that the reservation failed. + /// Returns `(old, new)`; the caller returns any negative delta. + fn cas( + &self, + pool: Option<&CreditPool>, + mut step: impl FnMut(u64) -> Result, + ) -> Result<(u64, u64), Option> { + let mut current = self.word.load(Ordering::Acquire); + loop { + let next = step(current).map_err(Some)?; + let up = credits(next).saturating_sub(credits(current)); + if up > 0 { + match pool { + Some(pool) if pool.reserve(up) => {} + _ => return Err(None), + } + } + match self + .word + .compare_exchange(current, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => return Ok((current, next)), + Err(actual) => { + if let (Some(pool), true) = (pool, up > 0) { + pool.release(up); + } + current = actual; + } + } + } + } + + /// After a winning CAS `old -> new`: collect the credit return and, when + /// the state became terminal, the waker. The caller runs it after every + /// lock is released. + fn after(&self, old: u64, new: u64) -> Wake { + let release = credits(old).saturating_sub(credits(new)); + let terminal = live(state(old)) && !live(state(new)); + if release == 0 && !terminal { + return Wake { + pool: None, + release: 0, + waker: None, + notify: false, + }; + } + let mut inner = lock(&self.inner); + Wake { + pool: if release > 0 { inner.pool.clone() } else { None }, + release, + waker: if terminal { inner.waker.take() } else { None }, + notify: terminal && inner.parked > 0, + } + } + + /// `OPEN -> to` (drop, overloaded, expiry, abandonment, registry close). + /// Returns whether this call won the transition. + pub(crate) fn terminate(&self, to: Terminal) -> bool { + debug_assert!(to != Terminal::Replied); + let target = to.state(); + match self.cas::<()>(None, |w| { + if state(w) == OPEN { + Ok(with_state(w, target)) + } else { + Err(()) + } + }) { + Ok((old, new)) => { + self.after(old, new).run(&self.parked); + true + } + Err(_) => false, + } + } + + /// Registry close at the owner's death: every slot still `OPEN` becomes + /// `STOPPED`. A reply in `WRITING` is a value write with no user code in + /// it, so the close waits for it to resolve. + pub(crate) fn close(&self) -> bool { + loop { + if self.terminate(Terminal::Stopped) { + return true; + } + if state(self.word()) != WRITING { + return false; + } + yield_now(); + } + } + + /// `[deadline]`: expire the slot if `now >= deadline`. Returns whether the + /// deadline has passed (whoever won the transition). + pub(crate) fn check_deadline(&self, now: &impl Fn() -> u64) -> bool { + let deadline = self.deadline(); + if deadline != NEVER && now() >= deadline { + self.terminate(Terminal::TimedOut); + true + } else { + false + } + } + + /// `is_closed()`: not `OPEN`, or past its deadline (then expired). + pub(crate) fn is_closed(&self, now: &impl Fn() -> u64) -> bool { + state(self.word()) != OPEN || self.check_deadline(now) + } + + /// `with_deadline(at)`: shorten only, then the `[deadline]` check. + pub(crate) fn shorten(&self, at: u64, now: &impl Fn() -> u64) { + let _ = self + .deadline + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |d| (at < d).then_some(at)); + self.check_deadline(now); + } + + /// `defer()`: `OPEN, TOKEN=0 -> OPEN, TOKEN=1, REG=1`, reserving the + /// token's credit when no linked job holds one. At the bound the caller + /// is answered `Overloaded` in the same step. + pub(crate) fn defer(&self, pool: &Arc, now: &impl Fn() -> u64) -> Result<(), Refused> { + if self.check_deadline(now) { + return Err(Refused::Closed); + } + lock(&self.inner).pool = Some(pool.clone()); + match self.cas(Some(pool), |w| { + if state(w) == OPEN && w & TOKEN == 0 { + Ok(w | TOKEN | REG) + } else { + Err(()) + } + }) { + Ok(_) => Ok(()), + Err(Some(())) => Err(Refused::Closed), + Err(None) => { + self.terminate(Terminal::Overloaded); + Err(Refused::Overloaded) + } + } + } + + /// `pipe_reply` attach: requires `OPEN` and `TOKEN`; `JOBS + 1`. The first + /// job shares the token's credit; a further one reserves a new credit. + pub(crate) fn attach(&self, now: &impl Fn() -> u64) -> Result<(), Refused> { + if self.check_deadline(now) { + return Err(Refused::Closed); + } + let pool = lock(&self.inner).pool.clone(); + match self.cas(pool.as_deref(), |w| { + if state(w) != OPEN || w & TOKEN == 0 { + Err(Refused::Closed) + } else if jobs(w) >= MAX_JOBS { + Err(Refused::Overloaded) + } else { + Ok(w + JOB_ONE) + } + }) { + Ok(_) => Ok(()), + Err(Some(refused)) => Err(refused), + Err(None) => Err(Refused::Overloaded), + } + } + + /// A linked job ended (in any state): `JOBS - 1`, returning a credit if + /// this lowers `credits`. + pub(crate) fn job_end(&self) { + match self.cas::<()>(None, |w| { + debug_assert!(jobs(w) >= 1, "job_end without a running job"); + Ok(w - JOB_ONE) + }) { + Ok((old, new)) => self.after(old, new).run(&self.parked), + Err(_) => unreachable!("job_end never aborts or reserves"), + } + } + + /// Change only the state field of a `WRITING` word, retrying on concurrent + /// `JOBS` changes (a whole-word store would lose a job-end decrement). + fn resolve_writing(&self, to: u64) -> (u64, u64) { + match self.cas::<()>(None, |w| { + debug_assert_eq!(state(w), WRITING); + Ok(with_state(w, to)) + }) { + Ok(pair) => pair, + Err(_) => unreachable!("resolving WRITING never aborts or reserves"), + } + } + + /// Reply: `OPEN -> WRITING`, then read the deadline: at or past it, + /// `WRITING -> TIMED_OUT` and the value comes back as `Expired`; + /// otherwise write the value and `WRITING -> REPLIED`. A value that was + /// not published is returned so the caller drops it outside every lock. + pub(crate) fn reply(&self, value: R, now: &impl Fn() -> u64) -> (Delivery, Option) { + if let Err(Some(s)) = self.cas(None, |w| { + if state(w) == OPEN { + Ok(with_state(w, WRITING)) + } else { + Err(state(w)) + } + }) { + let delivery = match s { + TIMED_OUT => Delivery::Expired, + STOPPED => Delivery::OwnerExited, + _ => Delivery::CallerGone, + }; + return (delivery, Some(value)); + } + let deadline = self.deadline(); + if deadline != NEVER && now() >= deadline { + let (old, new) = self.resolve_writing(TIMED_OUT); + self.after(old, new).run(&self.parked); + return (Delivery::Expired, Some(value)); + } + let wake = { + let mut inner = lock(&self.inner); + inner.value = Some(value); + let (old, new) = self.resolve_writing(REPLIED); + let release = credits(old).saturating_sub(credits(new)); + Wake { + pool: if release > 0 { inner.pool.clone() } else { None }, + release, + waker: inner.waker.take(), + notify: inner.parked > 0, + } + }; + wake.run(&self.parked); + (Delivery::Delivered, None) + } + + /// Receiver side: take the outcome of a terminal word. `Replied` with no + /// value means it was already taken. + fn take(&self, state: u64) -> Option> { + let terminal = Terminal::of(state)?; + Some(match terminal { + Terminal::Replied => match lock(&self.inner).value.take() { + Some(value) => Ok(value), + None => Err(Terminal::Replied), + }, + other => Err(other), + }) + } + + /// Poll from the caller's receiver. Expires the slot when its deadline has + /// passed; otherwise registers `waker` and returns `Pending`. + pub(crate) fn poll(&self, waker: &Waker, now: &impl Fn() -> u64) -> Poll> { + loop { + if let Some(result) = self.take(state(self.word())) { + return Poll::Ready(result); + } + if self.check_deadline(now) { + continue; + } + let mut inner = lock(&self.inner); + let w = self.word(); + if !live(state(w)) { + drop(inner); + continue; + } + match &inner.waker { + Some(old) if old.will_wake(waker) => {} + _ => { + let old = inner.waker.replace(waker.clone()); + drop(inner); + drop(old); + } + } + return Poll::Pending; + } + } + + /// Non-blocking check from the caller's receiver. + pub(crate) fn try_take(&self, now: &impl Fn() -> u64) -> Option> { + if let Some(result) = self.take(state(self.word())) { + return Some(result); + } + if self.check_deadline(now) { + return self.take(state(self.word())); + } + None + } + + /// Threads-mode receive: park until terminal, expiring the slot at its + /// deadline. + pub(crate) fn recv_blocking(&self, now: &impl Fn() -> u64) -> Result { + loop { + if let Some(result) = self.take(state(self.word())) { + return result; + } + if self.check_deadline(now) { + continue; + } + let mut inner = lock(&self.inner); + if !live(state(self.word())) { + continue; + } + let deadline = self.deadline(); + let remaining = (deadline != NEVER).then(|| deadline.saturating_sub(now())); + if remaining == Some(0) { + continue; + } + inner.parked += 1; + let mut inner = wait_ticks(&self.parked, inner, remaining); + inner.parked -= 1; + } + } + + /// The caller dropped its receiver: `OPEN -> ABANDONED`. + pub(crate) fn abandon(&self) -> bool { + self.terminate(Terminal::Abandoned) + } +} diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs new file mode 100644 index 00000000..b61d7c3f --- /dev/null +++ b/concurrency/tests/reply_slot_loom.rs @@ -0,0 +1,417 @@ +//! Loom model of the reply slot's packed-word protocol +//! (docs/design/deferred-replies.md §1). One test per concurrent pair: +//! reply, expiry, shortening, abandonment, token drop, owner close, defer, +//! linked-job attach and end, and the parked/polled receiver. +//! +//! Run: `RUSTFLAGS="--cfg spawned_scheduler_loom" cargo test -p spawned-concurrency +//! --no-default-features --test reply_slot_loom --release`. +#![cfg(spawned_scheduler_loom)] + +mod sync { + pub(crate) use loom::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + pub(crate) use loom::sync::{Arc, Condvar, Mutex, MutexGuard}; + pub(crate) use loom::thread::yield_now; + + /// Loom has no timed wait; tests park only on slots without a deadline. + pub(crate) fn wait_ticks<'a, T>( + condvar: &Condvar, + guard: MutexGuard<'a, T>, + ticks: Option, + ) -> MutexGuard<'a, T> { + assert!(ticks.is_none(), "timed park is not modelled"); + condvar.wait(guard).unwrap_or_else(|p| p.into_inner()) + } +} + +#[allow(dead_code)] +#[path = "../src/reply/slot.rs"] +mod slot; + +use loom::sync::atomic::{AtomicU64, Ordering}; +use loom::sync::Arc; +use slot::{credits, jobs, state, CreditPool, Delivery, Refused, Slot, Terminal, NEVER}; + +fn model(f: impl Fn() + Sync + Send + 'static) { + let mut builder = loom::model::Builder::new(); + builder.preemption_bound = Some(3); + builder.max_permutations = Some(200_000); + builder.check(f); +} + +/// A model clock: ticks advanced explicitly by the test. +#[derive(Clone)] +struct Clock(Arc); +impl Clock { + fn at(t: u64) -> Self { + Self(Arc::new(AtomicU64::new(t))) + } + fn now(&self) -> impl Fn() -> u64 + '_ { + move || self.0.load(Ordering::SeqCst) + } + fn set(&self, t: u64) { + self.0.store(t, Ordering::SeqCst); + } +} + +/// The pool's count always equals the sum of `credits(word)` over its slots +/// once every thread has joined. +fn assert_balanced(pool: &CreditPool, slots: &[&Slot]) { + let held: u64 = slots.iter().map(|s| credits(s.word())).sum(); + assert_eq!(pool.used() as u64, held, "credit counter diverged from slot words"); + assert!(pool.used() <= pool.max(), "credit bound exceeded"); +} + +fn deferred(pool: &Arc, deadline: u64) -> Arc> { + let slot = Arc::new(Slot::new(deadline)); + slot.defer(pool, &|| 0).expect("defer on a fresh slot"); + slot +} + +/// Reply versus expiry by a polling receiver as the clock reaches the +/// deadline: one terminal outcome, and no success at or after the deadline. +#[test] +fn reply_vs_expiry() { + model(|| { + let clock = Clock::at(0); + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, 10); + let replied_at = Arc::new(AtomicU64::new(NEVER)); + let replier = { + let (slot, clock, replied_at) = (slot.clone(), clock.clone(), replied_at.clone()); + loom::thread::spawn(move || { + let now = || { + let t = clock.now()(); + replied_at.store(t, Ordering::SeqCst); + t + }; + slot.reply(7, &now) + }) + }; + clock.set(10); + let expired = slot.check_deadline(&clock.now()); + let (delivery, back) = replier.join().unwrap(); + assert!(expired); + match delivery { + Delivery::Delivered => { + assert_eq!(state(slot.word()), slot::REPLIED); + assert!(replied_at.load(Ordering::SeqCst) < 10, "success at the deadline"); + assert!(back.is_none()); + } + Delivery::Expired => { + assert_eq!(state(slot.word()), slot::TIMED_OUT); + assert_eq!(back, Some(7)); + } + other => panic!("unexpected {other:?}"), + } + assert_balanced(&pool, &[&slot]); + }); +} + +/// Reply versus `with_deadline` shortening into the past: the reply's +/// `WRITING` CAS linearizes; success implies the reply read a live deadline. +#[test] +fn reply_vs_shortening() { + model(|| { + let clock = Clock::at(5); + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, 100); + let replier = { + let (slot, clock) = (slot.clone(), clock.clone()); + loom::thread::spawn(move || slot.reply(1, &clock.now())) + }; + slot.shorten(3, &clock.now()); + let (delivery, _) = replier.join().unwrap(); + assert_eq!(slot.deadline(), 3); + match delivery { + Delivery::Delivered => assert_eq!(state(slot.word()), slot::REPLIED), + Delivery::Expired => assert_eq!(state(slot.word()), slot::TIMED_OUT), + other => panic!("unexpected {other:?}"), + } + assert_balanced(&pool, &[&slot]); + }); +} + +/// Reply versus the caller dropping its receiver. +#[test] +fn reply_vs_abandon() { + model(|| { + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, NEVER); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply(1, &|| 0)) + }; + let abandoned = slot.abandon(); + let (delivery, back) = replier.join().unwrap(); + assert_ne!(abandoned, delivery == Delivery::Delivered); + if abandoned { + assert_eq!(state(slot.word()), slot::ABANDONED); + assert_eq!((delivery, back), (Delivery::CallerGone, Some(1))); + } else { + assert_eq!(state(slot.word()), slot::REPLIED); + } + assert_balanced(&pool, &[&slot]); + }); +} + +/// Reply versus the owner's registry close at death. A close that finds +/// `WRITING` waits for the reply to resolve. +#[test] +fn reply_vs_owner_close() { + model(|| { + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, NEVER); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply(1, &|| 0).0) + }; + let closed = slot.close(); + let delivery = replier.join().unwrap(); + if closed { + assert_eq!(state(slot.word()), slot::STOPPED); + assert_eq!(delivery, Delivery::OwnerExited); + } else { + assert_eq!(state(slot.word()), slot::REPLIED); + assert_eq!(delivery, Delivery::Delivered); + } + assert_balanced(&pool, &[&slot]); + }); +} + +/// A held token dropped (`DROPPED`) versus the owner close (`STOPPED`) versus +/// the deadline heap expiring it: exactly one wins, the credit returns once. +#[test] +fn drop_vs_close_vs_expiry() { + model(|| { + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, 10); + let dropper = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.terminate(Terminal::Dropped)) + }; + let expirer = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.check_deadline(&|| 10)) + }; + let closed = slot.close(); + let dropped = dropper.join().unwrap(); + assert!(expirer.join().unwrap()); + let s = state(slot.word()); + let expired = s == slot::TIMED_OUT; + assert_eq!([closed, dropped, expired].iter().filter(|b| **b).count(), 1); + assert_eq!(pool.used(), 0); + assert_balanced(&pool, &[&slot]); + }); +} + +/// `defer()` versus the caller abandoning: a slot that went terminal never +/// acquires a credit, and a deferred one releases it on abandonment. +#[test] +fn defer_vs_abandon() { + model(|| { + let pool = Arc::new(CreditPool::new(1)); + let slot: Arc> = Arc::new(Slot::new(NEVER)); + let caller = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.abandon()) + }; + let deferred = slot.defer(&pool, &|| 0); + caller.join().unwrap(); + assert_eq!(state(slot.word()), slot::ABANDONED); + match deferred { + Ok(()) | Err(Refused::Closed) => {} + Err(Refused::Overloaded) => panic!("bound of one exceeded by one token"), + } + assert_eq!(pool.used(), 0); + assert_balanced(&pool, &[&slot]); + }); +} + +/// Two slots defer concurrently against a bound of one: one wins, the other's +/// caller is answered `Overloaded` in the same step. +#[test] +fn defer_at_the_bound() { + model(|| { + let pool = Arc::new(CreditPool::new(1)); + let a: Arc> = Arc::new(Slot::new(NEVER)); + let b: Arc> = Arc::new(Slot::new(NEVER)); + let other = { + let (b, pool) = (b.clone(), pool.clone()); + loom::thread::spawn(move || b.defer(&pool, &|| 0)) + }; + let first = a.defer(&pool, &|| 0); + let second = other.join().unwrap(); + let outcomes = [first, second]; + assert_eq!(outcomes.iter().filter(|r| r.is_ok()).count(), 1); + let loser = if first.is_ok() { &b } else { &a }; + assert_eq!(state(loser.word()), slot::OVERLOADED); + assert_balanced(&pool, &[&a, &b]); + }); +} + +/// Linked jobs: a second attach, both job ends, and the owner close race. +/// The token and its first job share one credit; credits always match +/// `max(JOBS, live token)` and return to zero. +#[test] +fn attach_vs_job_end_vs_close() { + model(|| { + let pool = Arc::new(CreditPool::new(2)); + let slot = deferred(&pool, NEVER); + slot.attach(&|| 0).expect("first attach shares the token's credit"); + assert_eq!(pool.used(), 1); + let second = { + let slot = slot.clone(); + loom::thread::spawn(move || { + if slot.attach(&|| 0).is_ok() { + slot.job_end(); + } + }) + }; + let first_end = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.job_end()) + }; + slot.close(); + second.join().unwrap(); + first_end.join().unwrap(); + assert_eq!(jobs(slot.word()), 0); + assert_eq!(state(slot.word()), slot::STOPPED); + assert_eq!(pool.used(), 0); + assert_balanced(&pool, &[&slot]); + }); +} + +/// A job ending while the reply moves `WRITING -> REPLIED`: the state-field +/// CAS must not lose the `JOBS` decrement. +#[test] +fn job_end_during_reply() { + model(|| { + let pool = Arc::new(CreditPool::new(2)); + let slot = deferred(&pool, NEVER); + slot.attach(&|| 0).unwrap(); + slot.attach(&|| 0).unwrap(); + assert_eq!(pool.used(), 2); + let ender = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.job_end()) + }; + assert_eq!(slot.reply(9, &|| 0).0, Delivery::Delivered); + ender.join().unwrap(); + assert_eq!(jobs(slot.word()), 1); + assert_eq!(pool.used(), 1, "the job still running keeps its credit"); + slot.job_end(); + assert_eq!(pool.used(), 0); + assert_balanced(&pool, &[&slot]); + }); +} + +/// A further attach while a job still runs after the token closed is +/// refused (`Closed`), and the running job's credit survives the close. +#[test] +fn attach_after_close_is_refused() { + model(|| { + let pool = Arc::new(CreditPool::new(2)); + let slot = deferred(&pool, NEVER); + slot.attach(&|| 0).unwrap(); + let closer = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.terminate(Terminal::Dropped)) + }; + let attached = slot.attach(&|| 0); + closer.join().unwrap(); + match attached { + Ok(()) => { + assert_eq!(jobs(slot.word()), 2); + assert_eq!(pool.used(), 2); + slot.job_end(); + } + Err(refused) => assert_eq!(refused, Refused::Closed), + } + assert_eq!(pool.used(), 1); + slot.job_end(); + assert_eq!(pool.used(), 0); + assert_balanced(&pool, &[&slot]); + }); +} + +/// A parked threads-mode receiver is woken by the reply (no lost wakeup). +#[test] +fn parked_receiver_vs_reply() { + model(|| { + let slot: Arc> = Arc::new(Slot::new(NEVER)); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply(3, &|| 0).0) + }; + assert_eq!(slot.recv_blocking(&|| 0), Ok(3)); + assert_eq!(replier.join().unwrap(), Delivery::Delivered); + }); +} + +/// A parked receiver is woken by the owner close. +#[test] +fn parked_receiver_vs_close() { + model(|| { + let slot: Arc> = Arc::new(Slot::new(NEVER)); + let closer = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.close()) + }; + assert_eq!(slot.recv_blocking(&|| 0), Err(Terminal::Stopped)); + assert!(closer.join().unwrap()); + }); +} + +/// A polled (async) receiver is woken by the reply (no lost wakeup). +#[test] +fn polled_receiver_vs_reply() { + model(|| { + let slot: Arc> = Arc::new(Slot::new(NEVER)); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply(4, &|| 0).0) + }; + let got = loom::future::block_on(std::future::poll_fn(|cx| slot.poll(cx.waker(), &|| 0))); + assert_eq!(got, Ok(4)); + assert_eq!(replier.join().unwrap(), Delivery::Delivered); + }); +} + +/// A polled receiver woken by a drop of the deferred token. +#[test] +fn polled_receiver_vs_token_drop() { + model(|| { + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, NEVER); + let dropper = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.terminate(Terminal::Dropped)) + }; + let got = loom::future::block_on(std::future::poll_fn(|cx| slot.poll(cx.waker(), &|| 0))); + assert_eq!(got, Err(Terminal::Dropped)); + assert!(dropper.join().unwrap()); + assert_balanced(&pool, &[&slot]); + }); +} + +/// Caller give-up (abandon) versus deadline shortening versus the heap's +/// expiry: one terminal state and a single credit return. +#[test] +fn abandon_vs_shorten_vs_expiry() { + model(|| { + let clock = Clock::at(5); + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, 100); + let shortener = { + let (slot, clock) = (slot.clone(), clock.clone()); + loom::thread::spawn(move || slot.shorten(5, &clock.now())) + }; + let abandoned = slot.abandon(); + shortener.join().unwrap(); + let s = state(slot.word()); + assert!(s == slot::ABANDONED || s == slot::TIMED_OUT); + assert_eq!(abandoned, s == slot::ABANDONED); + assert_eq!(pool.used(), 0); + assert_balanced(&pool, &[&slot]); + }); +} diff --git a/rt/src/tasks/sim.rs b/rt/src/tasks/sim.rs index c5fecec2..bf24ab1a 100644 --- a/rt/src/tasks/sim.rs +++ b/rt/src/tasks/sim.rs @@ -1956,6 +1956,13 @@ fn running_world() -> Option> { pub struct Instant(Duration); impl Instant { + /// Virtual time since the simulation started (the clock-tick seam). + pub(crate) fn since_start(&self) -> Duration { + self.0 + } + pub(crate) fn from_start(elapsed: Duration) -> Self { + Self(elapsed) + } pub fn now() -> Self { match running_world() { Some(shared) => Self(lock(&shared.state).now), diff --git a/rt/src/time.rs b/rt/src/time.rs index 47b69500..f3290eb3 100644 --- a/rt/src/time.rs +++ b/rt/src/time.rs @@ -31,3 +31,70 @@ pub fn entropy() -> u64 { ^ NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed) } } + +/// Largest tick value an instant maps to; `u64::MAX` is reserved for "never". +pub const MAX_TICKS: u64 = u64::MAX - 1; + +/// Nanosecond clock ticks of `instant`, for packing deadlines into atomics. +/// +/// Ticks count from a process-wide epoch (the first call), or, under +/// simulation, from the start of the simulated world. Instants before the +/// epoch map to 0 and instants beyond the range saturate at [`MAX_TICKS`]. +pub fn to_ticks(instant: Instant) -> u64 { + let nanos = elapsed_since_epoch(instant).as_nanos(); + nanos.min(MAX_TICKS as u128) as u64 +} + +/// Inverse of [`to_ticks`]. +pub fn from_ticks(ticks: u64) -> Instant { + let elapsed = std::time::Duration::from_nanos(ticks.min(MAX_TICKS)); + #[cfg(all(spawned_sim, feature = "async"))] + { + crate::tasks::sim::Instant::from_start(elapsed) + } + #[cfg(not(all(spawned_sim, feature = "async")))] + { + let epoch = epoch(); + epoch.checked_add(elapsed).unwrap_or(epoch) + } +} + +/// Ticks of [`Instant::now`]. +pub fn now_ticks() -> u64 { + to_ticks(Instant::now()) +} + +#[cfg(not(all(spawned_sim, feature = "async")))] +fn epoch() -> Instant { + static EPOCH: std::sync::OnceLock = std::sync::OnceLock::new(); + *EPOCH.get_or_init(Instant::now) +} + +fn elapsed_since_epoch(instant: Instant) -> std::time::Duration { + #[cfg(all(spawned_sim, feature = "async"))] + { + instant.since_start() + } + #[cfg(not(all(spawned_sim, feature = "async")))] + { + instant.saturating_duration_since(epoch()) + } +} + +#[cfg(test)] +mod tick_tests { + use super::*; + use std::time::Duration; + + #[test] + fn ticks_round_trip_and_order() { + let _ = now_ticks(); // initialize the epoch before `a` + let a = Instant::now(); + let b = a + Duration::from_millis(5); + assert!(to_ticks(b) > to_ticks(a)); + assert_eq!(to_ticks(b) - to_ticks(a), 5_000_000); + let back = from_ticks(to_ticks(b)); + assert_eq!(to_ticks(back), to_ticks(b)); + assert!(now_ticks() >= to_ticks(a)); + } +} From 877989ed98acde00293ada6e63766b957b3c03e5 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:09:13 -0300 Subject: [PATCH 003/208] test(reply): enable loom futures for the polled receiver --- concurrency/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concurrency/Cargo.toml b/concurrency/Cargo.toml index 31f204df..8f3e04e2 100644 --- a/concurrency/Cargo.toml +++ b/concurrency/Cargo.toml @@ -83,7 +83,7 @@ required-features = ["backend-experiments"] pprof = { version = "0.15", optional = true, features = ["flamegraph"] } [target.'cfg(spawned_scheduler_loom)'.dev-dependencies] -loom = "0.7" +loom = { version = "0.7", features = ["futures"] } [lints.rust] unexpected_cfgs = { level = "warn", check-cfg = ['cfg(spawned_scheduler_loom)', 'cfg(spawned_sim)', 'cfg(kani)', 'cfg(kani_slow)'] } From 50888a802c93d88c7dc7da754282eda5b1f2b712 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:09:36 -0300 Subject: [PATCH 004/208] chore: lock pin-utils for loom futures --- Cargo.lock | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index dfcb957c..191e20e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1000,6 +1000,7 @@ checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" dependencies = [ "cfg-if", "generator", + "pin-utils", "scoped-tls", "tracing", "tracing-subscriber", @@ -1291,6 +1292,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +[[package]] +name = "pin-utils" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13bee6c73da26345c729282832b60b0363cf3dd9f4bfd81d8551b7a1c889a113" + [[package]] name = "ping_pong" version = "0.1.0" From f2630f865654cda28ace493bf066ee39678faa01 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:10:07 -0300 Subject: [PATCH 005/208] test(reply): close must leave the slot terminal --- concurrency/tests/reply_slot_loom.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index b61d7c3f..39076cf6 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -166,6 +166,7 @@ fn reply_vs_owner_close() { loom::thread::spawn(move || slot.reply(1, &|| 0).0) }; let closed = slot.close(); + assert!(slot.is_terminal(), "close returned while a reply was WRITING"); let delivery = replier.join().unwrap(); if closed { assert_eq!(state(slot.word()), slot::STOPPED); From bc46ffcd61f1790902441c390933f29de317b330 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:10:22 -0300 Subject: [PATCH 006/208] ci: run the reply slot Loom model --- .github/workflows/verification.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml index 8f0f15db..977bdba3 100644 --- a/.github/workflows/verification.yml +++ b/.github/workflows/verification.yml @@ -100,6 +100,10 @@ jobs: env: RUSTFLAGS: --cfg spawned_scheduler_loom run: cargo test --locked -p spawned-concurrency --no-default-features --test mailbox_admission_loom + - name: Model-check the reply slot's packed-word protocol + env: + RUSTFLAGS: --cfg spawned_scheduler_loom + run: cargo test --locked --release -p spawned-concurrency --no-default-features --test reply_slot_loom # Data races in the runtime and its dependencies. The crates forbid # unsafe code, so ThreadSanitizer is the sanitizer that matters here. From 1c5d918cf16dfbb45b2ada8033a0d846cbefb3dc Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:19:59 -0300 Subject: [PATCH 007/208] feat: reply slots on the request path, PendingReply/ReplyTo, handler trait migration (WIP: tests not yet migrated) --- concurrency/src/error.rs | 12 + concurrency/src/lib.rs | 4 +- concurrency/src/metrics.rs | 4 + concurrency/src/reply/mod.rs | 472 +++++++++++++++++++++++++++++- concurrency/src/reply/registry.rs | 242 +++++++++++++++ concurrency/src/reply/slot.rs | 30 ++ concurrency/src/response.rs | 124 ++------ concurrency/src/tasks/actor.rs | 185 ++++++++---- concurrency/src/threads/actor.rs | 160 ++++++---- macros/src/lib.rs | 127 +++++--- rt/src/tasks/mod.rs | 4 +- rt/src/tasks/tokio/mod.rs | 2 +- testing/src/lib.rs | 19 ++ testing/src/shared.rs | 24 +- testing/src/tasks.rs | 12 +- testing/src/threads.rs | 6 +- 16 files changed, 1169 insertions(+), 258 deletions(-) create mode 100644 concurrency/src/reply/registry.rs diff --git a/concurrency/src/error.rs b/concurrency/src/error.rs index 87d08996..f9810660 100644 --- a/concurrency/src/error.rs +++ b/concurrency/src/error.rs @@ -50,7 +50,11 @@ pub(crate) fn panic_message(panic: &(dyn std::any::Any + Send)) -> String { } /// Errors that can occur when communicating with an actor. +/// +/// Non-exhaustive: later releases add reasons (for example the exit reason of +/// a dead actor, ROADMAP rank 5e). Match with a wildcard arm. #[derive(Debug, thiserror::Error)] +#[non_exhaustive] pub enum ActorError { /// The actor has stopped or its mailbox channel is closed. /// @@ -63,6 +67,14 @@ pub enum ActorError { /// does not reply in time. #[error("Request to Actor timed out")] RequestTimeout, + /// The handler dropped the request's reply obligation (a `PendingReply` + /// or a deferred `ReplyTo`) without answering it. + #[error("Reply dropped by the actor")] + ReplyDropped, + /// The actor rejected the request at a bound (for example its deferred + /// replies are at `MAX_OUTSTANDING`). + #[error("Actor overloaded")] + Overloaded, } impl From> for ActorError { diff --git a/concurrency/src/lib.rs b/concurrency/src/lib.rs index 29696c02..db02f7a5 100644 --- a/concurrency/src/lib.rs +++ b/concurrency/src/lib.rs @@ -96,6 +96,7 @@ pub mod link; pub mod message; pub mod monitor; pub mod registry; +pub mod reply; pub mod response; #[cfg(feature = "async")] pub mod supervision; @@ -109,6 +110,7 @@ pub use child_handle::{ActorId, ChildHandle}; pub use error::{ActorError, ExitReason}; pub use link::Exit; pub use monitor::{Down, MonitorRef}; +pub use reply::{Deadline, DeferError, Delivery, PendingReply, ReplyReceiver, ReplyTo}; pub use response::Response; pub use spawned_macros::{actor, protocol}; @@ -121,8 +123,6 @@ mod native_link_regression_tests; mod actor_cleanup; mod mailbox; -#[allow(dead_code)] // wired into the request path in the next steps -mod reply; mod instrumentation; #[cfg(feature = "metrics")] diff --git a/concurrency/src/metrics.rs b/concurrency/src/metrics.rs index 3173516b..080845cf 100644 --- a/concurrency/src/metrics.rs +++ b/concurrency/src/metrics.rs @@ -193,6 +193,8 @@ counters!( requests_replied_total, requests_timed_out_total, requests_actor_stopped_total, + requests_reply_dropped_total, + requests_overloaded_total, requests_cancelled_total, ); static QUEUE_WAIT: Histogram = Histogram::new(); @@ -391,6 +393,8 @@ impl RequestObservation { Ok(_) => &COUNTERS.requests_replied_total, Err(ActorError::RequestTimeout) => &COUNTERS.requests_timed_out_total, Err(ActorError::ActorStopped) => &COUNTERS.requests_actor_stopped_total, + Err(ActorError::ReplyDropped) => &COUNTERS.requests_reply_dropped_total, + Err(ActorError::Overloaded) => &COUNTERS.requests_overloaded_total, } .fetch_add(1, Relaxed); } diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index f59daaaa..05d93171 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -1,8 +1,26 @@ //! Reply transport for requests: reply slots, tokens and receivers //! (docs/design/deferred-replies.md). +//! +//! Every request has a reply slot ([`slot::Slot`]) shared by the envelope, +//! then by the handler's [`PendingReply`] (and, after [`PendingReply::defer`], +//! a storable [`ReplyTo`]), and by the caller's [`ReplyReceiver`]. Each slot +//! gets exactly one terminal discharge. +pub(crate) mod registry; pub(crate) mod slot; +pub use registry::DEFAULT_MAX_OUTSTANDING; +pub use slot::Delivery; + +use crate::error::ActorError; +use crate::ActorId; +use registry::Registry; +use slot::{Slot, Terminal, NEVER}; +use spawned_rt::time::Instant; +use std::marker::PhantomData; +use std::sync::Arc; +use std::time::Duration; + /// Production synchronization types for [`slot`]; the Loom model supplies its /// own (`tests/reply_slot_loom.rs`). mod sync { @@ -21,15 +39,467 @@ mod sync { None => condvar.wait(guard).unwrap_or_else(|p| p.into_inner()), Some(ticks) => { condvar - .wait_timeout(guard, std::time::Duration::from_nanos(ticks)) + .wait_timeout(guard, Duration::from_nanos(ticks)) .unwrap_or_else(|p| p.into_inner()) .0 } } } + use std::time::Duration; } /// Clock ticks now, on the `spawned_rt` clock (virtual under simulation). pub(crate) fn now_ticks() -> u64 { spawned_rt::time::now_ticks() } + +// --------------------------------------------------------------------------- +// Deadline +// --------------------------------------------------------------------------- + +/// A request's deadline, fixed when it is sent. +/// +/// `gen_server:call/3` takes a timeout or `infinity`; a request takes an +/// absolute instant or [`Deadline::Never`]. The default for `request` and +/// `request_raw` is [`DEFAULT_REQUEST_TIMEOUT`](crate::response::DEFAULT_REQUEST_TIMEOUT) +/// after the send. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Deadline { + /// Expire at this instant (on the `spawned_rt::time` clock). + At(Instant), + /// Never expire; the caller waits until the reply or the owner's death. + Never, +} + +impl Deadline { + /// `timeout` from now. + pub fn after(timeout: Duration) -> Self { + match Instant::now().checked_add(timeout) { + Some(at) => Self::At(at), + None => Self::Never, + } + } + + pub(crate) fn ticks(self) -> u64 { + match self { + Self::At(at) => spawned_rt::time::to_ticks(at), + Self::Never => NEVER, + } + } + + fn from_ticks(ticks: u64) -> Option { + (ticks != NEVER).then(|| spawned_rt::time::from_ticks(ticks)) + } +} + +fn record(_delivery: Delivery) { + #[cfg(feature = "metrics")] + crate::instrumentation::reply(_delivery == Delivery::Delivered); +} + +fn terminal_error(terminal: Terminal) -> ActorError { + match terminal { + Terminal::Dropped => ActorError::ReplyDropped, + Terminal::Overloaded => ActorError::Overloaded, + Terminal::TimedOut => ActorError::RequestTimeout, + Terminal::Stopped | Terminal::Abandoned | Terminal::Replied => ActorError::ActorStopped, + } +} + +/// A new reply slot for a request with `deadline`, and the caller's receiver. +pub(crate) fn reply_slot(deadline: Deadline) -> (SlotHandle, ReplyReceiver) { + let slot = Arc::new(Slot::new(deadline.ticks())); + (SlotHandle(Some(slot.clone())), ReplyReceiver::new(slot)) +} + +// --------------------------------------------------------------------------- +// Envelope side +// --------------------------------------------------------------------------- + +/// The envelope's hold on its request's slot. Dropped undispatched (mailbox +/// disposal at death, a rejected enqueue), it answers `ActorStopped`. +pub(crate) struct SlotHandle(Option>>); + +impl SlotHandle { + /// No caller: the message was sent, not requested. + pub(crate) fn detached() -> Self { + Self(None) + } + pub(crate) fn take(&mut self) -> Option>> { + self.0.take() + } +} + +impl Drop for SlotHandle { + fn drop(&mut self) { + if let Some(slot) = self.0.take() { + slot.close(); + } + } +} + +// --------------------------------------------------------------------------- +// PendingReply +// --------------------------------------------------------------------------- + +/// Why [`PendingReply::defer`] did not return a token. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum DeferError { + /// The owner's `MAX_OUTSTANDING` credits are in use; the caller has + /// already been answered `Overloaded`. + #[error("deferred replies at the bound; the caller was answered Overloaded")] + Overloaded, + /// The request is no longer open: its deadline passed or its caller left. + #[error("the request is closed (expired or abandoned)")] + Closed, + /// The message was sent, not requested: there is no caller to answer. + #[error("the message was sent, not requested")] + NoCaller, +} + +/// The reply obligation of the request being handled, scoped to the turn. +/// +/// Answer it with [`reply`](Self::reply), hand it out of the turn with +/// [`defer`](Self::defer), or drop it: a request whose `PendingReply` is +/// dropped is answered `ReplyDropped` when the turn returns, or +/// `ActorStopped` when it panics. A message reached by `send` has a detached +/// `PendingReply`: `reply` drops the value and `defer` returns +/// [`DeferError::NoCaller`]. +pub struct PendingReply<'t, R: Send + 'static> { + slot: Option>>, + registry: Option<&'t Arc>, + _turn: PhantomData<&'t mut ()>, +} + +impl std::fmt::Debug for PendingReply<'_, R> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PendingReply") + .field("detached", &self.slot.is_none()) + .finish_non_exhaustive() + } +} + +impl<'t, R: Send + 'static> PendingReply<'t, R> { + pub(crate) fn new(slot: Option>>, registry: &'t Arc) -> Self { + Self { + slot, + registry: Some(registry), + _turn: PhantomData, + } + } + + /// A `PendingReply` for a sent message: there is no caller. + pub fn detached() -> Self { + Self { + slot: None, + registry: None, + _turn: PhantomData, + } + } + + /// Answer the caller now. A value that is not delivered is dropped. + pub fn reply(mut self, value: R) -> Delivery { + match self.slot.take() { + None => { + drop(value); + Delivery::CallerGone + } + Some(slot) => { + let (delivery, unsent) = slot.reply(value, &now_ticks); + drop(unsent); + record(delivery); + delivery + } + } + } + + /// Answer the caller `Err(Overloaded)` now. + pub fn overloaded(mut self) { + if let Some(slot) = self.slot.take() { + slot.terminate(Terminal::Overloaded); + } + } + + /// Take the obligation out of the turn as a storable [`ReplyTo`]. This + /// takes one of the owner's `MAX_OUTSTANDING` credits; at the bound the + /// caller is answered `Overloaded` and this returns + /// [`DeferError::Overloaded`]. + pub fn defer(mut self) -> Result, DeferError> { + let slot = self.slot.take().ok_or(DeferError::NoCaller)?; + let registry = self.registry.take().ok_or(DeferError::NoCaller)?; + match slot.defer(registry.pool(), &now_ticks) { + Ok(()) => { + registry.register(&slot); + Ok(ReplyTo { + slot: Some(slot), + registry: registry.clone(), + }) + } + Err(slot::Refused::Closed) => Err(DeferError::Closed), + Err(slot::Refused::Overloaded) => Err(DeferError::Overloaded), + } + } + + /// The request's deadline, or `None` for [`Deadline::Never`] and sends. + pub fn deadline(&self) -> Option { + self.slot.as_ref().and_then(|s| Deadline::from_ticks(s.deadline())) + } + + /// Whether the caller can no longer be answered (the deadline passed or + /// the caller left). Checks the deadline first. Always `true` for sends. + pub fn is_closed(&self) -> bool { + self.slot.as_ref().is_none_or(|s| s.is_closed(&now_ticks)) + } + + /// Answer `ActorStopped` (test doubles standing in for an absent actor). + #[doc(hidden)] + pub fn __discharge_stopped(mut self) { + if let Some(slot) = self.slot.take() { + slot.terminate(Terminal::Stopped); + } + } + + /// Whether this is a request (has a caller) rather than a send. + pub fn has_caller(&self) -> bool { + self.slot.is_some() + } +} + +impl Drop for PendingReply<'_, R> { + fn drop(&mut self) { + if let Some(slot) = self.slot.take() { + match self.registry { + Some(registry) => registry.dropped(slot), + None => { + slot.terminate(Terminal::Dropped); + } + } + } + } +} + +// --------------------------------------------------------------------------- +// ReplyTo +// --------------------------------------------------------------------------- + +/// A deferred reply obligation: answer it from a later turn, a job, or +/// `stopped()`. Not `Clone`; `Send + 'static`. +/// +/// Dropping it answers the caller `ReplyDropped` (or `ActorStopped` while the +/// owner is exiting). The owner's death answers every open token +/// `ActorStopped`, wherever it is. +#[must_use = "dropping a ReplyTo answers ReplyDropped (or ActorStopped while the owner exits)"] +pub struct ReplyTo { + slot: Option>>, + registry: Arc, +} + +impl std::fmt::Debug for ReplyTo { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ReplyTo") + .field("owner", &self.registry.owner()) + .finish_non_exhaustive() + } +} + +impl ReplyTo { + /// Answer the caller. Returns how the reply was delivered. + pub fn reply(mut self, value: R) -> Delivery { + let slot = self.slot.take().expect("ReplyTo holds its slot until consumed"); + let (delivery, unsent) = slot.reply(value, &now_ticks); + drop(unsent); + record(delivery); + delivery + } + + /// Answer the caller `Err(Overloaded)`, e.g. on `InsertError::Full`. + pub fn overloaded(mut self) { + if let Some(slot) = self.slot.take() { + slot.terminate(Terminal::Overloaded); + } + } + + /// The request's deadline, or `None` for [`Deadline::Never`]. + pub fn deadline(&self) -> Option { + Deadline::from_ticks(self.slot().deadline()) + } + + /// Shorten the deadline to `at` (never extends it). + pub fn with_deadline(&self, at: Instant) { + self.slot().shorten(spawned_rt::time::to_ticks(at), &now_ticks); + self.registry.track_deadline(self.slot()); + } + + /// Whether the caller can no longer be answered. Checks the deadline + /// first. + pub fn is_closed(&self) -> bool { + self.slot().is_closed(&now_ticks) + } + + /// Wait until the caller can no longer be answered (expired, abandoned, + /// answered elsewhere or closed by the owner's death). + #[cfg(feature = "async")] + pub async fn closed(&self) { + let slot = self.slot(); + let waiting = std::future::poll_fn(|cx| slot.poll_closed(cx.waker(), &now_ticks)); + let deadline = slot.deadline(); + if deadline == NEVER { + return waiting.await; + } + let wait = Duration::from_nanos(deadline.saturating_sub(now_ticks())); + if spawned_rt::tasks::timeout(wait, waiting).await.is_err() { + slot.terminate(Terminal::TimedOut); + } + } + + /// The actor instance that owns this obligation. + pub fn owner(&self) -> ActorId { + self.registry.owner() + } + + fn slot(&self) -> &Arc> { + self.slot.as_ref().expect("ReplyTo holds its slot until consumed") + } + + pub(crate) fn slot_arc(&self) -> &Arc> { + self.slot() + } +} + +impl Drop for ReplyTo { + fn drop(&mut self) { + if let Some(slot) = self.slot.take() { + self.registry.dropped(slot); + } + } +} + +// --------------------------------------------------------------------------- +// ReplyReceiver +// --------------------------------------------------------------------------- + +#[cfg(feature = "async")] +type DeadlineSleep = spawned_rt::tasks::Sleep; +#[cfg(not(feature = "async"))] +type DeadlineSleep = std::future::Pending<()>; + +pin_project_lite::pin_project! { + /// The caller's side of a request: a future in `tasks` mode, and + /// [`recv`](Self::recv) in `threads` mode. Yields `Ok(reply)`, or + /// `RequestTimeout` at the deadline fixed at send, `ReplyDropped`, + /// `Overloaded`, or `ActorStopped` when the owner died. + /// + /// While polled it sleeps until the slot's deadline and then expires the + /// slot. Dropping it abandons the request: a later reply is dropped and + /// any deferred credit returns at once. + pub struct ReplyReceiver { + slot: Arc>, + done: bool, + armed: u64, + #[pin] + sleep: Option, + } + impl PinnedDrop for ReplyReceiver { + fn drop(this: Pin<&mut Self>) { + let this = this.project(); + if !*this.done { + this.slot.abandon(); + } + } + } +} + +impl std::fmt::Debug for ReplyReceiver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ReplyReceiver") + .field("done", &self.done) + .finish_non_exhaustive() + } +} + +impl ReplyReceiver { + fn new(slot: Arc>) -> Self { + Self { + slot, + done: false, + armed: NEVER, + sleep: None, + } + } + + /// The deadline fixed at send, or `None` for [`Deadline::Never`]. + pub fn deadline(&self) -> Option { + Deadline::from_ticks(self.slot.deadline()) + } + + /// Block the calling thread until the reply, the deadline or the owner's + /// death. Do not call it on an async executor thread. + pub fn recv(mut self) -> Result { + let result = self.slot.recv_blocking(&now_ticks).map_err(terminal_error); + self.done = true; + result + } + + /// Non-blocking check: `None` while the request is still open. + pub fn try_recv(&mut self) -> Option> { + assert!(!self.done, "ReplyReceiver used after completion"); + let result = self.slot.try_take(&now_ticks)?.map_err(terminal_error); + self.done = true; + Some(result) + } +} + +#[cfg(feature = "async")] +impl std::future::Future for ReplyReceiver { + type Output = Result; + + fn poll( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll { + use std::task::Poll; + let mut this = self.project(); + assert!(!*this.done, "ReplyReceiver polled after completion"); + loop { + if let Poll::Ready(result) = this.slot.poll(cx.waker(), &now_ticks) { + *this.done = true; + return Poll::Ready(result.map_err(terminal_error)); + } + let deadline = this.slot.deadline(); + if deadline == NEVER { + return Poll::Pending; + } + if *this.armed != deadline || this.sleep.is_none() { + let wait = Duration::from_nanos(deadline.saturating_sub(now_ticks())); + this.sleep.set(Some(spawned_rt::tasks::sleep(wait))); + *this.armed = deadline; + } + let sleep = this.sleep.as_mut().as_pin_mut().expect("armed above"); + match sleep.poll(cx) { + Poll::Pending => return Poll::Pending, + // The timer was armed for the slot's deadline (which only + // decreases), so the deadline has passed: expire the slot. + // This follows the runtime's timer, so paused test clocks + // keep working. + Poll::Ready(()) => { + this.sleep.set(None); + this.slot.terminate(Terminal::TimedOut); + } + } + } + } +} + +// --------------------------------------------------------------------------- +// Test helper +// --------------------------------------------------------------------------- + +/// A `PendingReply` not tied to an actor turn, and its receiver, for calling +/// a manual `Handler::handle` directly in tests (`spawned_test::reply_pair`). +/// Await the receiver only after driving the turns the reply depends on. +#[doc(hidden)] +pub fn reply_pair(deadline: Deadline) -> (PendingReply<'static, R>, ReplyReceiver) { + let (mut handle, receiver) = reply_slot(deadline); + let reply = PendingReply::new(handle.take(), Registry::detached()); + (reply, receiver) +} diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs new file mode 100644 index 00000000..34d8d2da --- /dev/null +++ b/concurrency/src/reply/registry.rs @@ -0,0 +1,242 @@ +//! Per-instance reply registry (docs/design/deferred-replies.md §4, §5). +//! +//! The registry holds the owner's credit pool, a list of weak references to +//! its deferred tokens' slots, and a phase that classifies reply drops: +//! +//! | phase | a dropped reply obligation | +//! |---|---| +//! | `InTurn` | recorded on the turn; committed at turn end: `Dropped` if the handler returned, `Stopped` if it panicked | +//! | `Running` | `Dropped` at once | +//! | `Exiting` | `Stopped` at once | +//! | `Dead` | no-op: the close already answered it | +//! +//! No rule uses `thread::panicking()`. The phase is one atomic so a turn costs +//! two uncontended atomic stores; the turn list is locked only when a drop +//! lands on it (`IN_TURN_PENDING`). + +use super::slot::{CreditPool, Slot, Terminal}; +use crate::ActorId; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::{Arc, Mutex, OnceLock, Weak}; + +/// Default `Actor::MAX_OUTSTANDING`: deferred credits per actor instance. +pub const DEFAULT_MAX_OUTSTANDING: usize = 1024; + +const RUNNING: u8 = 0; +const IN_TURN: u8 = 1; +const IN_TURN_PENDING: u8 = 2; +const EXITING: u8 = 3; +const DEAD: u8 = 4; + +/// A slot seen without its reply type. +pub(crate) trait Closeable: Send + Sync { + fn close(&self) -> bool; + fn terminate(&self, to: Terminal) -> bool; + fn is_terminal(&self) -> bool; +} + +impl Closeable for Slot { + fn close(&self) -> bool { + Slot::close(self) + } + fn terminate(&self, to: Terminal) -> bool { + Slot::terminate(self, to) + } + fn is_terminal(&self) -> bool { + Slot::is_terminal(self) + } +} + +#[derive(Default)] +struct Lists { + tokens: Vec>, + prune_at: usize, + turn_drops: Vec>, +} + +pub(crate) struct Registry { + owner: ActorId, + pool: Arc, + phase: AtomicU8, + lists: Mutex, +} + +fn lock(lists: &Mutex) -> std::sync::MutexGuard<'_, Lists> { + lists.lock().unwrap_or_else(|p| p.into_inner()) +} + +impl Registry { + pub(crate) fn new(owner: ActorId, max_outstanding: usize) -> Arc { + let max = max_outstanding.min(super::slot::MAX_JOBS as usize); + Arc::new(Self { + owner, + pool: Arc::new(CreditPool::new(max)), + phase: AtomicU8::new(RUNNING), + lists: Mutex::new(Lists { + prune_at: 16, + ..Lists::default() + }), + }) + } + + /// The registry of replies not tied to an actor (`reply_pair`): always + /// between turns, with the largest bound. + pub(crate) fn detached() -> &'static Arc { + static DETACHED: OnceLock> = OnceLock::new(); + DETACHED.get_or_init(|| Self::new(ActorId::next(), usize::MAX)) + } + + pub(crate) fn pool(&self) -> &Arc { + &self.pool + } + + pub(crate) fn owner(&self) -> ActorId { + self.owner + } + + /// Credits in use: live deferred tokens and running linked jobs. + pub(crate) fn outstanding(&self) -> usize { + self.pool.used() + } + + /// List a newly deferred token. Called by `defer()` after its CAS, in the + /// owner's turn, so it never races the close. + pub(crate) fn register(&self, slot: &Arc>) { + let weak: Weak = Arc::downgrade(slot) as Weak; + let mut lists = lock(&self.lists); + if lists.tokens.len() >= lists.prune_at { + lists + .tokens + .retain(|t| t.upgrade().is_some_and(|s| !s.is_terminal())); + lists.prune_at = (lists.tokens.len() * 2).max(16); + } + lists.tokens.push(weak); + } + + /// Placeholder for the deadline heap (step 5); deferred tokens are + /// otherwise expired by their polling receiver or a reply. + pub(crate) fn track_deadline(&self, _slot: &Arc>) {} + + /// A reply obligation (`PendingReply` or `ReplyTo`) was dropped unanswered. + pub(crate) fn dropped(&self, slot: Arc>) { + if slot.is_terminal() { + return; + } + let slot: Arc = slot; + loop { + match self.phase.load(Ordering::Acquire) { + RUNNING => { + slot.terminate(Terminal::Dropped); + return; + } + EXITING => { + slot.terminate(Terminal::Stopped); + return; + } + DEAD => { + slot.terminate(Terminal::Stopped); + return; + } + _ => { + let mut lists = lock(&self.lists); + lists.turn_drops.push(slot.clone()); + let mut current = self.phase.load(Ordering::Acquire); + let recorded = loop { + if current != IN_TURN && current != IN_TURN_PENDING { + break false; + } + match self.phase.compare_exchange( + current, + IN_TURN_PENDING, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => break true, + Err(actual) => current = actual, + } + }; + if recorded { + return; + } + // The turn ended first: classify by the new phase. + lists.turn_drops.pop(); + } + } + } + } + + fn commit(&self, outcome: Terminal) { + let drops = std::mem::take(&mut lock(&self.lists).turn_drops); + for slot in drops { + slot.terminate(outcome); + } + } + + /// The run loop is about to dispatch a handler. + pub(crate) fn begin_turn(&self) { + let _ = self + .phase + .compare_exchange(RUNNING, IN_TURN, Ordering::AcqRel, Ordering::Acquire); + } + + /// The handler returned (`panicked == false`) or unwound. + pub(crate) fn end_turn(&self, panicked: bool) { + let previous = self + .phase + .compare_exchange(IN_TURN, RUNNING, Ordering::AcqRel, Ordering::Acquire) + .or_else(|_| { + self.phase.compare_exchange( + IN_TURN_PENDING, + RUNNING, + Ordering::AcqRel, + Ordering::Acquire, + ) + }); + if previous == Ok(IN_TURN_PENDING) { + self.commit(if panicked { + Terminal::Stopped + } else { + Terminal::Dropped + }); + } + } + + /// Exit begins: later drops answer `Stopped`, and the current turn's + /// recorded drops commit as `Stopped`. + pub(crate) fn exiting(&self) { + let previous = self.phase.load(Ordering::Acquire); + if previous == DEAD { + return; + } + if self.phase.swap(EXITING, Ordering::AcqRel) == IN_TURN_PENDING { + self.commit(Terminal::Stopped); + } + } + + pub(crate) fn is_exiting(&self) -> bool { + self.phase.load(Ordering::Acquire) >= EXITING + } + + /// Death: every open token listed becomes `STOPPED`, wherever it is. + /// Returns how many were closed. + pub(crate) fn close(&self) -> usize { + if self.phase.swap(DEAD, Ordering::AcqRel) == IN_TURN_PENDING { + self.commit(Terminal::Stopped); + } + let tokens = std::mem::take(&mut lock(&self.lists).tokens); + tokens + .into_iter() + .filter_map(|t| t.upgrade()) + .filter(|s| s.close()) + .count() + } +} + +impl Drop for Registry { + fn drop(&mut self) { + // Every handle to the instance is gone; nothing can still answer. + if self.phase.load(Ordering::Acquire) != DEAD { + self.close(); + } + } +} diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index ac1c3bba..6a7824cc 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -165,7 +165,10 @@ struct Inner { /// Written only by the reply that moved the word to `WRITING`, and /// published by `WRITING -> REPLIED` under this lock. value: Option, + /// The caller's receiver. waker: Option, + /// The token holder waiting in `ReplyTo::closed()`. + watcher: Option, /// The owner's pool, set by `defer()` before its CAS installs `TOKEN`. pool: Option>, /// Threads parked in `recv_blocking`. @@ -188,6 +191,7 @@ struct Wake { pool: Option>, release: u64, waker: Option, + watcher: Option, notify: bool, } @@ -199,6 +203,9 @@ impl Wake { if let Some(waker) = self.waker { waker.wake(); } + if let Some(watcher) = self.watcher { + watcher.wake(); + } if self.notify { slot_parked.notify_all(); } @@ -214,6 +221,7 @@ impl Slot { inner: Mutex::new(Inner { value: None, waker: None, + watcher: None, pool: None, parked: 0, }), @@ -278,6 +286,7 @@ impl Slot { pool: None, release: 0, waker: None, + watcher: None, notify: false, }; } @@ -286,6 +295,7 @@ impl Slot { pool: if release > 0 { inner.pool.clone() } else { None }, release, waker: if terminal { inner.waker.take() } else { None }, + watcher: if terminal { inner.watcher.take() } else { None }, notify: terminal && inner.parked > 0, } } @@ -454,6 +464,7 @@ impl Slot { pool: if release > 0 { inner.pool.clone() } else { None }, release, waker: inner.waker.take(), + watcher: inner.watcher.take(), notify: inner.parked > 0, } }; @@ -502,6 +513,25 @@ impl Slot { } } + /// Token side: ready once the slot is terminal (or its deadline passed, + /// which expires it). Registers `waker` as the watcher otherwise. + pub(crate) fn poll_closed(&self, waker: &Waker, now: &impl Fn() -> u64) -> Poll<()> { + if !live(state(self.word())) || self.check_deadline(now) { + return Poll::Ready(()); + } + let mut inner = lock(&self.inner); + if !live(state(self.word())) { + return Poll::Ready(()); + } + let old = match &inner.watcher { + Some(old) if old.will_wake(waker) => None, + _ => inner.watcher.replace(waker.clone()), + }; + drop(inner); + drop(old); + Poll::Pending + } + /// Non-blocking check from the caller's receiver. pub(crate) fn try_take(&self, now: &impl Fn() -> u64) -> Option> { if let Some(result) = self.take(state(self.word())) { diff --git a/concurrency/src/response.rs b/concurrency/src/response.rs index d235d0cc..2d6f9d56 100644 --- a/concurrency/src/response.rs +++ b/concurrency/src/response.rs @@ -1,6 +1,6 @@ use crate::error::ActorError; #[cfg(feature = "async")] -use spawned_rt::tasks::oneshot; +use crate::reply::ReplyReceiver; use std::future::Future; use std::pin::Pin; use std::time::Duration; @@ -38,10 +38,9 @@ impl RequestOptions { // --------------------------------------------------------------------------- enum ResponseState { + /// The deadline is already in the receiver's slot, fixed at send. #[cfg(feature = "async")] - Receiver(oneshot::Receiver), - #[cfg(feature = "async")] - TimedReceiver(Pin> + Send>>), + Receiver(Pin>>), Ready(Result), Done, } @@ -49,7 +48,9 @@ enum ResponseState { /// Concrete wrapper for protocol request-response methods that works in both /// `tasks` (async) and `threads` (blocking) modes. /// -/// - **Tasks mode**: wraps a oneshot receiver; `.await` returns `Result` +/// - **Tasks mode**: wraps a [`ReplyReceiver`](crate::reply::ReplyReceiver); +/// `.await` returns `Result`. The request's deadline was fixed +/// when it was sent. /// - **Threads mode**: wraps a pre-computed result; use `.unwrap()` / `.expect()` directly /// /// Protocol methods return `Response`: @@ -67,9 +68,8 @@ pub struct Response( #[cfg(feature = "metrics")] Option, ); -// Safety: tokio::sync::oneshot::Receiver implements Unpin, so pinning -// Response is safe. This is required for the Future::poll implementation -// which uses self.get_mut(). +// The receiver is boxed and pinned, so `Response` is `Unpin` whatever `T` +// is; `Future::poll` uses `self.get_mut()`. impl Unpin for Response {} impl Response { @@ -93,7 +93,7 @@ impl Response { match self.0 { ResponseState::Ready(result) => result.unwrap(), #[cfg(feature = "async")] - ResponseState::Receiver(_) | ResponseState::TimedReceiver(_) => { + ResponseState::Receiver(_) => { panic!("called unwrap() on a pending Response; use .await in async contexts") } ResponseState::Done => panic!("Response already consumed"), @@ -105,7 +105,7 @@ impl Response { match self.0 { ResponseState::Ready(result) => result.expect(msg), #[cfg(feature = "async")] - ResponseState::Receiver(_) | ResponseState::TimedReceiver(_) => { + ResponseState::Receiver(_) => { panic!("{msg}: called expect() on a pending Response; use .await in async contexts") } ResponseState::Done => panic!("{msg}: Response already consumed"), @@ -137,7 +137,7 @@ impl Response { None, ), #[cfg(feature = "async")] - ResponseState::Receiver(_) | ResponseState::TimedReceiver(_) => { + ResponseState::Receiver(_) => { panic!("called map() on a pending Response; use .await in async contexts") } ResponseState::Done => panic!("Response already consumed"), @@ -146,54 +146,13 @@ impl Response { } #[cfg(feature = "async")] -impl Response { - /// Create a `Response` from a oneshot receiver with a timeout. - /// - /// Used by tasks-mode protocol blanket impls. Returns - /// `Err(ActorError::RequestTimeout)` if the timeout expires. - pub fn from_with_timeout( - result: Result, ActorError>, - duration: Duration, - ) -> Self { - #[cfg(feature = "metrics")] - let observation = crate::instrumentation::RequestObservation::new(); - match result { - Ok(rx) => { - let fut = Box::pin(async move { - match spawned_rt::tasks::timeout(duration, rx).await { - Ok(Ok(val)) => Ok(val), - Ok(Err(_)) => Err(ActorError::ActorStopped), - Err(_) => Err(ActorError::RequestTimeout), - } - }); - Self( - ResponseState::TimedReceiver(fut), - #[cfg(feature = "metrics")] - Some(observation), - ) - } - Err(e) => { - let result = Err(e); - #[cfg(feature = "metrics")] - observation.finish(&result); - Self( - ResponseState::Ready(result), - #[cfg(feature = "metrics")] - None, - ) - } - } - } -} - -#[cfg(feature = "async")] -impl From, ActorError>> for Response { - fn from(result: Result, ActorError>) -> Self { +impl From, ActorError>> for Response { + fn from(result: Result, ActorError>) -> Self { #[cfg(feature = "metrics")] let observation = crate::instrumentation::RequestObservation::new(); match result { Ok(rx) => Self( - ResponseState::Receiver(rx), + ResponseState::Receiver(Box::pin(rx)), #[cfg(feature = "metrics")] Some(observation), ), @@ -221,19 +180,7 @@ impl Future for Response { let this = self.get_mut(); let result = match &mut this.0 { #[cfg(feature = "async")] - ResponseState::Receiver(rx) => match Pin::new(rx).poll(_cx) { - std::task::Poll::Ready(Ok(val)) => { - this.0 = ResponseState::Done; - std::task::Poll::Ready(Ok(val)) - } - std::task::Poll::Ready(Err(_)) => { - this.0 = ResponseState::Done; - std::task::Poll::Ready(Err(ActorError::ActorStopped)) - } - std::task::Poll::Pending => std::task::Poll::Pending, - }, - #[cfg(feature = "async")] - ResponseState::TimedReceiver(fut) => match fut.as_mut().poll(_cx) { + ResponseState::Receiver(rx) => match rx.as_mut().poll(_cx) { std::task::Poll::Ready(result) => { this.0 = ResponseState::Done; std::task::Poll::Ready(result) @@ -260,7 +207,7 @@ impl Future for Response { mod tests { use super::*; #[cfg(feature = "async")] - use spawned_rt::tasks::oneshot; + use crate::reply::Deadline; #[test] fn ready_ok_unwrap() { @@ -286,24 +233,22 @@ mod tests { fn future_resolves_from_receiver() { let rt = spawned_rt::tasks::Runtime::new().unwrap(); rt.block_on(async { - let (tx, rx) = oneshot::channel::(); + let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); let resp: Response = Response::from(Ok(rx)); - tx.send(99).unwrap(); - let val = resp.await.unwrap(); - assert_eq!(val, 99); + reply.reply(99); + assert_eq!(resp.await.unwrap(), 99); }); } #[test] #[cfg(feature = "async")] - fn future_err_on_dropped_sender() { + fn future_err_on_dropped_reply() { let rt = spawned_rt::tasks::Runtime::new().unwrap(); rt.block_on(async { - let (tx, rx) = oneshot::channel::(); + let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); let resp: Response = Response::from(Ok(rx)); - drop(tx); - let result = resp.await; - assert!(matches!(result, Err(ActorError::ActorStopped))); + drop(reply); + assert!(matches!(resp.await, Err(ActorError::ReplyDropped))); }); } @@ -316,25 +261,14 @@ mod tests { #[test] #[cfg(feature = "async")] - fn timed_receiver_resolves() { + fn receiver_times_out_at_its_send_deadline() { let rt = spawned_rt::tasks::Runtime::new().unwrap(); rt.block_on(async { - let (tx, rx) = oneshot::channel::(); - let resp = Response::from_with_timeout(Ok(rx), Duration::from_secs(5)); - tx.send(42).unwrap(); - assert_eq!(resp.await.unwrap(), 42); - }); - } - - #[test] - #[cfg(feature = "async")] - fn timed_receiver_times_out() { - let rt = spawned_rt::tasks::Runtime::new().unwrap(); - rt.block_on(async { - let (_tx, rx) = oneshot::channel::(); - let resp = Response::from_with_timeout(Ok(rx), Duration::from_millis(50)); - let result = resp.await; - assert!(matches!(result, Err(ActorError::RequestTimeout))); + let (reply, rx) = + crate::reply::reply_pair::(Deadline::after(Duration::from_millis(50))); + let resp: Response = Response::from(Ok(rx)); + assert!(matches!(resp.await, Err(ActorError::RequestTimeout))); + assert_eq!(reply.reply(1), crate::reply::Delivery::Expired); }); } } diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index a7fb0ec2..a585a757 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -7,10 +7,14 @@ use crate::link::{ use crate::mailbox::tasks as mpsc; use crate::message::Message; use crate::monitor::{Down, MonitorCleanup, MonitorRef, MonitorTable}; +use crate::reply::{registry::Registry, reply_slot, Deadline, SlotHandle}; +pub use crate::reply::{ + DeferError, Delivery, PendingReply, ReplyReceiver, ReplyTo, DEFAULT_MAX_OUTSTANDING, +}; use core::pin::pin; use futures::future::{self, FutureExt as _}; use spawned_rt::{ - tasks::{self as rt, oneshot, timeout, watch, CancellationToken, JoinHandle}, + tasks::{self as rt, watch, CancellationToken, JoinHandle}, threads, }; use std::sync::atomic::Ordering; @@ -61,6 +65,11 @@ pub enum Backend { /// /// Actors must be `Send + Sized + 'static` so they can be moved to a spawned task. pub trait Actor: Send + Sized + 'static { + /// Deferred-reply credits per instance (live `ReplyTo` tokens plus running + /// linked jobs). At the bound `defer()` answers the caller `Overloaded`. + /// Capped at 65535. + const MAX_OUTSTANDING: usize = DEFAULT_MAX_OUTSTANDING; + fn started(&mut self, _ctx: &Context) -> impl Future + Send { async {} } @@ -86,10 +95,32 @@ pub trait Actor: Send + Sized + 'static { /// Per-message handler trait. Implement once for each message type the actor handles. /// +/// `handle` is the envelope dispatch for both `send` and `request`. Answer +/// through `reply`: [`PendingReply::reply`] now, or [`PendingReply::defer`] to +/// answer from a later turn, a job or `stopped()`. A request whose +/// `PendingReply` is dropped unanswered gets `ReplyDropped`. A sent message +/// has a detached `PendingReply` (no caller). +/// +/// `#[actor]` generates this from `#[request_handler]`, `#[send_handler]` and +/// `#[deferred_handler]` methods. Manual impls: +/// +/// ```ignore +/// impl Handler for Counter { +/// async fn handle(&mut self, _: GetCount, reply: PendingReply<'_, u64>, _: &Context) { +/// reply.reply(self.count); +/// } +/// } +/// ``` +/// /// Uses RPITIT (return-position `impl Trait` in traits), which means this trait /// is **not** object-safe. For type-erased references, use [`Receiver`] / [`Recipient`]. pub trait Handler: Actor { - fn handle(&mut self, msg: M, ctx: &Context) -> impl Future + Send; + fn handle( + &mut self, + msg: M, + reply: PendingReply<'_, M::Result>, + ctx: &Context, + ) -> impl Future + Send; } // --------------------------------------------------------------------------- @@ -117,7 +148,7 @@ trait Envelope: Send { struct MessageEnvelope { msg: M, - tx: Option>, + slot: SlotHandle, #[cfg(feature = "metrics")] observation: Option, } @@ -146,16 +177,9 @@ where actor: &'a mut A, ctx: &'a Context, ) -> Pin + Send + 'a>> { - Box::pin(async move { - let result = actor.handle(self.msg, ctx).await; - if let Some(tx) = self.tx { - let delivered = tx.send(result).is_ok(); - #[cfg(feature = "metrics")] - crate::instrumentation::reply(delivered); - #[cfg(not(feature = "metrics"))] - let _ = delivered; - } - }) + let mut this = *self; + let reply = PendingReply::new(this.slot.take(), &ctx.shared.replies); + Box::pin(actor.handle(this.msg, reply, ctx)) } } @@ -163,7 +187,7 @@ fn enqueue( sender: &mpsc::Sender + Send>>, _mailbox: &crate::instrumentation::Mailbox, msg: M, - tx: Option>, + slot: SlotHandle, ) -> Result<(), ActorError> where A: Actor + Handler, @@ -171,7 +195,7 @@ where { let envelope: Box + Send> = Box::new(MessageEnvelope { msg, - tx, + slot, #[cfg(feature = "metrics")] observation: Some(_mailbox.enqueued()), }); @@ -266,6 +290,8 @@ struct ActorShared { linked_reason: LinkedExitReason, mailbox: crate::instrumentation::Mailbox, managed: Option>, + /// This instance's reply registry: deferred tokens, credits, turn phase. + replies: Arc, } impl Clone for Context { @@ -328,17 +354,38 @@ impl Context { A: Handler, M: Message, { - enqueue(&self.shared.sender, &self.shared.mailbox, msg, None) + enqueue( + &self.shared.sender, + &self.shared.mailbox, + msg, + SlotHandle::detached(), + ) + } + + /// Send a request and get its [`ReplyReceiver`], with the default + /// deadline ([`DEFAULT_REQUEST_TIMEOUT`] after the send, as + /// `gen_server:call/2`). + pub fn request_raw(&self, msg: M) -> Result, ActorError> + where + A: Handler, + M: Message, + { + self.request_raw_until(msg, Deadline::after(DEFAULT_REQUEST_TIMEOUT)) } - /// Send a request and get a raw oneshot receiver for the reply. - pub fn request_raw(&self, msg: M) -> Result, ActorError> + /// Send a request whose deadline is `deadline` ([`Deadline::Never`] for + /// an unbounded wait). + pub fn request_raw_until( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> where A: Handler, M: Message, { - let (tx, rx) = oneshot::channel(); - enqueue(&self.shared.sender, &self.shared.mailbox, msg, Some(tx))?; + let (slot, rx) = reply_slot(deadline); + enqueue(&self.shared.sender, &self.shared.mailbox, msg, slot)?; Ok(rx) } @@ -363,13 +410,9 @@ impl Context { M: Message, { let observation = crate::instrumentation::RequestObservation::new(); - let result = match self.request_raw(msg) { + let result = match self.request_raw_until(msg, Deadline::after(duration)) { Err(error) => Err(error), - Ok(rx) => match timeout(duration, rx).await { - Ok(Ok(result)) => Ok(result), - Ok(Err(_)) => Err(ActorError::ActorStopped), - Err(_) => Err(ActorError::RequestTimeout), - }, + Ok(rx) => rx.await, }; observation.finish(&result); result @@ -653,8 +696,12 @@ where Context::send(self, msg) } - fn request_raw(&self, msg: M) -> Result, ActorError> { - Context::request_raw(self, msg) + fn request_raw_until( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + Context::request_raw_until(self, msg, deadline) } } @@ -668,7 +715,17 @@ where /// message type that `A` handles. pub trait Receiver: Send + Sync { fn send(&self, msg: M) -> Result<(), ActorError>; - fn request_raw(&self, msg: M) -> Result, ActorError>; + /// Send a request whose deadline is fixed now (the slot carries it). + fn request_raw_until( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError>; + /// Send a request with the default deadline + /// ([`DEFAULT_REQUEST_TIMEOUT`] after the send). + fn request_raw(&self, msg: M) -> Result, ActorError> { + self.request_raw_until(msg, Deadline::after(DEFAULT_REQUEST_TIMEOUT)) + } } /// Type-erased reference for sending a single message type. @@ -683,13 +740,10 @@ pub async fn request( timeout_duration: Duration, ) -> Result { let observation = crate::instrumentation::RequestObservation::new(); - let result = match recipient.request_raw(msg) { + // The deadline is computed before enqueueing (send time, as OTP's call). + let result = match recipient.request_raw_until(msg, Deadline::after(timeout_duration)) { Err(error) => Err(error), - Ok(rx) => match timeout(timeout_duration, rx).await { - Ok(Ok(result)) => Ok(result), - Ok(Err(_)) => Err(ActorError::ActorStopped), - Err(_) => Err(ActorError::RequestTimeout), - }, + Ok(rx) => rx.await, }; observation.finish(&result); result @@ -749,17 +803,38 @@ impl ActorRef { A: Handler, M: Message, { - enqueue(&self.shared.sender, &self.shared.mailbox, msg, None) + enqueue( + &self.shared.sender, + &self.shared.mailbox, + msg, + SlotHandle::detached(), + ) + } + + /// Send a request and get its [`ReplyReceiver`], with the default + /// deadline ([`DEFAULT_REQUEST_TIMEOUT`] after the send, as + /// `gen_server:call/2`). + pub fn request_raw(&self, msg: M) -> Result, ActorError> + where + A: Handler, + M: Message, + { + self.request_raw_until(msg, Deadline::after(DEFAULT_REQUEST_TIMEOUT)) } - /// Send a request and get a raw oneshot receiver for the reply. - pub fn request_raw(&self, msg: M) -> Result, ActorError> + /// Send a request whose deadline is `deadline` ([`Deadline::Never`] for + /// an unbounded wait). + pub fn request_raw_until( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> where A: Handler, M: Message, { - let (tx, rx) = oneshot::channel(); - enqueue(&self.shared.sender, &self.shared.mailbox, msg, Some(tx))?; + let (slot, rx) = reply_slot(deadline); + enqueue(&self.shared.sender, &self.shared.mailbox, msg, slot)?; Ok(rx) } @@ -784,13 +859,9 @@ impl ActorRef { M: Message, { let observation = crate::instrumentation::RequestObservation::new(); - let result = match self.request_raw(msg) { + let result = match self.request_raw_until(msg, Deadline::after(duration)) { Err(error) => Err(error), - Ok(rx) => match timeout(duration, rx).await { - Ok(Ok(result)) => Ok(result), - Ok(Err(_)) => Err(ActorError::ActorStopped), - Err(_) => Err(ActorError::RequestTimeout), - }, + Ok(rx) => rx.await, }; observation.finish(&result); result @@ -901,8 +972,12 @@ where ActorRef::send(self, msg) } - fn request_raw(&self, msg: M) -> Result, ActorError> { - ActorRef::request_raw(self, msg) + fn request_raw_until( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + ActorRef::request_raw_until(self, msg, deadline) } } @@ -948,9 +1023,11 @@ impl ActorRef { .map(|s| s.cause.clone()) .unwrap_or_else(new_linked_exit_reason); + let id = ActorId::next(); let actor_ref = ActorRef { shared: Arc::new(ActorShared { - id: ActorId::next(), + id, + replies: Registry::new(id, A::MAX_OUTSTANDING), sender: tx, cancellation_token: cancellation_token.clone(), completion_rx, @@ -1186,6 +1263,7 @@ async fn run_actor( cancellation_token: CancellationToken, #[cfg(feature = "backend-experiments")] turn: Option, ) -> Result { + let replies = ctx.shared.replies.clone(); let primary = match AssertUnwindSafe(run_actor_lifecycle( &mut actor, ctx, @@ -1206,6 +1284,11 @@ async fn run_actor( ))) } }; + // Death (deferred-replies §5 step 3), only now that `stopped()` returned: + // close the registry (every open token answers ActorStopped), dispose + // the mailbox (queued requests answer ActorStopped), drop the state. + replies.close(); + drop(rx); crate::actor_cleanup::finish_actor_state(actor, primary) } @@ -1249,6 +1332,7 @@ async fn run_actor_lifecycle( if let Some(scope) = &ctx.shared.managed { scope.end_running(); } + ctx.shared.replies.exiting(); if let Err(panic) = AssertUnwindSafe(async { fail_point!("actor.before_stopped", async); actor.stopped(&ctx).await @@ -1308,9 +1392,11 @@ async fn run_actor_lifecycle( let observation = envelope .observation() .map(|observation| observation.begin()); + ctx.shared.replies.begin_turn(); let result = AssertUnwindSafe(envelope.handle(actor, &ctx)) .catch_unwind() .await; + ctx.shared.replies.end_turn(result.is_err()); #[cfg(feature = "diagnostics")] drop(_turn); #[cfg(feature = "metrics")] @@ -1343,6 +1429,7 @@ async fn run_actor_lifecycle( scope.end_running(); } cancellation_token.cancel(); + ctx.shared.replies.exiting(); let stop_result = AssertUnwindSafe(async { fail_point!("actor.before_stopped", async); actor.stopped(&ctx).await diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 0a07cf9f..11a1ad1a 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -1,5 +1,5 @@ use crate::mailbox::threads as mpsc; -use spawned_rt::threads::{self as rt, oneshot, oneshot::RecvTimeoutError, CancellationToken}; +use spawned_rt::threads::{self as rt, CancellationToken}; use std::{ collections::HashMap, fmt::Debug, @@ -16,6 +16,10 @@ use crate::link::{ }; use crate::message::Message; use crate::monitor::{Down, MonitorCleanup, MonitorRef, MonitorTable}; +use crate::reply::{registry::Registry, reply_slot, Deadline, SlotHandle}; +pub use crate::reply::{ + DeferError, Delivery, PendingReply, ReplyReceiver, ReplyTo, DEFAULT_MAX_OUTSTANDING, +}; use std::sync::atomic::Ordering; pub use crate::response::DEFAULT_REQUEST_TIMEOUT; @@ -31,6 +35,11 @@ pub use crate::response::DEFAULT_REQUEST_TIMEOUT; /// /// Actors must be `Send + Sized + 'static` so they can be moved to a spawned thread. pub trait Actor: Send + Sized + 'static { + /// Deferred-reply credits per instance (live `ReplyTo` tokens plus running + /// linked jobs). At the bound `defer()` answers the caller `Overloaded`. + /// Capped at 65535. + const MAX_OUTSTANDING: usize = DEFAULT_MAX_OUTSTANDING; + fn started(&mut self, _ctx: &Context) {} fn stopped(&mut self, _ctx: &Context) {} @@ -46,8 +55,13 @@ pub trait Actor: Send + Sized + 'static { /// Per-message handler trait. Implement once for each message type the actor handles. /// /// Unlike the `tasks` version, handlers are synchronous — no `async`/`.await`. +/// +/// `handle` is the envelope dispatch for both `send` and `request`: answer +/// through `reply` ([`PendingReply::reply`], or [`PendingReply::defer`] to +/// answer later). A request whose `PendingReply` is dropped unanswered gets +/// `ReplyDropped`; a sent message has a detached `PendingReply`. pub trait Handler: Actor { - fn handle(&mut self, msg: M, ctx: &Context) -> M::Result; + fn handle(&mut self, msg: M, reply: PendingReply<'_, M::Result>, ctx: &Context); } // --------------------------------------------------------------------------- @@ -67,7 +81,7 @@ trait Envelope: Send { struct MessageEnvelope { msg: M, - tx: Option>, + slot: SlotHandle, #[cfg(feature = "metrics")] observation: Option, } @@ -88,14 +102,9 @@ where } } fn handle(self: Box, actor: &mut A, ctx: &Context) { - let result = actor.handle(self.msg, ctx); - if let Some(tx) = self.tx { - let delivered = tx.send(result).is_ok(); - #[cfg(feature = "metrics")] - crate::instrumentation::reply(delivered); - #[cfg(not(feature = "metrics"))] - let _ = delivered; - } + let mut this = *self; + let reply = PendingReply::new(this.slot.take(), &ctx.replies); + actor.handle(this.msg, reply, ctx); } } @@ -103,7 +112,7 @@ fn enqueue( sender: &mpsc::Sender>, _mailbox: &crate::instrumentation::Mailbox, msg: M, - tx: Option>, + slot: SlotHandle, ) -> Result<(), ActorError> where A: Actor + Handler, @@ -111,7 +120,7 @@ where { let envelope: Box + Send> = Box::new(MessageEnvelope { msg, - tx, + slot, #[cfg(feature = "metrics")] observation: Some(_mailbox.enqueued()), }); @@ -178,6 +187,7 @@ pub struct Context { links: LinkTable, linked_reason: LinkedExitReason, mailbox: crate::instrumentation::Mailbox, + replies: Arc, } impl Clone for Context { @@ -192,6 +202,7 @@ impl Clone for Context { links: self.links.clone(), linked_reason: self.linked_reason.clone(), mailbox: self.mailbox.clone(), + replies: self.replies.clone(), } } } @@ -216,6 +227,7 @@ impl Context { links: actor_ref.links.clone(), linked_reason: actor_ref.linked_reason.clone(), mailbox: actor_ref.mailbox.clone(), + replies: actor_ref.replies.clone(), } } @@ -256,17 +268,32 @@ impl Context { A: Handler, M: Message, { - enqueue(&self.sender, &self.mailbox, msg, None) + enqueue(&self.sender, &self.mailbox, msg, SlotHandle::detached()) + } + + /// Send a request and get its [`ReplyReceiver`], with the default + /// deadline ([`DEFAULT_REQUEST_TIMEOUT`] after the send). + pub fn request_raw(&self, msg: M) -> Result, ActorError> + where + A: Handler, + M: Message, + { + self.request_raw_until(msg, Deadline::after(DEFAULT_REQUEST_TIMEOUT)) } - /// Send a request and get a raw oneshot receiver for the reply. - pub fn request_raw(&self, msg: M) -> Result, ActorError> + /// Send a request whose deadline is `deadline` ([`Deadline::Never`] for + /// an unbounded wait). + pub fn request_raw_until( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> where A: Handler, M: Message, { - let (tx, rx) = oneshot::channel(); - enqueue(&self.sender, &self.mailbox, msg, Some(tx))?; + let (slot, rx) = reply_slot(deadline); + enqueue(&self.sender, &self.mailbox, msg, slot)?; Ok(rx) } @@ -290,13 +317,9 @@ impl Context { M: Message, { let observation = crate::instrumentation::RequestObservation::new(); - let result = match self.request_raw(msg) { + let result = match self.request_raw_until(msg, Deadline::after(duration)) { Err(error) => Err(error), - Ok(rx) => match rx.recv_timeout(duration) { - Ok(result) => Ok(result), - Err(RecvTimeoutError::Timeout) => Err(ActorError::RequestTimeout), - Err(RecvTimeoutError::Disconnected) => Err(ActorError::ActorStopped), - }, + Ok(rx) => rx.recv(), }; observation.finish(&result); result @@ -324,6 +347,7 @@ impl Context { links: self.links.clone(), linked_reason: self.linked_reason.clone(), mailbox: self.mailbox.clone(), + replies: self.replies.clone(), } } @@ -515,8 +539,12 @@ where Context::send(self, msg) } - fn request_raw(&self, msg: M) -> Result, ActorError> { - Context::request_raw(self, msg) + fn request_raw_until( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + Context::request_raw_until(self, msg, deadline) } } @@ -530,7 +558,17 @@ where /// message type that `A` handles. pub trait Receiver: Send + Sync { fn send(&self, msg: M) -> Result<(), ActorError>; - fn request_raw(&self, msg: M) -> Result, ActorError>; + /// Send a request whose deadline is fixed now (the slot carries it). + fn request_raw_until( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError>; + /// Send a request with the default deadline + /// ([`DEFAULT_REQUEST_TIMEOUT`] after the send). + fn request_raw(&self, msg: M) -> Result, ActorError> { + self.request_raw_until(msg, Deadline::after(DEFAULT_REQUEST_TIMEOUT)) + } } /// Type-erased reference for sending a single message type. @@ -543,13 +581,9 @@ pub fn request( timeout: Duration, ) -> Result { let observation = crate::instrumentation::RequestObservation::new(); - let result = match recipient.request_raw(msg) { + let result = match recipient.request_raw_until(msg, Deadline::after(timeout)) { Err(error) => Err(error), - Ok(rx) => match rx.recv_timeout(timeout) { - Ok(result) => Ok(result), - Err(RecvTimeoutError::Timeout) => Err(ActorError::RequestTimeout), - Err(RecvTimeoutError::Disconnected) => Err(ActorError::ActorStopped), - }, + Ok(rx) => rx.recv(), }; observation.finish(&result); result @@ -574,6 +608,7 @@ pub struct ActorRef { links: LinkTable, linked_reason: LinkedExitReason, mailbox: crate::instrumentation::Mailbox, + replies: Arc, } impl Debug for ActorRef { @@ -594,6 +629,7 @@ impl Clone for ActorRef { links: self.links.clone(), linked_reason: self.linked_reason.clone(), mailbox: self.mailbox.clone(), + replies: self.replies.clone(), } } } @@ -612,17 +648,32 @@ impl ActorRef { A: Handler, M: Message, { - enqueue(&self.sender, &self.mailbox, msg, None) + enqueue(&self.sender, &self.mailbox, msg, SlotHandle::detached()) } - /// Send a request and get a raw oneshot receiver for the reply. - pub fn request_raw(&self, msg: M) -> Result, ActorError> + /// Send a request and get its [`ReplyReceiver`], with the default + /// deadline ([`DEFAULT_REQUEST_TIMEOUT`] after the send). + pub fn request_raw(&self, msg: M) -> Result, ActorError> where A: Handler, M: Message, { - let (tx, rx) = oneshot::channel(); - enqueue(&self.sender, &self.mailbox, msg, Some(tx))?; + self.request_raw_until(msg, Deadline::after(DEFAULT_REQUEST_TIMEOUT)) + } + + /// Send a request whose deadline is `deadline` ([`Deadline::Never`] for + /// an unbounded wait). + pub fn request_raw_until( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> + where + A: Handler, + M: Message, + { + let (slot, rx) = reply_slot(deadline); + enqueue(&self.sender, &self.mailbox, msg, slot)?; Ok(rx) } @@ -646,13 +697,9 @@ impl ActorRef { M: Message, { let observation = crate::instrumentation::RequestObservation::new(); - let result = match self.request_raw(msg) { + let result = match self.request_raw_until(msg, Deadline::after(duration)) { Err(error) => Err(error), - Ok(rx) => match rx.recv_timeout(duration) { - Ok(result) => Ok(result), - Err(RecvTimeoutError::Timeout) => Err(ActorError::RequestTimeout), - Err(RecvTimeoutError::Disconnected) => Err(ActorError::ActorStopped), - }, + Ok(rx) => rx.recv(), }; observation.finish(&result); result @@ -741,8 +788,12 @@ where ActorRef::send(self, msg) } - fn request_raw(&self, msg: M) -> Result, ActorError> { - ActorRef::request_raw(self, msg) + fn request_raw_until( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + ActorRef::request_raw_until(self, msg, deadline) } } @@ -772,6 +823,7 @@ impl ActorRef { links: links.clone(), linked_reason: linked_reason.clone(), mailbox: crate::instrumentation::Mailbox::default(), + replies: Registry::new(id, A::MAX_OUTSTANDING), }; let ctx = Context { @@ -784,6 +836,7 @@ impl ActorRef { links: links.clone(), linked_reason: linked_reason.clone(), mailbox: actor_ref.mailbox.clone(), + replies: actor_ref.replies.clone(), }; if let Some(parent) = parent { @@ -803,6 +856,7 @@ impl ActorRef { .with_mailbox_failures(actor_ref.sender.failures()); let startup = crate::actor_cleanup::ActorStartup::new((actor, ctx, rx, cancellation_token), cleanup); + let replies = actor_ref.replies.clone(); let _thread_handle = rt::spawn(move || { let ((mut actor, ctx, mut rx, cancellation_token), mut cleanup) = startup.into_parts(); let primary = match catch_unwind(AssertUnwindSafe(|| { @@ -817,10 +871,14 @@ impl ActorRef { ))) } }; - let outcome = crate::actor_cleanup::finish_actor_state(actor, primary); - // State destructors may still send through a retained context. Keep - // the receiver alive until state is gone, then drain before publish. + // Death (deferred-replies §5 step 3), only now that `stopped()` + // returned: close the registry (every open token answers + // ActorStopped), dispose the mailbox (queued requests answer + // ActorStopped; later sends from state destructors fail), then drop + // the state, all before the exit is published. + replies.close(); drop(rx); + let outcome = crate::actor_cleanup::finish_actor_state(actor, primary); match outcome { Ok(reason) => cleanup.complete(reason), Err(reason) => cleanup.complete_secondary(reason), @@ -870,6 +928,7 @@ fn run_actor_receiving( } if cancellation_token.is_cancelled() { + ctx.replies.exiting(); if let Err(panic) = catch_unwind(AssertUnwindSafe(|| { fail_point!("actor.before_stopped"); actor.stopped(&ctx) @@ -912,9 +971,11 @@ fn run_actor_receiving( let observation = envelope .observation() .map(|observation| observation.begin()); + ctx.replies.begin_turn(); let result = catch_unwind(AssertUnwindSafe(|| { envelope.handle(actor, &ctx); })); + ctx.replies.end_turn(result.is_err()); #[cfg(feature = "metrics")] if let Some(observation) = observation { observation.finish(result.is_err()); @@ -936,6 +997,7 @@ fn run_actor_receiving( } cancellation_token.cancel(); + ctx.replies.exiting(); let stop_result = catch_unwind(AssertUnwindSafe(|| { fail_point!("actor.before_stopped"); actor.stopped(&ctx); diff --git a/macros/src/lib.rs b/macros/src/lib.rs index a279aea9..92aa6bb6 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -489,7 +489,9 @@ fn generate_target_impl( let body = match &m.kind { MethodKind::Send => quote! { target.send(message) }, MethodKind::Request(_) => match mode { - RuntimeMode::Tasks => quote! { spawned_concurrency::Response::from_with_timeout(target.request_raw(message), timeout) }, + // The timeout was chosen before the message was built; the + // deadline is fixed now, at send, and travels in the slot. + RuntimeMode::Tasks => quote! { spawned_concurrency::Response::from(target.request_raw_until(message, spawned_concurrency::reply::Deadline::after(timeout))) }, RuntimeMode::Threads => quote! { spawned_concurrency::Response::ready(target.request_with_timeout(message, timeout)) }, }, }; @@ -722,7 +724,9 @@ struct ProtocolMethodInfo { /// supports conversion with the generated `ToXRef` trait. Configure before /// erasing the concrete handle. These are reply-wait limits, not cancellation /// or inherited deadlines; accepted work may still execute after timeout. -/// Native protocol calls block immediately; async response timers start on poll. +/// The deadline is fixed when the request is sent (send time plus the +/// timeout), as in OTP's `gen_server:call`. Native protocol calls block +/// immediately; async responses wait until that deadline when polled. #[proc_macro_attribute] pub fn protocol(attr: TokenStream, item: TokenStream) -> TokenStream { let options = parse_macro_input!(attr with syn::punctuated::Punctuated::::parse_terminated); @@ -1073,7 +1077,10 @@ pub fn protocol(attr: TokenStream, item: TokenStream) -> TokenStream { fn is_handler_attribute(attr: &Attribute) -> bool { let path = attr.path(); - path.is_ident("handler") || path.is_ident("send_handler") || path.is_ident("request_handler") + path.is_ident("handler") + || path.is_ident("send_handler") + || path.is_ident("request_handler") + || path.is_ident("deferred_handler") } fn is_actor_method_attribute(attr: &Attribute) -> bool { @@ -1116,11 +1123,16 @@ fn is_actor_method_attribute(attr: &Attribute) -> bool { /// | `#[request_handler]` | Messages that expect a reply (`Response`) | /// | `#[send_handler]` | Fire-and-forget messages | /// | `#[handler]` | Generic handler (works for either kind) | +/// | `#[deferred_handler]` | Requests answered through a `PendingReply` (now, or later via `defer()`) | /// /// Handler signature: `fn name(&mut self, msg: MessageType, ctx: &Context) -> ReturnType` /// /// The return type must match the message's `Message::Result`. For request handlers -/// returning `()`, omit the return type. +/// returning `()`, omit the return type. The generated `Handler::handle` calls +/// the method and replies with its result; the method stays callable directly. +/// +/// A deferred handler takes the reply obligation instead of returning: +/// `fn name(&mut self, msg: M, reply: PendingReply<'_, R>, ctx: &Context)`. /// /// # Lifecycle Hooks /// @@ -1286,11 +1298,7 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { } if is_started || is_stopped - || method.attrs.iter().any(|a| { - a.path().is_ident("handler") - || a.path().is_ident("send_handler") - || a.path().is_ident("request_handler") - }) + || method.attrs.iter().any(is_handler_attribute) { return syn::Error::new_spanned(&method.sig, "#[exit_received] cannot be combined with other lifecycle or handler attributes").to_compile_error().into(); } @@ -1333,11 +1341,7 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { .to_compile_error() .into(); } - if method.attrs.iter().any(|a| { - a.path().is_ident("handler") - || a.path().is_ident("send_handler") - || a.path().is_ident("request_handler") - }) { + if method.attrs.iter().any(is_handler_attribute) { return syn::Error::new_spanned( &method.sig, "#[started] cannot be combined with handler attributes", @@ -1383,11 +1387,7 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { .to_compile_error() .into(); } - if method.attrs.iter().any(|a| { - a.path().is_ident("handler") - || a.path().is_ident("send_handler") - || a.path().is_ident("request_handler") - }) { + if method.attrs.iter().any(is_handler_attribute) { return syn::Error::new_spanned( &method.sig, "#[stopped] cannot be combined with handler attributes", @@ -1433,18 +1433,15 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { for item in &mut impl_block.items { if let ImplItem::Fn(method) = item { - let handler_idx = method.attrs.iter().position(|attr| { - attr.path().is_ident("handler") - || attr.path().is_ident("send_handler") - || attr.path().is_ident("request_handler") - }); + let handler_idx = method.attrs.iter().position(is_handler_attribute); if let Some(idx) = handler_idx { + let deferred = method.attrs[idx].path().is_ident("deferred_handler"); method.attrs.remove(idx); if let Some(extra) = method.attrs.iter().find(|a| is_handler_attribute(a)) { return syn::Error::new_spanned( extra, - "a method can carry only one of #[handler], #[send_handler] and #[request_handler]", + "a method can carry only one of #[handler], #[send_handler], #[request_handler] and #[deferred_handler]", ) .to_compile_error() .into(); @@ -1464,9 +1461,7 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { .attrs .iter() .filter(|a| { - !a.path().is_ident("handler") - && !a.path().is_ident("send_handler") - && !a.path().is_ident("request_handler") + !is_handler_attribute(a) && !a.path().is_ident("started") && !a.path().is_ident("stopped") }) @@ -1478,9 +1473,21 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { has_async = true; } - // Validate handler has exactly 3 parameters: &mut self, msg, ctx + // Validate parameters: (&mut self, msg, ctx), or for a deferred + // handler (&mut self, msg, reply, ctx). let param_count = method.sig.inputs.len(); - if param_count != 3 { + let ctx_index = if deferred { 3 } else { 2 }; + if deferred && param_count != 4 { + return syn::Error::new_spanned( + &method.sig, + format!( + "#[deferred_handler] method must have 4 parameters (&mut self, msg: M, reply: PendingReply<'_, R>, ctx: &Context), found {param_count}" + ), + ) + .to_compile_error() + .into(); + } + if !deferred && param_count != 3 { return syn::Error::new_spanned( &method.sig, format!( @@ -1490,6 +1497,18 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { .to_compile_error() .into(); } + if deferred { + if let ReturnType::Type(_, ty) = &method.sig.output { + if !is_unit_type(ty) { + return syn::Error::new_spanned( + ty, + "#[deferred_handler] answers through its PendingReply; remove the return type", + ) + .to_compile_error() + .into(); + } + } + } // `&self` is accepted too; a missing or by-value receiver is not. match method.sig.inputs.first() { @@ -1503,9 +1522,9 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { .to_compile_error() .into(); } - None => unreachable!("three parameters were checked above"), + None => unreachable!("the parameters were checked above"), } - if let Some(FnArg::Typed(ctx)) = method.sig.inputs.iter().nth(2) { + if let Some(FnArg::Typed(ctx)) = method.sig.inputs.iter().nth(ctx_index) { if !matches!(&*ctx.ty, Type::Reference(r) if r.mutability.is_none()) { return syn::Error::new_spanned( &ctx.ty, @@ -1529,29 +1548,47 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { } }; - // Keep an omitted return type omitted: rustc then suggests - // adding `-> R` after the user's signature for a request reply. - let output = match &method.sig.output { - ReturnType::Default => quote! {}, - ReturnType::Type(_, ty) => quote! { -> #ty }, + // The generated `handle` is the envelope dispatch: it takes the + // request's reply obligation and answers it with the method's + // result (a deferred handler receives the obligation itself). + let reply_ty = quote! { + spawned_concurrency::reply::PendingReply<'_, <#msg_ty as spawned_concurrency::message::Message>::Result> }; - // The parameter list carries the user's parentheses span, so - // that suggestion lands after the user's `)`. + // The parameter list carries the user's parentheses span. let mut params = proc_macro2::Group::new( proc_macro2::Delimiter::Parenthesis, - quote! { &mut self, msg: #msg_ty, ctx: &Context }, + quote! { &mut self, msg: #msg_ty, reply: #reply_ty, ctx: &Context }, ); params.set_span(method.sig.paren_token.span.join()); // Span the generated impl at the method so trait errors // (conflicting impls, signature mismatches) point at it. let span = method.sig.span(); + // The reply's type must equal the message's `Result`; a + // mismatch is reported at the user's return type. + let ret_span = match &method.sig.output { + ReturnType::Default => method.sig.paren_token.span.join(), + ReturnType::Type(_, ty) => ty.span(), + }; + let asynchronous = method.sig.asyncness.is_some(); + let call = match (deferred, asynchronous) { + (true, true) => quote! { self.#method_name(msg, reply, ctx).await }, + (true, false) => quote! { self.#method_name(msg, reply, ctx) }, + (false, true) => { + let value = quote::quote_spanned! { ret_span => self.#method_name(msg, ctx).await }; + quote! { let _ = reply.reply(#value); } + } + (false, false) => { + let value = quote::quote_spanned! { ret_span => self.#method_name(msg, ctx) }; + quote! { let _ = reply.reply(#value); } + } + }; - let handler_impl = if method.sig.asyncness.is_some() { + let handler_impl = if asynchronous { quote::quote_spanned! { span => #(#extra_attrs)* impl #impl_generics Handler<#msg_ty> for #self_ty #where_clause { - async fn handle #params #output { - self.#method_name(msg, ctx).await + async fn handle #params { + #call } } } @@ -1559,8 +1596,8 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { quote::quote_spanned! { span => #(#extra_attrs)* impl #impl_generics Handler<#msg_ty> for #self_ty #where_clause { - fn handle #params #output { - self.#method_name(msg, ctx) + fn handle #params { + #call } } } diff --git a/rt/src/tasks/mod.rs b/rt/src/tasks/mod.rs index d6a1f898..f302cf23 100644 --- a/rt/src/tasks/mod.rs +++ b/rt/src/tasks/mod.rs @@ -39,12 +39,12 @@ pub mod affinity; #[cfg(spawned_sim)] pub use crate::tasks::sim::{ block_in_place, sleep, spawn, spawn_blocking, task_id, timeout, unconstrained, yield_now, - Handle, JoinHandle, Runtime, RuntimeFlavor, + Handle, JoinHandle, Runtime, RuntimeFlavor, Sleep, }; pub use crate::tasks::tokio::mpsc; pub use crate::tasks::tokio::oneshot; #[cfg(not(spawned_sim))] -pub use crate::tasks::tokio::sleep; +pub use crate::tasks::tokio::{sleep, Sleep}; #[cfg(not(spawned_sim))] pub use crate::tasks::tokio::timeout; pub use crate::tasks::tokio::watch; diff --git a/rt/src/tasks/tokio/mod.rs b/rt/src/tasks/tokio/mod.rs index 1705d005..23648c0a 100644 --- a/rt/src/tasks/tokio/mod.rs +++ b/rt/src/tasks/tokio/mod.rs @@ -8,7 +8,7 @@ pub use tokio::{ task::{ block_in_place, id as task_id, spawn, spawn_blocking, unconstrained, yield_now, JoinHandle, }, - time::{sleep, timeout}, + time::{sleep, timeout, Sleep}, }; pub use tokio_stream::wrappers::{BroadcastStream, UnboundedReceiverStream as ReceiverStream}; pub use tokio_util::sync::CancellationToken; diff --git a/testing/src/lib.rs b/testing/src/lib.rs index 6e4cb78e..38f353e0 100644 --- a/testing/src/lib.rs +++ b/testing/src/lib.rs @@ -53,3 +53,22 @@ pub mod tasks; pub mod threads; pub use inbox::{Outcome, Received}; + +/// A `PendingReply` and its receiver for calling a manual +/// `Handler::handle` directly in a test, without an actor: +/// +/// ```ignore +/// let (reply, rx) = spawned_test::reply_pair::(); +/// actor.handle(GetCount, reply, &ctx).await; +/// assert_eq!(rx.await.unwrap(), 3); +/// ``` +/// +/// The reply has no deadline. Await (or `recv`) the receiver only after +/// driving every turn the reply depends on: a deferred reply answered by a +/// later message is never answered while the test waits for it first. +pub fn reply_pair() -> ( + spawned_concurrency::reply::PendingReply<'static, R>, + spawned_concurrency::reply::ReplyReceiver, +) { + spawned_concurrency::reply::reply_pair(spawned_concurrency::reply::Deadline::Never) +} diff --git a/testing/src/shared.rs b/testing/src/shared.rs index 7942d318..d8a441ad 100644 --- a/testing/src/shared.rs +++ b/testing/src/shared.rs @@ -1,9 +1,9 @@ //! Items identical in the `tasks` and `threads` modules, expanded in each. //! -//! The two modes differ in their actor traits, `oneshot` channels and whether +//! The two modes differ in their actor traits and whether //! waits are `async`; everything else about mocks and probes is written once //! here. Each macro resolves `Mock`, `MockRef`, `Probe`, `Recipient`, -//! `Receiver`, `oneshot` and friends in the module that invokes it. +//! `Receiver` and friends in the module that invokes it. /// Builder methods of `Mock`, except `delay` and `start`. macro_rules! mock_builder { @@ -152,12 +152,20 @@ macro_rules! probe_common { fn send(&self, msg: M) -> Result<(), ActorError> { ProbeCore::send(self, msg) } - fn request_raw(&self, msg: M) -> Result, ActorError> { - let (tx, rx) = oneshot::channel(); - if let Some((value, record)) = self.request(msg)? { - let _ = tx.send(value); - // Logged `Handled` only once the reply is in the channel. - record.finish(); + fn request_raw_until( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + let (reply, rx) = spawned_concurrency::reply::reply_pair(deadline); + match self.request(msg)? { + Some((value, record)) => { + let _ = reply.reply(value); + // Logged `Handled` only once the reply is in the slot. + record.finish(); + } + // An unexpected request is discharged as `ActorStopped`. + None => reply.__discharge_stopped(), } Ok(rx) } diff --git a/testing/src/tasks.rs b/testing/src/tasks.rs index 21b69154..43a3f090 100644 --- a/testing/src/tasks.rs +++ b/testing/src/tasks.rs @@ -21,7 +21,8 @@ use spawned_concurrency::tasks::{ Actor, ActorRef, ActorStart, Context, Handler, Receiver, Recipient, }; use spawned_concurrency::{ChildHandle, ExitReason}; -use spawned_rt::tasks::{oneshot, sleep, timeout}; +use spawned_concurrency::reply::{Deadline, PendingReply, ReplyReceiver}; +use spawned_rt::tasks::{sleep, timeout}; use std::future::Future; use std::ops::Deref; use std::panic::Location; @@ -93,7 +94,12 @@ impl Mock { impl Actor for Mock {} impl Handler for Mock { - fn handle(&mut self, msg: M, _: &Context) -> impl Future + Send { + fn handle( + &mut self, + msg: M, + reply: PendingReply<'_, M::Result>, + _: &Context, + ) -> impl Future + Send { let plan = self.rules.plan(&self.inbox, msg); async move { match plan { @@ -108,7 +114,7 @@ impl Handler for Mock { if let Some(record) = record { record.finish(); } - value + let _ = reply.reply(value); } Plan::Crash(reason) => panic!("{reason}"), } diff --git a/testing/src/threads.rs b/testing/src/threads.rs index c452bdf5..aed2e565 100644 --- a/testing/src/threads.rs +++ b/testing/src/threads.rs @@ -10,7 +10,7 @@ use spawned_concurrency::threads::{ Actor, ActorRef, ActorStart, Context, Handler, Receiver, Recipient, }; use spawned_concurrency::{ChildHandle, ExitReason}; -use spawned_rt::threads::oneshot; +use spawned_concurrency::reply::{Deadline, PendingReply, ReplyReceiver}; use std::ops::Deref; use std::panic::Location; use std::sync::Arc; @@ -52,7 +52,7 @@ impl Mock { impl Actor for Mock {} impl Handler for Mock { - fn handle(&mut self, msg: M, _: &Context) -> M::Result { + fn handle(&mut self, msg: M, reply: PendingReply<'_, M::Result>, _: &Context) { match self.rules.plan(&self.inbox, msg) { Plan::Reply { delay, @@ -65,7 +65,7 @@ impl Handler for Mock { if let Some(record) = record { record.finish(); } - value + let _ = reply.reply(value); } Plan::Crash(reason) => panic!("{reason}"), } From f4a981569f7c19bc4938d6c9071e670f812bda10 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:21:33 -0300 Subject: [PATCH 008/208] refactor: migrate manual Handler impls to the PendingReply signature (scripted) --- concurrency/benches/baseline.rs | 10 +- concurrency/benches/designs.rs | 20 +- concurrency/benches/lifecycle_soak.rs | 25 +- concurrency/benches/metrics_overhead.rs | 10 +- concurrency/benches/pinned_pipeline.rs | 15 +- concurrency/benches/runtime_isolation.rs | 11 +- concurrency/benches/workload_research.rs | 10 +- concurrency/examples/panic_abort.rs | 10 +- concurrency/src/child_handle.rs | 20 +- concurrency/src/link_regression_tests.rs | 27 +- concurrency/src/tasks/actor.rs | 232 +++++++++++++----- concurrency/src/tasks/stream_tests.rs | 15 +- concurrency/src/tasks/timer_tests.rs | 30 ++- concurrency/src/threads/actor.rs | 161 +++++++++--- concurrency/src/threads/stream_tests.rs | 10 +- concurrency/src/threads/timer_tests.rs | 30 ++- concurrency/tests/actor_scenarios.rs | 20 +- concurrency/tests/backend_experiments.rs | 15 +- concurrency/tests/backend_resource_limits.rs | 4 +- concurrency/tests/borrowed_runtime.rs | 10 +- concurrency/tests/canary_feedback.rs | 15 +- concurrency/tests/cancellation_safety.rs | 15 +- concurrency/tests/cfg_gated_protocol.rs | 14 +- concurrency/tests/cohort_trace.rs | 10 +- concurrency/tests/context_exit.rs | 10 +- concurrency/tests/coverage_trace.rs | 10 +- concurrency/tests/cross_backend.rs | 47 +++- concurrency/tests/dedicated_runtime.rs | 20 +- concurrency/tests/execution_groups.rs | 19 +- concurrency/tests/execution_profiles.rs | 29 ++- concurrency/tests/exit_signal_matrix.rs | 13 +- concurrency/tests/failpoints.rs | 28 ++- concurrency/tests/foreign_panics.rs | 40 ++- concurrency/tests/full_actor_shards.rs | 54 ++-- concurrency/tests/future_layout.rs | 9 +- concurrency/tests/kill_links.rs | 10 +- concurrency/tests/leaks.rs | 97 +++++--- concurrency/tests/mailbox_drop_panic.rs | 19 +- concurrency/tests/metrics.rs | 19 +- concurrency/tests/metrics_composable.rs | 4 +- concurrency/tests/metrics_counts_only.rs | 4 +- concurrency/tests/metrics_sampling.rs | 4 +- concurrency/tests/native_dispatcher.rs | 5 +- concurrency/tests/native_mailbox_drop.rs | 10 +- concurrency/tests/nested_supervision.rs | 15 +- concurrency/tests/panic_safety.rs | 100 ++++++-- concurrency/tests/process_failures.rs | 5 +- concurrency/tests/protocol_timeouts.rs | 54 +++- concurrency/tests/resource_cleanup.rs | 10 +- concurrency/tests/semantics_gaps.rs | 65 ++++- concurrency/tests/shutdown_publication.rs | 5 +- concurrency/tests/sim.rs | 5 +- concurrency/tests/sim_explore.rs | 25 +- concurrency/tests/stack_budget.rs | 20 +- concurrency/tests/startup_admission.rs | 5 +- concurrency/tests/supervision.rs | 40 ++- concurrency/tests/supervision_strategies.rs | 20 +- concurrency/tests/supervision_trace.rs | 30 ++- concurrency/tests/supervisor_policies.rs | 10 +- concurrency/tests/support/failure_actor.rs | 28 ++- concurrency/tests/support/store_actor.rs | 18 +- concurrency/tests/tree_trace.rs | 5 +- .../src/bin/byte_budget_pipeline.rs | 15 +- .../src/bin/coordinated_shutdown.rs | 19 +- .../capabilities/src/bin/dedicated_thread.rs | 5 +- .../capabilities/src/bin/execution_groups.rs | 10 +- .../src/bin/execution_profiles.rs | 5 +- .../capabilities/src/bin/framework_metrics.rs | 4 +- .../capabilities/src/bin/graceful_drain.rs | 10 +- .../capabilities/src/bin/native_lifecycle.rs | 10 +- .../capabilities/src/bin/protocol_timeouts.rs | 20 +- .../src/bin/registrations_and_fakes.rs | 5 +- .../capabilities/src/bin/runtime_isolation.rs | 10 +- .../src/bin/shutdown_diagnostics.rs | 10 +- .../src/bin/streams_and_recipients.rs | 20 +- .../src/bin/supervision_recovery.rs | 10 +- .../capabilities/src/bin/supervision_tree.rs | 5 +- .../capabilities/src/bin/timeout_outcomes.rs | 10 +- examples/exit_reason/src/main.rs | 24 +- examples/thread_only/src/main.rs | 10 +- testing/src/faults.rs | 5 +- testing/tests/tasks_kit.rs | 15 +- testing/tests/threads_kit.rs | 4 +- 83 files changed, 1446 insertions(+), 430 deletions(-) diff --git a/concurrency/benches/baseline.rs b/concurrency/benches/baseline.rs index 0f8ca9c0..f65e4065 100644 --- a/concurrency/benches/baseline.rs +++ b/concurrency/benches/baseline.rs @@ -98,14 +98,20 @@ fn process(work: Work) -> Ack { struct Sink; impl tasks::Actor for Sink {} impl tasks::Handler for Sink { - async fn handle(&mut self, work: Work, _: &tasks::Context) -> Ack { + async fn handle(&mut self, work: Work, reply: spawned_concurrency::reply::PendingReply<'_, Ack>, _: &tasks::Context) { + let __reply_value: Ack = { process(work) + }; + let _ = reply.reply(__reply_value); } } impl threads::Actor for Sink {} impl threads::Handler for Sink { - fn handle(&mut self, work: Work, _: &threads::Context) -> Ack { + fn handle(&mut self, work: Work, reply: spawned_concurrency::reply::PendingReply<'_, Ack>, _: &threads::Context) { + let __reply_value: Ack = { process(work) + }; + let _ = reply.reply(__reply_value); } } #[derive(Clone)] diff --git a/concurrency/benches/designs.rs b/concurrency/benches/designs.rs index 538c1b8e..8db4d352 100644 --- a/concurrency/benches/designs.rs +++ b/concurrency/benches/designs.rs @@ -122,7 +122,8 @@ impl Sink { } impl tasks::Actor for Sink {} impl tasks::Handler for Sink { - async fn handle(&mut self, work: Work, _: &tasks::Context) -> Ack { + async fn handle(&mut self, work: Work, reply: spawned_concurrency::reply::PendingReply<'_, Ack>, _: &tasks::Context) { + let __reply_value: Ack = { let start = if work.timing { Instant::now() } else { @@ -151,12 +152,17 @@ impl tasks::Handler for Sink { _ => {} } process_at(work, start) + }; + let _ = reply.reply(__reply_value); } } impl threads::Actor for Sink {} impl threads::Handler for Sink { - fn handle(&mut self, work: Work, _: &threads::Context) -> Ack { + fn handle(&mut self, work: Work, reply: spawned_concurrency::reply::PendingReply<'_, Ack>, _: &threads::Context) { + let __reply_value: Ack = { process(work) + }; + let _ = reply.reply(__reply_value); } } enum ReplySender { @@ -187,14 +193,20 @@ impl Message for Probe { type Result = (); } impl tasks::Handler for Sink { - async fn handle(&mut self, probe: Probe, _: &tasks::Context) { + async fn handle(&mut self, probe: Probe, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &tasks::Context) { + { probe.reply.send(process(probe.work)); } + let _ = reply.reply(()); + } } impl threads::Handler for Sink { - fn handle(&mut self, probe: Probe, _: &threads::Context) { + fn handle(&mut self, probe: Probe, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context) { + { probe.reply.send(process(probe.work)); } + let _ = reply.reply(()); + } } enum TokioMessage { Request(Work, tokio::sync::oneshot::Sender), diff --git a/concurrency/benches/lifecycle_soak.rs b/concurrency/benches/lifecycle_soak.rs index d31d8800..1a2864a3 100644 --- a/concurrency/benches/lifecycle_soak.rs +++ b/concurrency/benches/lifecycle_soak.rs @@ -61,7 +61,8 @@ impl Message for Step { type Result = u64; } impl Handler for Worker { - async fn handle(&mut self, msg: Step, _: &Context) -> u64 { + async fn handle(&mut self, msg: Step, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { assert_eq!(self.state, msg.expected); assert_eq!(self.bytes[0], 13); if msg.helper { @@ -78,6 +79,8 @@ impl Handler for Worker { self.state += 1; self.counts.requests.fetch_add(1, Ordering::Relaxed); self.state + }; + let _ = reply.reply(__reply_value); } } struct Crash; @@ -85,9 +88,12 @@ impl Message for Crash { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("soak injected handler failure"); } + let _ = reply.reply(()); + } } struct Watcher { counts: Arc, @@ -99,14 +105,18 @@ impl Message for Watch { type Result = (); } impl Handler for Watcher { - async fn handle(&mut self, msg: Watch, ctx: &Context) { + async fn handle(&mut self, msg: Watch, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + { for (child, panic) in msg.0 { assert!(self.pending.insert(ctx.monitor(&child), panic).is_none()); } } + let _ = reply.reply(()); + } } impl Handler for Watcher { - async fn handle(&mut self, msg: Down, _: &Context) { + async fn handle(&mut self, msg: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { let panic = self .pending .remove(&msg.monitor_ref) @@ -117,14 +127,19 @@ impl Handler for Watcher { } self.counts.down.fetch_add(1, Ordering::Release); } + let _ = reply.reply(()); + } } struct Remaining; impl Message for Remaining { type Result = usize; } impl Handler for Watcher { - async fn handle(&mut self, _: Remaining, _: &Context) -> usize { + async fn handle(&mut self, _: Remaining, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { + let __reply_value: usize = { self.pending.len() + }; + let _ = reply.reply(__reply_value); } } fn pin(cpus: &str) { diff --git a/concurrency/benches/metrics_overhead.rs b/concurrency/benches/metrics_overhead.rs index 6eba29d9..99f94825 100644 --- a/concurrency/benches/metrics_overhead.rs +++ b/concurrency/benches/metrics_overhead.rs @@ -18,14 +18,20 @@ fn compute(work: Work) -> u64 { } impl tasks::Actor for Sink {} impl tasks::Handler for Sink { - async fn handle(&mut self, work: Work, _: &tasks::Context) -> u64 { + async fn handle(&mut self, work: Work, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &tasks::Context) { + let __reply_value: u64 = { compute(work) + }; + let _ = reply.reply(__reply_value); } } impl threads::Actor for Sink {} impl threads::Handler for Sink { - fn handle(&mut self, work: Work, _: &threads::Context) -> u64 { + fn handle(&mut self, work: Work, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &threads::Context) { + let __reply_value: u64 = { compute(work) + }; + let _ = reply.reply(__reply_value); } } fn main() { diff --git a/concurrency/benches/pinned_pipeline.rs b/concurrency/benches/pinned_pipeline.rs index c024b39e..04c4f590 100644 --- a/concurrency/benches/pinned_pipeline.rs +++ b/concurrency/benches/pinned_pipeline.rs @@ -49,9 +49,12 @@ mod linux { type Result = (); } impl Handler for Stage { - async fn handle(&mut self, m: Wire, _: &Context) { + async fn handle(&mut self, m: Wire, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.next = m.0; } + let _ = reply.reply(()); + } } struct Work { id: u64, @@ -63,7 +66,8 @@ mod linux { type Result = (); } impl Handler for Stage { - async fn handle(&mut self, mut m: Work, _: &Context) { + async fn handle(&mut self, mut m: Work, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { for _ in 0..self.iterations { m.value = black_box(m.value.wrapping_mul(MUL).wrapping_add(1)); } @@ -77,13 +81,16 @@ mod linux { .unwrap(); } } + let _ = reply.reply(()); + } } struct Audit; impl Message for Audit { type Result = (String, String, u64); } impl Handler for Stage { - async fn handle(&mut self, _: Audit, _: &Context) -> (String, String, u64) { + async fn handle(&mut self, _: Audit, reply: spawned_concurrency::reply::PendingReply<'_, (String, String, u64)>, _: &Context) { + let __reply_value: (String, String, u64) = { let status = fs::read_to_string("/proc/thread-self/status").unwrap(); let affinity = status .lines() @@ -96,6 +103,8 @@ mod linux { affinity, self.handled, ) + }; + let _ = reply.reply(__reply_value); } } fn pin(mask: &str) { diff --git a/concurrency/benches/runtime_isolation.rs b/concurrency/benches/runtime_isolation.rs index df8981c6..9b11d9c0 100644 --- a/concurrency/benches/runtime_isolation.rs +++ b/concurrency/benches/runtime_isolation.rs @@ -46,7 +46,8 @@ mod linux { type Result = (u64, u64, Instant); } impl Handler for Stage { - async fn handle(&mut self, m: Work, _: &Context) -> (u64, u64, Instant) { + async fn handle(&mut self, m: Work, reply: spawned_concurrency::reply::PendingReply<'_, (u64, u64, Instant)>, _: &Context) { + let __reply_value: (u64, u64, Instant) = async { self.handled += 1; if let Some(next) = &self.next { return next.request_raw(m).unwrap().await.unwrap(); @@ -57,13 +58,17 @@ mod linux { } (m.0, value, Instant::now()) } + .await; + let _ = reply.reply(__reply_value); + } } struct Audit; impl Message for Audit { type Result = (String, u64); } impl Handler for Stage { - async fn handle(&mut self, _: Audit, _: &Context) -> (String, u64) { + async fn handle(&mut self, _: Audit, reply: spawned_concurrency::reply::PendingReply<'_, (String, u64)>, _: &Context) { + let __reply_value: (String, u64) = { let status = fs::read_to_string("/proc/thread-self/status").unwrap(); ( status @@ -74,6 +79,8 @@ mod linux { .into(), self.handled, ) + }; + let _ = reply.reply(__reply_value); } } fn pin(mask: &str) { diff --git a/concurrency/benches/workload_research.rs b/concurrency/benches/workload_research.rs index 8086108f..941b44b1 100644 --- a/concurrency/benches/workload_research.rs +++ b/concurrency/benches/workload_research.rs @@ -230,7 +230,8 @@ struct Node { } impl Actor for Node {} impl Handler for Node { - async fn handle(&mut self, msg: Wire, _: &Context) { + async fn handle(&mut self, msg: Wire, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { if std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").as_deref() == Ok("sharded") { assert_eq!( std::thread::current().name().unwrap(), @@ -239,6 +240,8 @@ impl Handler for Node { } self.children = msg.children; } + let _ = reply.reply(()); + } } #[inline(never)] fn cpu_work(mut value: u64, iterations: u32) -> u64 { @@ -248,7 +251,8 @@ fn cpu_work(mut value: u64, iterations: u32) -> u64 { value } impl Handler for Node { - async fn handle(&mut self, msg: Route, _: &Context) -> Ack { + async fn handle(&mut self, msg: Route, reply: spawned_concurrency::reply::PendingReply<'_, Ack>, _: &Context) { + let __reply_value: Ack = { let started = Instant::now(); if msg.measured { self.meter.started.fetch_add(1, Ordering::Relaxed); @@ -283,6 +287,8 @@ impl Handler for Node { queue_ns: started.duration_since(msg.enqueued).as_nanos() as u64, service_ns: completed.duration_since(started).as_nanos() as u64, } + }; + let _ = reply.reply(__reply_value); } } fn layout(c: &Config) -> (Vec, Vec) { diff --git a/concurrency/examples/panic_abort.rs b/concurrency/examples/panic_abort.rs index 90b56798..1e62efa2 100644 --- a/concurrency/examples/panic_abort.rs +++ b/concurrency/examples/panic_abort.rs @@ -43,8 +43,11 @@ mod native { } } impl Handler for Victim { - fn handle(&mut self, _: Boom, _: &Context) -> u64 { + fn handle(&mut self, _: Boom, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { panic!("{PANIC_TEXT}") + }; + let _ = reply.reply(__reply_value); } } pub fn handler_panic() { @@ -66,8 +69,11 @@ mod tasks { } } impl Handler for Victim { - async fn handle(&mut self, _: Boom, _: &Context) -> u64 { + async fn handle(&mut self, _: Boom, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { panic!("{PANIC_TEXT}") + }; + let _ = reply.reply(__reply_value); } } pub fn handler_panic() { diff --git a/concurrency/src/child_handle.rs b/concurrency/src/child_handle.rs index dbcd8611..1d2fb827 100644 --- a/concurrency/src/child_handle.rs +++ b/concurrency/src/child_handle.rs @@ -472,9 +472,12 @@ mod tests { } impl Actor for Worker {} impl Handler for Worker { - fn handle(&mut self, _msg: Stop, ctx: &Context) { + fn handle(&mut self, _msg: Stop, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.stop(); } + let _ = reply.reply(()); + } } let actor = Worker.start(); @@ -503,9 +506,12 @@ mod tests { } impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _msg: Stop, ctx: &Context) { + async fn handle(&mut self, _msg: Stop, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.stop(); } + let _ = reply.reply(()); + } } let runtime = spawned_rt::tasks::Runtime::new().unwrap(); @@ -620,9 +626,12 @@ mod tests { } impl Actor for Boomer {} impl Handler for Boomer { - fn handle(&mut self, _msg: Boom, _ctx: &Context) { + fn handle(&mut self, _msg: Boom, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { panic!("intentional panic in handler"); } + let _ = reply.reply(()); + } } let actor = Boomer.start(); @@ -652,9 +661,12 @@ mod tests { } impl Actor for Boomer {} impl Handler for Boomer { - async fn handle(&mut self, _msg: Boom, _ctx: &Context) { + async fn handle(&mut self, _msg: Boom, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { panic!("intentional panic in handler"); } + let _ = reply.reply(()); + } } let runtime = spawned_rt::tasks::Runtime::new().unwrap(); diff --git a/concurrency/src/link_regression_tests.rs b/concurrency/src/link_regression_tests.rs index a30d849a..e5dbfd8a 100644 --- a/concurrency/src/link_regression_tests.rs +++ b/concurrency/src/link_regression_tests.rs @@ -119,7 +119,9 @@ impl crate::message::Message for Ping { type Result = (); } impl crate::tasks::Handler for Idler { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle(&mut self, _: Ping, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } #[test] @@ -157,9 +159,12 @@ fn polled_actor_propagates_runtime_shutdown_to_peer() { struct MonitorObserver(Option>); impl Actor for MonitorObserver {} impl crate::tasks::Handler for MonitorObserver { - async fn handle(&mut self, down: crate::Down, _: &Context) { + async fn handle(&mut self, down: crate::Down, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.take().unwrap().send(down).unwrap(); } + let _ = reply.reply(()); + } } #[test] fn never_polled_actor_notifies_monitor_on_runtime_shutdown() { @@ -205,9 +210,12 @@ impl crate::message::Message for Crash { type Result = (); } impl crate::tasks::Handler for Idler { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("linked failure during another actor's stopped hook"); } + let _ = reply.reply(()); + } } struct Stopping { entered: Option>, @@ -220,9 +228,12 @@ impl Actor for Stopping { } } impl crate::tasks::Handler for Stopping { - async fn handle(&mut self, _: Stop, ctx: &Context) { + async fn handle(&mut self, _: Stop, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.stop(); } + let _ = reply.reply(()); + } } #[test] fn late_link_failure_cannot_relabel_an_already_requested_normal_stop() { @@ -466,7 +477,9 @@ fn linked_runtime_abort_cancels_an_idle_threads_actor() { struct ThreadPeer; impl ThreadActor for ThreadPeer {} impl crate::threads::Handler for ThreadPeer { - fn handle(&mut self, _: Ping, _: &crate::threads::Context) {} + fn handle(&mut self, _: Ping, reply: crate::reply::PendingReply<'_, ()>, _: &crate::threads::Context) { + let _ = reply.reply(()); + } } let owner = runtime(); let waiter = runtime(); @@ -508,7 +521,9 @@ fn dropping_runtime_with_large_polled_link_cycle_reclaims_every_actor() { struct Node(Arc); impl Actor for Node {} impl crate::tasks::Handler for Node { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle(&mut self, _: Ping, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } impl Drop for Node { fn drop(&mut self) { diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index a585a757..ec3eb8bd 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -172,14 +172,13 @@ where observation.reject(); } } - fn handle<'a>( - self: Box, - actor: &'a mut A, - ctx: &'a Context, - ) -> Pin + Send + 'a>> { + fn handle<'a>(self: Box, actor: &'a mut A, reply: crate::reply::PendingReply<'_, Pin + Send + 'a>>>, ctx: &'a Context) { + let __reply_value: Pin + Send + 'a>> = { let mut this = *self; let reply = PendingReply::new(this.slot.take(), &ctx.shared.replies); Box::pin(actor.handle(this.msg, reply, ctx)) + }; + let _ = reply.reply(__reply_value); } } @@ -1600,22 +1599,31 @@ mod tests { impl Actor for Counter {} impl Handler for Counter { - async fn handle(&mut self, _msg: GetCount, _ctx: &Context) -> u64 { + async fn handle(&mut self, _msg: GetCount, reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context) { + let __reply_value: u64 = { self.count + }; + let _ = reply.reply(__reply_value); } } impl Handler for Counter { - async fn handle(&mut self, _msg: Increment, _ctx: &Context) -> u64 { + async fn handle(&mut self, _msg: Increment, reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context) { + let __reply_value: u64 = { self.count += 1; self.count + }; + let _ = reply.reply(__reply_value); } } impl Handler for Counter { - async fn handle(&mut self, _msg: StopCounter, ctx: &Context) -> u64 { + async fn handle(&mut self, _msg: StopCounter, reply: crate::reply::PendingReply<'_, u64>, ctx: &Context) { + let __reply_value: u64 = { ctx.stop(); self.count + }; + let _ = reply.reply(__reply_value); } } @@ -1756,11 +1764,14 @@ mod tests { } impl Actor for SlowActor {} impl Handler for SlowActor { - async fn handle(&mut self, _msg: SlowOp, _ctx: &Context) { + async fn handle(&mut self, _msg: SlowOp, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { // Never replies, so the timeout is the only possible outcome // however late the runtime polls it. std::future::pending::<()>().await; } + let _ = reply.reply(()); + } } let actor = SlowActor.start(); @@ -1806,9 +1817,12 @@ mod tests { } impl Handler for SlowShutdownActor { - async fn handle(&mut self, _msg: StopSlow, ctx: &Context) { + async fn handle(&mut self, _msg: StopSlow, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.stop(); } + let _ = reply.reply(()); + } } /// Blocks its thread in a callback until the test releases it. @@ -1846,16 +1860,22 @@ mod tests { } impl Handler for HeldThread { - async fn handle(&mut self, _msg: StopSlow, ctx: &Context) { + async fn handle(&mut self, _msg: StopSlow, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.stop(); } + let _ = reply.reply(()); + } } impl Handler for HeldThread { - async fn handle(&mut self, _msg: DoBlock, ctx: &Context) { + async fn handle(&mut self, _msg: DoBlock, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { self.hold(); ctx.stop(); } + let _ = reply.reply(()); + } } #[test] @@ -1927,11 +1947,14 @@ mod tests { impl Actor for BadlyBehavedTask {} impl Handler for BadlyBehavedTask { - async fn handle(&mut self, _msg: DoBlock, ctx: &Context) { + async fn handle(&mut self, _msg: DoBlock, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { rt::sleep(Duration::from_millis(20)).await; thread::sleep(Duration::from_secs(2)); ctx.stop(); } + let _ = reply.reply(()); + } } struct IncrementWell; @@ -1946,24 +1969,33 @@ mod tests { impl Actor for WellBehavedTask {} impl Handler for WellBehavedTask { - async fn handle(&mut self, _msg: GetCount, _ctx: &Context) -> u64 { + async fn handle(&mut self, _msg: GetCount, reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context) { + let __reply_value: u64 = { self.count + }; + let _ = reply.reply(__reply_value); } } impl Handler for WellBehavedTask { - async fn handle(&mut self, _msg: StopCounter, ctx: &Context) -> u64 { + async fn handle(&mut self, _msg: StopCounter, reply: crate::reply::PendingReply<'_, u64>, ctx: &Context) { + let __reply_value: u64 = { ctx.stop(); self.count + }; + let _ = reply.reply(__reply_value); } } impl Handler for WellBehavedTask { - async fn handle(&mut self, _msg: IncrementWell, ctx: &Context) { + async fn handle(&mut self, _msg: IncrementWell, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { self.count += 1; use crate::tasks::send_after; send_after(Duration::from_millis(100), ctx.clone(), IncrementWell); } + let _ = reply.reply(()); + } } #[test] @@ -2052,7 +2084,9 @@ mod tests { } } impl Handler for PanicOnStart { - async fn handle(&mut self, _msg: Ping, _ctx: &Context) {} + async fn handle(&mut self, _msg: Ping, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + let _ = reply.reply(()); + } } let actor = PanicOnStart.start(); @@ -2077,13 +2111,19 @@ mod tests { } impl Actor for PanicOnMsg {} impl Handler for PanicOnMsg { - async fn handle(&mut self, _msg: Explode, _ctx: &Context) { + async fn handle(&mut self, _msg: Explode, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { panic!("boom in handler"); } + let _ = reply.reply(()); + } } impl Handler for PanicOnMsg { - async fn handle(&mut self, _msg: Check, _ctx: &Context) -> u32 { + async fn handle(&mut self, _msg: Check, reply: crate::reply::PendingReply<'_, u32>, _ctx: &Context) { + let __reply_value: u32 = { 42 + }; + let _ = reply.reply(__reply_value); } } @@ -2110,9 +2150,12 @@ mod tests { } } impl Handler for PanicOnStop { - async fn handle(&mut self, _msg: StopMe, ctx: &Context) { + async fn handle(&mut self, _msg: StopMe, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.stop(); } + let _ = reply.reply(()); + } } let actor = PanicOnStop.start(); @@ -2178,7 +2221,9 @@ mod tests { } } impl Handler for PanicStart { - async fn handle(&mut self, _msg: Ping, _ctx: &Context) {} + async fn handle(&mut self, _msg: Ping, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + let _ = reply.reply(()); + } } let runtime = rt::Runtime::new().unwrap(); @@ -2200,9 +2245,12 @@ mod tests { } impl Actor for PanicHandler {} impl Handler for PanicHandler { - async fn handle(&mut self, _msg: Explode, _ctx: &Context) { + async fn handle(&mut self, _msg: Explode, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { panic!("boom in handler"); } + let _ = reply.reply(()); + } } let runtime = rt::Runtime::new().unwrap(); @@ -2253,34 +2301,38 @@ mod tests { impl Actor for Watcher {} impl Handler for Watcher { - async fn handle( - &mut self, - msg: StartMonitor, - ctx: &Context, - ) -> crate::monitor::MonitorRef { + async fn handle(&mut self, msg: StartMonitor, reply: crate::reply::PendingReply<'_, crate::monitor::MonitorRef>, ctx: &Context) { + let __reply_value: crate::monitor::MonitorRef = { ctx.monitor(&msg.0) + }; + let _ = reply.reply(__reply_value); } } impl Handler for Watcher { - async fn handle(&mut self, msg: CallDemonitor, ctx: &Context) { + async fn handle(&mut self, msg: CallDemonitor, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.demonitor(msg.0); } + let _ = reply.reply(()); + } } impl Handler for Watcher { - async fn handle(&mut self, msg: crate::monitor::Down, _ctx: &Context) { + async fn handle(&mut self, msg: crate::monitor::Down, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { self.downs.lock().unwrap().push(msg); } + let _ = reply.reply(()); + } } impl Handler for Watcher { - async fn handle( - &mut self, - _msg: GetDowns, - _ctx: &Context, - ) -> Vec { + async fn handle(&mut self, _msg: GetDowns, reply: crate::reply::PendingReply<'_, Vec>, _ctx: &Context) { + let __reply_value: Vec = { self.downs.lock().unwrap().clone() + }; + let _ = reply.reply(__reply_value); } } @@ -2408,13 +2460,16 @@ mod tests { type Result = usize; } impl Handler for Watcher { - async fn handle(&mut self, _msg: Inspect, ctx: &Context) -> usize { + async fn handle(&mut self, _msg: Inspect, reply: crate::reply::PendingReply<'_, usize>, ctx: &Context) { + let __reply_value: usize = { ctx.shared .monitors .lock() .unwrap() .as_ref() .map_or(0, HashMap::len) + }; + let _ = reply.reply(__reply_value); } } @@ -2456,9 +2511,12 @@ mod tests { struct PanicMe; impl Actor for PanicMe {} impl Handler for PanicMe { - async fn handle(&mut self, _msg: PanicMsg, _ctx: &Context) { + async fn handle(&mut self, _msg: PanicMsg, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { panic!("intentional panic"); } + let _ = reply.reply(()); + } } let runtime = rt::Runtime::new().unwrap(); @@ -2544,14 +2602,20 @@ mod tests { } impl Handler for TrapActor { - async fn handle(&mut self, msg: LinkTo, ctx: &Context) { + async fn handle(&mut self, msg: LinkTo, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.link(&msg.0); } + let _ = reply.reply(()); + } } impl Handler for TrapActor { - async fn handle(&mut self, _msg: GetExits, _ctx: &Context) -> Vec { + async fn handle(&mut self, _msg: GetExits, reply: crate::reply::PendingReply<'_, Vec>, _ctx: &Context) { + let __reply_value: Vec = { self.exits.lock().unwrap().clone() + }; + let _ = reply.reply(__reply_value); } } @@ -2579,9 +2643,12 @@ mod tests { type Result = (); } impl Handler for TrapActor { - async fn handle(&mut self, _msg: Boom, _ctx: &Context) { + async fn handle(&mut self, _msg: Boom, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { panic!("boom"); } + let _ = reply.reply(()); + } } let _ = a.send(Boom); @@ -2607,9 +2674,12 @@ mod tests { type Result = (); } impl Handler for TrapActor { - async fn handle(&mut self, _msg: Boom2, _ctx: &Context) { + async fn handle(&mut self, _msg: Boom2, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { panic!("boom2"); } + let _ = reply.reply(()); + } } let _ = a.send(Boom2); @@ -2666,9 +2736,12 @@ mod tests { type Result = (); } impl Handler for TrapActor { - async fn handle(&mut self, _msg: Boom3, _ctx: &Context) { + async fn handle(&mut self, _msg: Boom3, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { panic!("boom3"); } + let _ = reply.reply(()); + } } let target = make_trapper(false); let _ = target.send(Boom3); @@ -2710,9 +2783,12 @@ mod tests { type Result = (); } impl Handler for TrapActor { - async fn handle(&mut self, msg: UnlinkFrom, ctx: &Context) { + async fn handle(&mut self, msg: UnlinkFrom, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.unlink(&msg.0); } + let _ = reply.reply(()); + } } b.request(UnlinkFrom(a.child_handle())).await.unwrap(); @@ -2722,8 +2798,11 @@ mod tests { type Result = u64; } impl Handler for Counter { - async fn handle(&mut self, _msg: Boom4, _ctx: &Context) -> u64 { + async fn handle(&mut self, _msg: Boom4, reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context) { + let __reply_value: u64 = { panic!("boom4"); + }; + let _ = reply.reply(__reply_value); } } // Counter's Boom4 returns u64; just use send with no expectation of reply @@ -2755,8 +2834,11 @@ mod tests { type Result = u64; } impl Handler for Counter { - async fn handle(&mut self, _msg: Boom5, _ctx: &Context) -> u64 { + async fn handle(&mut self, _msg: Boom5, reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context) { + let __reply_value: u64 = { panic!("boom5"); + }; + let _ = reply.reply(__reply_value); } } let _ = a.send(Boom5); @@ -2798,9 +2880,12 @@ mod tests { type Result = (); } impl Handler for TrapActor { - async fn handle(&mut self, _msg: Boom6, _ctx: &Context) { + async fn handle(&mut self, _msg: Boom6, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { panic!("boom6"); } + let _ = reply.reply(()); + } } let _ = a.send(Boom6); a.wait_exit().await; @@ -2831,9 +2916,12 @@ mod tests { type Result = (); } impl Handler for TrapActor { - async fn handle(&mut self, _msg: Boom7, _ctx: &Context) { + async fn handle(&mut self, _msg: Boom7, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { panic!("boom7"); } + let _ = reply.reply(()); + } } let runtime = rt::Runtime::new().unwrap(); @@ -2879,9 +2967,12 @@ mod tests { } impl Actor for Parent {} impl Handler for Parent { - async fn handle(&mut self, _msg: PanicParent, _ctx: &Context) { + async fn handle(&mut self, _msg: PanicParent, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { panic!("parent boom"); } + let _ = reply.reply(()); + } } struct Child; @@ -2909,7 +3000,9 @@ mod monitor_cleanup_regressions { struct Idle; impl Actor for Idle {} impl Handler for Idle { - async fn handle(&mut self, _: Down, _: &Context) {} + async fn handle(&mut self, _: Down, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } #[test] fn closed_monitor_table_rejects_late_registration() { @@ -3050,9 +3143,12 @@ mod turn_regressions { type Result = (); } impl Handler for Count { - async fn handle(&mut self, _: Inc, _: &Context) { + async fn handle(&mut self, _: Inc, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.fetch_add(1, Ordering::SeqCst); } + let _ = reply.reply(()); + } } rt::Runtime::new().unwrap().block_on(async { let (tx, rx) = mpsc::channel(); @@ -3116,9 +3212,12 @@ mod idle_stop_regression { type Result = (); } impl Handler for Counter { - async fn handle(&mut self, _: Increment, _: &Context) { + async fn handle(&mut self, _: Increment, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.fetch_add(1, Ordering::SeqCst); } + let _ = reply.reply(()); + } } #[test] fn stop_while_idle_does_not_dispatch_a_queued_request_on_resume() { @@ -3199,29 +3298,37 @@ mod api_coverage { } impl Actor for Echo {} impl Handler for Echo { - async fn handle(&mut self, msg: Ask, _: &Context) -> u32 { + async fn handle(&mut self, msg: Ask, reply: crate::reply::PendingReply<'_, u32>, _: &Context) { + let __reply_value: u32 = { msg.0 + }; + let _ = reply.reply(__reply_value); } } impl Handler for Echo { - async fn handle(&mut self, msg: StopAfter, ctx: &Context) { + async fn handle(&mut self, msg: StopAfter, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { let _ = msg.0.await; ctx.stop(); } + let _ = reply.reply(()); + } } impl Handler for Echo { - async fn handle( - &mut self, - _: OwnRecipient, - ctx: &Context, - ) -> (Recipient, ActorId) { + async fn handle(&mut self, _: OwnRecipient, reply: crate::reply::PendingReply<'_, (Recipient, ActorId)>, ctx: &Context) { + let __reply_value: (Recipient, ActorId) = { (ctx.recipient::(), ctx.id()) + }; + let _ = reply.reply(__reply_value); } } impl Handler for Echo { - async fn handle(&mut self, _: Boom, _: &Context) { + async fn handle(&mut self, _: Boom, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("boom"); } + let _ = reply.reply(()); + } } #[test] @@ -3301,8 +3408,11 @@ mod api_coverage { } } impl Handler for Trapper { - async fn handle(&mut self, msg: Ask, _: &Context) -> u32 { + async fn handle(&mut self, msg: Ask, reply: crate::reply::PendingReply<'_, u32>, _: &Context) { + let __reply_value: u32 = { msg.0 + }; + let _ = reply.reply(__reply_value); } } rt::Runtime::new().unwrap().block_on(async { @@ -3353,7 +3463,9 @@ mod shared_state_release { struct Plain; impl Actor for Plain {} impl Handler for Plain { - async fn handle(&mut self, _: Down, _: &Context) {} + async fn handle(&mut self, _: Down, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } /// Keeps a clone of its own `Context` in its state. diff --git a/concurrency/src/tasks/stream_tests.rs b/concurrency/src/tasks/stream_tests.rs index f7f18376..26622e4c 100644 --- a/concurrency/src/tasks/stream_tests.rs +++ b/concurrency/src/tasks/stream_tests.rs @@ -43,23 +43,32 @@ impl Summatory { impl Actor for Summatory {} impl Handler for Summatory { - async fn handle(&mut self, msg: StreamMsg, ctx: &Context) { + async fn handle(&mut self, msg: StreamMsg, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { match msg { StreamMsg::Add(val) => self.count += val, StreamMsg::Error => ctx.stop(), } } + let _ = reply.reply(()); + } } impl Handler for Summatory { - async fn handle(&mut self, _msg: StopSum, ctx: &Context) { + async fn handle(&mut self, _msg: StopSum, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.stop(); } + let _ = reply.reply(()); + } } impl Handler for Summatory { - async fn handle(&mut self, _msg: GetValue, _ctx: &Context) -> u16 { + async fn handle(&mut self, _msg: GetValue, reply: crate::reply::PendingReply<'_, u16>, _ctx: &Context) { + let __reply_value: u16 = { self.count + }; + let _ = reply.reply(__reply_value); } } diff --git a/concurrency/src/tasks/timer_tests.rs b/concurrency/src/tasks/timer_tests.rs index 40902cab..42ad58a6 100644 --- a/concurrency/src/tasks/timer_tests.rs +++ b/concurrency/src/tasks/timer_tests.rs @@ -46,22 +46,31 @@ impl Actor for Repeater { } impl Handler for Repeater { - async fn handle(&mut self, _msg: Inc, _ctx: &Context) { + async fn handle(&mut self, _msg: Inc, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { self.count += 1; } + let _ = reply.reply(()); + } } impl Handler for Repeater { - async fn handle(&mut self, _msg: StopTimer, _ctx: &Context) { + async fn handle(&mut self, _msg: StopTimer, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { if let Some(ct) = self.cancellation_token.clone() { ct.cancel(); } } + let _ = reply.reply(()); + } } impl Handler for Repeater { - async fn handle(&mut self, _msg: GetRepCount, _ctx: &Context) -> i32 { + async fn handle(&mut self, _msg: GetRepCount, reply: crate::reply::PendingReply<'_, i32>, _ctx: &Context) { + let __reply_value: i32 = { self.count + }; + let _ = reply.reply(__reply_value); } } @@ -122,21 +131,30 @@ impl Delayed { impl Actor for Delayed {} impl Handler for Delayed { - async fn handle(&mut self, _msg: Inc, _ctx: &Context) { + async fn handle(&mut self, _msg: Inc, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { self.count += 1; } + let _ = reply.reply(()); + } } impl Handler for Delayed { - async fn handle(&mut self, _msg: GetDelCount, _ctx: &Context) -> i32 { + async fn handle(&mut self, _msg: GetDelCount, reply: crate::reply::PendingReply<'_, i32>, _ctx: &Context) { + let __reply_value: i32 = { self.count + }; + let _ = reply.reply(__reply_value); } } impl Handler for Delayed { - async fn handle(&mut self, _msg: StopDelayed, ctx: &Context) -> i32 { + async fn handle(&mut self, _msg: StopDelayed, reply: crate::reply::PendingReply<'_, i32>, ctx: &Context) { + let __reply_value: i32 = { ctx.stop(); self.count + }; + let _ = reply.reply(__reply_value); } } diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 11a1ad1a..9714e773 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -101,11 +101,14 @@ where observation.reject(); } } - fn handle(self: Box, actor: &mut A, ctx: &Context) { + fn handle(self: Box, actor: &mut A, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { let mut this = *self; let reply = PendingReply::new(this.slot.take(), &ctx.replies); actor.handle(this.msg, reply, ctx); } + let _ = reply.reply(()); + } } fn enqueue( @@ -1093,22 +1096,31 @@ mod tests { impl Actor for Counter {} impl Handler for Counter { - fn handle(&mut self, _msg: GetCount, _ctx: &Context) -> u64 { + fn handle(&mut self, _msg: GetCount, reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context) { + let __reply_value: u64 = { self.count + }; + let _ = reply.reply(__reply_value); } } impl Handler for Counter { - fn handle(&mut self, _msg: Increment, _ctx: &Context) -> u64 { + fn handle(&mut self, _msg: Increment, reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context) { + let __reply_value: u64 = { self.count += 1; self.count + }; + let _ = reply.reply(__reply_value); } } impl Handler for Counter { - fn handle(&mut self, _msg: StopCounter, ctx: &Context) -> u64 { + fn handle(&mut self, _msg: StopCounter, reply: crate::reply::PendingReply<'_, u64>, ctx: &Context) { + let __reply_value: u64 = { ctx.stop(); self.count + }; + let _ = reply.reply(__reply_value); } } @@ -1136,9 +1148,12 @@ mod tests { } } impl Handler for SlowStop { - fn handle(&mut self, _msg: StopSlow, ctx: &Context) { + fn handle(&mut self, _msg: StopSlow, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.stop(); } + let _ = reply.reply(()); + } } let actor = SlowStop.start(); @@ -1160,9 +1175,12 @@ mod tests { } } impl Handler for SlowStop2 { - fn handle(&mut self, _msg: StopSlow2, ctx: &Context) { + fn handle(&mut self, _msg: StopSlow2, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.stop(); } + let _ = reply.reply(()); + } } let actor = SlowStop2.start(); @@ -1194,7 +1212,9 @@ mod tests { } } impl Handler for PanicOnStart { - fn handle(&mut self, _msg: PingThread, _ctx: &Context) {} + fn handle(&mut self, _msg: PingThread, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + let _ = reply.reply(()); + } } let actor = PanicOnStart.start(); @@ -1216,13 +1236,19 @@ mod tests { } impl Actor for PanicOnMsg {} impl Handler for PanicOnMsg { - fn handle(&mut self, _msg: ExplodeThread, _ctx: &Context) { + fn handle(&mut self, _msg: ExplodeThread, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { panic!("boom in handler"); } + let _ = reply.reply(()); + } } impl Handler for PanicOnMsg { - fn handle(&mut self, _msg: CheckThread, _ctx: &Context) -> u32 { + fn handle(&mut self, _msg: CheckThread, reply: crate::reply::PendingReply<'_, u32>, _ctx: &Context) { + let __reply_value: u32 = { 42 + }; + let _ = reply.reply(__reply_value); } } @@ -1246,9 +1272,12 @@ mod tests { } } impl Handler for PanicOnStop { - fn handle(&mut self, _msg: StopMeThread, ctx: &Context) { + fn handle(&mut self, _msg: StopMeThread, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.stop(); } + let _ = reply.reply(()); + } } let actor = PanicOnStop.start(); @@ -1297,7 +1326,9 @@ mod tests { } } impl Handler for PanicStartThread { - fn handle(&mut self, _msg: PingThread2, _ctx: &Context) {} + fn handle(&mut self, _msg: PingThread2, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + let _ = reply.reply(()); + } } let actor = PanicStartThread.start(); @@ -1314,9 +1345,12 @@ mod tests { } impl Actor for PanicHandlerThread {} impl Handler for PanicHandlerThread { - fn handle(&mut self, _msg: ExplodeThread2, _ctx: &Context) { + fn handle(&mut self, _msg: ExplodeThread2, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { panic!("boom in handler"); } + let _ = reply.reply(()); + } } let actor = PanicHandlerThread.start(); @@ -1357,26 +1391,38 @@ mod tests { impl Actor for Watcher {} impl Handler for Watcher { - fn handle(&mut self, msg: StartMonitor, ctx: &Context) -> crate::monitor::MonitorRef { + fn handle(&mut self, msg: StartMonitor, reply: crate::reply::PendingReply<'_, crate::monitor::MonitorRef>, ctx: &Context) { + let __reply_value: crate::monitor::MonitorRef = { ctx.monitor(&msg.0) + }; + let _ = reply.reply(__reply_value); } } impl Handler for Watcher { - fn handle(&mut self, msg: CallDemonitor, ctx: &Context) { + fn handle(&mut self, msg: CallDemonitor, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.demonitor(msg.0); } + let _ = reply.reply(()); + } } impl Handler for Watcher { - fn handle(&mut self, msg: crate::monitor::Down, _ctx: &Context) { + fn handle(&mut self, msg: crate::monitor::Down, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { self.downs.lock().unwrap().push(msg); } + let _ = reply.reply(()); + } } impl Handler for Watcher { - fn handle(&mut self, _msg: GetDowns, _ctx: &Context) -> Vec { + fn handle(&mut self, _msg: GetDowns, reply: crate::reply::PendingReply<'_, Vec>, _ctx: &Context) { + let __reply_value: Vec = { self.downs.lock().unwrap().clone() + }; + let _ = reply.reply(__reply_value); } } @@ -1475,9 +1521,12 @@ mod tests { struct PanicMe; impl Actor for PanicMe {} impl Handler for PanicMe { - fn handle(&mut self, _msg: PanicMsg, _ctx: &Context) { + fn handle(&mut self, _msg: PanicMsg, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { panic!("intentional panic"); } + let _ = reply.reply(()); + } } let target = PanicMe.start(); @@ -1522,14 +1571,20 @@ mod tests { } impl Handler for TrapActor { - fn handle(&mut self, msg: LinkTo, ctx: &Context) { + fn handle(&mut self, msg: LinkTo, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.link(&msg.0); } + let _ = reply.reply(()); + } } impl Handler for TrapActor { - fn handle(&mut self, _msg: GetExits, _ctx: &Context) -> Vec { + fn handle(&mut self, _msg: GetExits, reply: crate::reply::PendingReply<'_, Vec>, _ctx: &Context) { + let __reply_value: Vec = { self.exits.lock().unwrap().clone() + }; + let _ = reply.reply(__reply_value); } } @@ -1548,9 +1603,12 @@ mod tests { type Result = (); } impl Handler for TrapActor { - fn handle(&mut self, _msg: Boom, _ctx: &Context) { + fn handle(&mut self, _msg: Boom, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { panic!("boom"); } + let _ = reply.reply(()); + } } let a = make_trapper(false); @@ -1571,9 +1629,12 @@ mod tests { type Result = (); } impl Handler for TrapActor { - fn handle(&mut self, _msg: Boom, _ctx: &Context) { + fn handle(&mut self, _msg: Boom, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { panic!("boom"); } + let _ = reply.reply(()); + } } let a = make_trapper(false); @@ -1606,9 +1667,12 @@ mod tests { } impl Actor for Parent {} impl Handler for Parent { - fn handle(&mut self, _msg: PanicParent, _ctx: &Context) { + fn handle(&mut self, _msg: PanicParent, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { panic!("parent boom"); } + let _ = reply.reply(()); + } } struct Child; @@ -1630,7 +1694,9 @@ mod monitor_cleanup_regressions { struct Idle; impl Actor for Idle {} impl Handler for Idle { - fn handle(&mut self, _: Down, _: &Context) {} + fn handle(&mut self, _: Down, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } #[test] fn closed_monitor_table_rejects_late_registration() { @@ -1740,9 +1806,12 @@ mod idle_stop_regression { type Result = (); } impl Handler for Counter { - fn handle(&mut self, _: Increment, _: &Context) { + fn handle(&mut self, _: Increment, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.fetch_add(1, Ordering::SeqCst); } + let _ = reply.reply(()); + } } #[test] fn stop_while_receiving_does_not_dispatch_a_later_request() { @@ -1828,7 +1897,9 @@ mod stop_wakeup_regression { type Result = (); } impl Handler for Idle { - fn handle(&mut self, _: Noop, _: &Context) {} + fn handle(&mut self, _: Noop, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } #[test] @@ -1906,9 +1977,12 @@ mod stop_wakeup_regression { struct Failing; impl Actor for Failing {} impl Handler for Failing { - fn handle(&mut self, _: Crash, _: &Context) { + fn handle(&mut self, _: Crash, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("linked wakeup regression"); } + let _ = reply.reply(()); + } } #[test] @@ -1962,11 +2036,14 @@ mod stop_wakeup_regression { type Result = (); } impl Handler for Busy { - fn handle(&mut self, _: Work, _: &Context) { + fn handle(&mut self, _: Work, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + { self.handled.fetch_add(1, Ordering::SeqCst); let _ = self.entered.send(()); self.release.recv_timeout(Duration::from_secs(5)).unwrap(); } + let _ = reply.reply(()); + } } #[test] @@ -2038,25 +2115,37 @@ mod api_coverage { } impl Actor for Echo {} impl Handler for Echo { - fn handle(&mut self, msg: Ask, _: &Context) -> u32 { + fn handle(&mut self, msg: Ask, reply: crate::reply::PendingReply<'_, u32>, _: &Context) { + let __reply_value: u32 = { msg.0 + }; + let _ = reply.reply(__reply_value); } } impl Handler for Echo { - fn handle(&mut self, msg: StopAfter, ctx: &Context) { + fn handle(&mut self, msg: StopAfter, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { let _ = msg.0.recv_timeout(WAIT); ctx.stop(); } + let _ = reply.reply(()); + } } impl Handler for Echo { - fn handle(&mut self, _: OwnRecipient, ctx: &Context) -> (Recipient, ActorId) { + fn handle(&mut self, _: OwnRecipient, reply: crate::reply::PendingReply<'_, (Recipient, ActorId)>, ctx: &Context) { + let __reply_value: (Recipient, ActorId) = { (ctx.recipient::(), ctx.id()) + }; + let _ = reply.reply(__reply_value); } } impl Handler for Echo { - fn handle(&mut self, _: Boom, _: &Context) { + fn handle(&mut self, _: Boom, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("boom"); } + let _ = reply.reply(()); + } } #[test] @@ -2133,14 +2222,20 @@ mod api_coverage { } } impl Handler for Trapper { - fn handle(&mut self, msg: Ask, _: &Context) -> u32 { + fn handle(&mut self, msg: Ask, reply: crate::reply::PendingReply<'_, u32>, _: &Context) { + let __reply_value: u32 = { msg.0 + }; + let _ = reply.reply(__reply_value); } } impl Handler for Trapper { - fn handle(&mut self, msg: Unlink, ctx: &Context) { + fn handle(&mut self, msg: Unlink, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.unlink(&msg.0); } + let _ = reply.reply(()); + } } #[test] diff --git a/concurrency/src/threads/stream_tests.rs b/concurrency/src/threads/stream_tests.rs index f5cb01db..8ef180f0 100644 --- a/concurrency/src/threads/stream_tests.rs +++ b/concurrency/src/threads/stream_tests.rs @@ -28,15 +28,21 @@ impl Message for GetItems { impl Actor for Collector {} impl Handler for Collector { - fn handle(&mut self, msg: Push, _ctx: &Context) { + fn handle(&mut self, msg: Push, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { self.items.push(msg.value); self.done.fetch_add(1, Ordering::SeqCst); } + let _ = reply.reply(()); + } } impl Handler for Collector { - fn handle(&mut self, _msg: GetItems, _ctx: &Context) -> Vec { + fn handle(&mut self, _msg: GetItems, reply: crate::reply::PendingReply<'_, Vec>, _ctx: &Context) { + let __reply_value: Vec = { self.items.clone() + }; + let _ = reply.reply(__reply_value); } } diff --git a/concurrency/src/threads/timer_tests.rs b/concurrency/src/threads/timer_tests.rs index 7babcd73..04a76dab 100644 --- a/concurrency/src/threads/timer_tests.rs +++ b/concurrency/src/threads/timer_tests.rs @@ -46,22 +46,31 @@ impl Actor for Repeater { } impl Handler for Repeater { - fn handle(&mut self, _msg: Inc, _ctx: &Context) { + fn handle(&mut self, _msg: Inc, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { self.count += 1; } + let _ = reply.reply(()); + } } impl Handler for Repeater { - fn handle(&mut self, _msg: StopTimer, _ctx: &Context) { + fn handle(&mut self, _msg: StopTimer, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { if let Some(ct) = self.cancellation_token.clone() { ct.cancel(); } } + let _ = reply.reply(()); + } } impl Handler for Repeater { - fn handle(&mut self, _msg: GetRepCount, _ctx: &Context) -> i32 { + fn handle(&mut self, _msg: GetRepCount, reply: crate::reply::PendingReply<'_, i32>, _ctx: &Context) { + let __reply_value: i32 = { self.count + }; + let _ = reply.reply(__reply_value); } } @@ -119,21 +128,30 @@ impl Delayed { impl Actor for Delayed {} impl Handler for Delayed { - fn handle(&mut self, _msg: Inc, _ctx: &Context) { + fn handle(&mut self, _msg: Inc, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + { self.count += 1; } + let _ = reply.reply(()); + } } impl Handler for Delayed { - fn handle(&mut self, _msg: GetDelCount, _ctx: &Context) -> i32 { + fn handle(&mut self, _msg: GetDelCount, reply: crate::reply::PendingReply<'_, i32>, _ctx: &Context) { + let __reply_value: i32 = { self.count + }; + let _ = reply.reply(__reply_value); } } impl Handler for Delayed { - fn handle(&mut self, _msg: StopDelayed, ctx: &Context) -> i32 { + fn handle(&mut self, _msg: StopDelayed, reply: crate::reply::PendingReply<'_, i32>, ctx: &Context) { + let __reply_value: i32 = { ctx.stop(); self.count + }; + let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/actor_scenarios.rs b/concurrency/tests/actor_scenarios.rs index 5d2bd22a..f923cfe4 100644 --- a/concurrency/tests/actor_scenarios.rs +++ b/concurrency/tests/actor_scenarios.rs @@ -37,7 +37,8 @@ mod tasks { pub struct TestActor(pub Worker); impl Actor for TestActor {} impl Handler for TestActor { - async fn handle(&mut self, command: Command, ctx: &Context) -> Reply { + async fn handle(&mut self, command: Command, reply: spawned_concurrency::reply::PendingReply<'_, Reply>, ctx: &Context) { + let __reply_value: Reply = { match command { Command::Work(id) => { self.0 @@ -54,12 +55,17 @@ mod tasks { } Command::Boom => panic!("scenario injected panic"), } + }; + let _ = reply.reply(__reply_value); } } impl Handler for TestActor { - async fn handle(&mut self, event: Down, _: &Context) { + async fn handle(&mut self, event: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { let _ = self.0.events.send(event); } + let _ = reply.reply(()); + } } } @@ -69,7 +75,8 @@ mod threads { pub struct TestActor(pub Worker); impl Actor for TestActor {} impl Handler for TestActor { - fn handle(&mut self, command: Command, ctx: &Context) -> Reply { + fn handle(&mut self, command: Command, reply: spawned_concurrency::reply::PendingReply<'_, Reply>, ctx: &Context) { + let __reply_value: Reply = { match command { Command::Work(id) => { self.0 @@ -86,12 +93,17 @@ mod threads { } Command::Boom => panic!("scenario injected panic"), } + }; + let _ = reply.reply(__reply_value); } } impl Handler for TestActor { - fn handle(&mut self, event: Down, _: &Context) { + fn handle(&mut self, event: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { let _ = self.0.events.send(event); } + let _ = reply.reply(()); + } } } diff --git a/concurrency/tests/backend_experiments.rs b/concurrency/tests/backend_experiments.rs index 068dfb14..7421262e 100644 --- a/concurrency/tests/backend_experiments.rs +++ b/concurrency/tests/backend_experiments.rs @@ -17,22 +17,28 @@ impl Message for Nested { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: Probe, _: &Context) -> u64 { + async fn handle(&mut self, _: Probe, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { let child = tokio::spawn(async { tokio::time::sleep(Duration::from_millis(1)).await; 42 }); child.await.unwrap() + }; + let _ = reply.reply(__reply_value); } } impl Handler for Worker { - async fn handle(&mut self, _: Nested, _: &Context) -> bool { + async fn handle(&mut self, _: Nested, reply: spawned_concurrency::reply::PendingReply<'_, bool>, _: &Context) { + let __reply_value: bool = { let (tx, rx) = mpsc::channel(); tokio::spawn(async move { let _ = tx.send(()); }); // Characterizes progress of a detached helper while its actor blocks. rx.recv_timeout(Duration::from_millis(100)).is_ok() + }; + let _ = reply.reply(__reply_value); } } #[test] @@ -63,7 +69,8 @@ impl Message for Network { type Result = u64; } impl Handler for Worker { - async fn handle(&mut self, _: Network, _: &Context) -> u64 { + async fn handle(&mut self, _: Network, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { use std::io::{Read, Write}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); @@ -83,6 +90,8 @@ impl Handler for Worker { socket.read_exact(&mut data).await.unwrap(); server.join().unwrap(); u64::from_le_bytes(data) + }; + let _ = reply.reply(__reply_value); } } #[test] diff --git a/concurrency/tests/backend_resource_limits.rs b/concurrency/tests/backend_resource_limits.rs index 6df9f59f..facaf343 100644 --- a/concurrency/tests/backend_resource_limits.rs +++ b/concurrency/tests/backend_resource_limits.rs @@ -30,7 +30,9 @@ impl Actor for Worker { } } impl Handler for Worker { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } // Not simulated: Blocking backend workers are OS threads. #[cfg(not(spawned_sim))] diff --git a/concurrency/tests/borrowed_runtime.rs b/concurrency/tests/borrowed_runtime.rs index 14bed951..3b53d59d 100644 --- a/concurrency/tests/borrowed_runtime.rs +++ b/concurrency/tests/borrowed_runtime.rs @@ -20,13 +20,19 @@ impl Message for Ambient { } impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: Identity, _: &Context) -> ThreadId { + async fn handle(&mut self, _: Identity, reply: spawned_concurrency::reply::PendingReply<'_, ThreadId>, _: &Context) { + let __reply_value: ThreadId = { std::thread::current().id() + }; + let _ = reply.reply(__reply_value); } } impl Handler for Worker { - async fn handle(&mut self, _: Ambient, _: &Context) -> ActorRef { + async fn handle(&mut self, _: Ambient, reply: spawned_concurrency::reply::PendingReply<'_, ActorRef>, _: &Context) { + let __reply_value: ActorRef = { Worker.start() + }; + let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/canary_feedback.rs b/concurrency/tests/canary_feedback.rs index 20f58821..94ca3e29 100644 --- a/concurrency/tests/canary_feedback.rs +++ b/concurrency/tests/canary_feedback.rs @@ -33,15 +33,21 @@ impl Message for Quit { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, hold: Hold, _: &Context) { + async fn handle(&mut self, hold: Hold, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { let _ = hold.entered.send(()); let _ = hold.release.await; } + let _ = reply.reply(()); + } } impl Handler for Worker { - async fn handle(&mut self, quit: Quit, ctx: &Context) { + async fn handle(&mut self, quit: Quit, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.stop_with_reason(quit.0); } + let _ = reply.reply(()); + } } #[spawned_macros::sim_test] @@ -394,9 +400,12 @@ async fn panicked_turn_is_cleared_before_stopped_hook() { } } impl Handler for Panicking { - async fn handle(&mut self, _: Quit, _: &Context) { + async fn handle(&mut self, _: Quit, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("diagnostic turn cleanup regression"); } + let _ = reply.reply(()); + } } let system = ActorSystem::new().unwrap(); let group = system.async_group("canary-panic", 1).unwrap(); diff --git a/concurrency/tests/cancellation_safety.rs b/concurrency/tests/cancellation_safety.rs index 20311750..648a3891 100644 --- a/concurrency/tests/cancellation_safety.rs +++ b/concurrency/tests/cancellation_safety.rs @@ -189,7 +189,8 @@ impl Message for Cmd { type Result = u64; } impl Handler for Probe { - async fn handle(&mut self, cmd: Cmd, ctx: &Context) -> u64 { + async fn handle(&mut self, cmd: Cmd, reply: spawned_concurrency::reply::PendingReply<'_, u64>, ctx: &Context) { + let __reply_value: u64 = { let c = &self.counters; match cmd { Cmd::Bump => c.effects.fetch_add(1, Ordering::SeqCst) as u64 + 1, @@ -216,12 +217,17 @@ impl Handler for Probe { 0 } } + }; + let _ = reply.reply(__reply_value); } } impl Handler for Probe { - async fn handle(&mut self, _: Down, _: &Context) { + async fn handle(&mut self, _: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.counters.downs.fetch_add(1, Ordering::SeqCst); } + let _ = reply.reply(()); + } } #[protocol] @@ -229,8 +235,11 @@ pub trait Bumper: Send + Sync { fn bump(&self) -> Response; } impl Handler for Probe { - async fn handle(&mut self, _: bumper::Bump, _: &Context) -> u64 { + async fn handle(&mut self, _: bumper::Bump, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { self.counters.effects.fetch_add(1, Ordering::SeqCst) as u64 + 1 + }; + let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/cfg_gated_protocol.rs b/concurrency/tests/cfg_gated_protocol.rs index e1045d25..57ebf5f8 100644 --- a/concurrency/tests/cfg_gated_protocol.rs +++ b/concurrency/tests/cfg_gated_protocol.rs @@ -23,20 +23,22 @@ struct MixedActor { impl Actor for MixedActor {} impl Handler for MixedActor { - async fn handle(&mut self, _msg: mixed_protocol::GetValue, _ctx: &Context) -> u64 { + async fn handle(&mut self, _msg: mixed_protocol::GetValue, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _ctx: &Context) { + let __reply_value: u64 = { self.value + }; + let _ = reply.reply(__reply_value); } } #[cfg(test)] impl Handler for MixedActor { - async fn handle( - &mut self, - _msg: mixed_protocol::TestOnlyIncrement, - _ctx: &Context, - ) -> u64 { + async fn handle(&mut self, _msg: mixed_protocol::TestOnlyIncrement, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _ctx: &Context) { + let __reply_value: u64 = { self.value += 1; self.value + }; + let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/cohort_trace.rs b/concurrency/tests/cohort_trace.rs index 5dc9aba1..5bfaeca6 100644 --- a/concurrency/tests/cohort_trace.rs +++ b/concurrency/tests/cohort_trace.rs @@ -15,9 +15,12 @@ impl Message for Crash { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("trace cohort failure"); } + let _ = reply.reply(()); + } } fn write_trace(name: &str, id: u64, strategy: SupervisorStrategy, modes: &[&str]) { let trace = lifecycle_trace::take(id); @@ -268,9 +271,12 @@ impl Actor for MaybeStop { } } impl Handler for MaybeStop { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("trace cohort failure"); } + let _ = reply.reply(()); + } } /// A replacement stopping before readiness inside a round requests a new /// round: OneForAll rebuilds the whole cohort again. `Cohorts.tla` allows one diff --git a/concurrency/tests/context_exit.rs b/concurrency/tests/context_exit.rs index b2da17ac..c6954c7b 100644 --- a/concurrency/tests/context_exit.rs +++ b/concurrency/tests/context_exit.rs @@ -20,10 +20,13 @@ fn native_explicit_exit_is_cooperative_and_first_reason_wins() { } } impl Handler for Worker { - fn handle(&mut self, exit: Exit, ctx: &Context) { + fn handle(&mut self, exit: Exit, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.stop_with_reason(exit.0); ctx.stop(); } + let _ = reply.reply(()); + } } for reason in [ExitReason::Shutdown, ExitReason::Kill] { let cleaned = Arc::new(AtomicBool::new(false)); @@ -50,10 +53,13 @@ async fn async_explicit_exit_is_cooperative_and_first_reason_wins() { } } impl Handler for Worker { - async fn handle(&mut self, exit: Exit, ctx: &Context) { + async fn handle(&mut self, exit: Exit, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.stop_with_reason(exit.0); ctx.stop(); } + let _ = reply.reply(()); + } } for reason in [ExitReason::Shutdown, ExitReason::Kill] { let cleaned = Arc::new(AtomicBool::new(false)); diff --git a/concurrency/tests/coverage_trace.rs b/concurrency/tests/coverage_trace.rs index 3042ee2d..42c41448 100644 --- a/concurrency/tests/coverage_trace.rs +++ b/concurrency/tests/coverage_trace.rs @@ -91,9 +91,12 @@ impl Message for Crash { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("injected child failure"); } + let _ = reply.reply(()); + } } fn worker(name: &str, group: &spawned_concurrency::system::ExecutionGroup) -> ChildSpec { @@ -290,9 +293,12 @@ impl Actor for QuitInStarted { } } impl Handler for QuitInStarted { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("injected child failure"); } + let _ = reply.reply(()); + } } /// A ready child crashes and restarts while a later sibling is still being diff --git a/concurrency/tests/cross_backend.rs b/concurrency/tests/cross_backend.rs index c928fdb9..e437e201 100644 --- a/concurrency/tests/cross_backend.rs +++ b/concurrency/tests/cross_backend.rs @@ -156,7 +156,8 @@ mod tasks_node { } } impl Handler for Node { - async fn handle(&mut self, cmd: Cmd, ctx: &Context) -> Reply { + async fn handle(&mut self, cmd: Cmd, reply: spawned_concurrency::reply::PendingReply<'_, Reply>, ctx: &Context) { + let __reply_value: Reply = async { match cmd { Cmd::Echo(v) => self.state.echo(v), Cmd::Chain(v) => match self.peers.first() { @@ -215,18 +216,25 @@ mod tasks_node { } } } + .await; + let _ = reply.reply(__reply_value); + } } impl Handler for Node { - async fn handle(&mut self, down: Down, _: &Context) { + async fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.state.log.push(Obs::Down { observer: self.state.index, monitor: down.monitor_ref, reason: down.reason, }); } + let _ = reply.reply(()); + } } impl Handler for Node { - async fn handle(&mut self, Tick(n): Tick, ctx: &Context) { + async fn handle(&mut self, Tick(n): Tick, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + { self.state.ticks_seen += 1; self.state.log.push(Obs::Tick { node: self.state.index, @@ -236,9 +244,12 @@ mod tasks_node { ctx.stop(); } } + let _ = reply.reply(()); + } } impl Handler for Node { - async fn handle(&mut self, _: Pulse, ctx: &Context) { + async fn handle(&mut self, _: Pulse, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + let __reply_value: () = async { if self.state.pulses_seen == self.state.pulses_wanted { return; // A tick already in flight when the interval was cancelled. } @@ -255,6 +266,9 @@ mod tasks_node { ctx.stop(); } } + .await; + let _ = reply.reply(__reply_value); + } } } @@ -281,7 +295,8 @@ mod threads_node { } } impl Handler for Node { - fn handle(&mut self, cmd: Cmd, ctx: &Context) -> Reply { + fn handle(&mut self, cmd: Cmd, reply: spawned_concurrency::reply::PendingReply<'_, Reply>, ctx: &Context) { + let __reply_value: Reply = (|| { match cmd { Cmd::Echo(v) => self.state.echo(v), Cmd::Chain(v) => match self.peers.first() { @@ -341,19 +356,25 @@ mod threads_node { Reply::Unit } } + })(); + let _ = reply.reply(__reply_value); } } impl Handler for Node { - fn handle(&mut self, down: Down, _: &Context) { + fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.state.log.push(Obs::Down { observer: self.state.index, monitor: down.monitor_ref, reason: down.reason, }); } + let _ = reply.reply(()); + } } impl Handler for Node { - fn handle(&mut self, Tick(n): Tick, ctx: &Context) { + fn handle(&mut self, Tick(n): Tick, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + { self.state.ticks_seen += 1; self.state.log.push(Obs::Tick { node: self.state.index, @@ -363,9 +384,12 @@ mod threads_node { ctx.stop(); } } + let _ = reply.reply(()); + } } impl Handler for Node { - fn handle(&mut self, _: Pulse, ctx: &Context) { + fn handle(&mut self, _: Pulse, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + let __reply_value: () = (|| { if self.state.pulses_seen == self.state.pulses_wanted { return; } @@ -381,6 +405,8 @@ mod threads_node { if self.state.timers_done() { ctx.stop(); } + })(); + let _ = reply.reply(__reply_value); } } } @@ -1281,11 +1307,14 @@ mod early_stop { } } impl spawned_concurrency::tasks::Handler for Hooks { - async fn handle(&mut self, block: Block, _: &spawned_concurrency::tasks::Context) { + async fn handle(&mut self, block: Block, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &spawned_concurrency::tasks::Context) { + { // Deliberately blocks the only async worker of the group. let _ = block.1.send(()); let _ = block.0.lock().unwrap().recv_timeout(LIMIT); } + let _ = reply.reply(()); + } } impl spawned_concurrency::threads::Actor for Hooks { fn started(&mut self, _: &spawned_concurrency::threads::Context) { diff --git a/concurrency/tests/dedicated_runtime.rs b/concurrency/tests/dedicated_runtime.rs index e91d20f9..64dc2c3d 100644 --- a/concurrency/tests/dedicated_runtime.rs +++ b/concurrency/tests/dedicated_runtime.rs @@ -92,7 +92,8 @@ impl Message for Inspect { type Result = Observation; } impl Handler for Probe { - async fn handle(&mut self, operation: Inspect, _: &Context) -> Observation { + async fn handle(&mut self, operation: Inspect, reply: spawned_concurrency::reply::PendingReply<'_, Observation>, _: &Context) { + let __reply_value: Observation = { let root = self.assert_root(); let handle = tokio::runtime::Handle::current(); let workers = handle.metrics().num_workers(); @@ -137,6 +138,8 @@ impl Handler for Probe { workers, flavor, } + }; + let _ = reply.reply(__reply_value); } } @@ -147,9 +150,12 @@ impl Message for Ping { struct Child; impl Actor for Child {} impl Handler for Child { - async fn handle(&mut self, _: Ping, _: &Context) -> u64 { + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { tokio::time::sleep(Duration::from_millis(1)).await; 42 + }; + let _ = reply.reply(__reply_value); } } struct Nested; @@ -157,7 +163,8 @@ impl Message for Nested { type Result = u64; } impl Handler for Probe { - async fn handle(&mut self, _: Nested, _: &Context) -> u64 { + async fn handle(&mut self, _: Nested, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { self.assert_root(); // Default Async startup uses this actor's isolated runtime. let child = Child.start(); @@ -167,6 +174,8 @@ impl Handler for Probe { tokio::time::timeout(LIMIT, child.join()).await.unwrap(); self.assert_root(); answer + }; + let _ = reply.reply(__reply_value); } } @@ -175,7 +184,8 @@ impl Message for Exchange { type Result = u64; } impl Handler for Probe { - async fn handle(&mut self, message: Exchange, _: &Context) -> u64 { + async fn handle(&mut self, message: Exchange, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { use tokio::io::{AsyncReadExt, AsyncWriteExt}; self.assert_root(); let answer = tokio::time::timeout(LIMIT, async { @@ -189,6 +199,8 @@ impl Handler for Probe { .expect("dedicated runtime I/O stalled"); self.assert_root(); answer + }; + let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/execution_groups.rs b/concurrency/tests/execution_groups.rs index 5ab8ac0f..01c55ddc 100644 --- a/concurrency/tests/execution_groups.rs +++ b/concurrency/tests/execution_groups.rs @@ -23,15 +23,20 @@ impl Message for Ping { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: Ping, _: &Context) -> String { + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, String>, _: &Context) { + let __reply_value: String = { std::thread::current() .name() .unwrap_or_default() .to_string() + }; + let _ = reply.reply(__reply_value); } } impl Handler for Worker { - async fn handle(&mut self, _: Down, _: &Context) {} + async fn handle(&mut self, _: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } // Not simulated: asserts OS worker thread names. @@ -338,10 +343,13 @@ impl Actor for FailedStartup { } } impl Handler for FailedStartup { - async fn handle(&mut self, msg: Queued, _: &Context) { + async fn handle(&mut self, msg: Queued, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.handled.fetch_add(1, Ordering::SeqCst); drop(msg.0); } + let _ = reply.reply(()); + } } #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] async fn failed_startup_discards_queued_payloads_and_reply_channels_without_dispatch() { @@ -562,8 +570,11 @@ impl Message for CheckAdmission { struct HelperAware; impl Actor for HelperAware {} impl Handler for HelperAware { - async fn handle(&mut self, _: CheckAdmission, ctx: &Context) -> bool { + async fn handle(&mut self, _: CheckAdmission, reply: spawned_concurrency::reply::PendingReply<'_, bool>, ctx: &Context) { + let __reply_value: bool = { ctx.spawn_helper(async {}).is_err() + }; + let _ = reply.reply(__reply_value); } } struct Dependent { diff --git a/concurrency/tests/execution_profiles.rs b/concurrency/tests/execution_profiles.rs index fac66149..d08ef09b 100644 --- a/concurrency/tests/execution_profiles.rs +++ b/concurrency/tests/execution_profiles.rs @@ -22,12 +22,15 @@ impl Message for Audit { type Result = (String, Option); } impl Handler for Probe { - async fn handle(&mut self, _: Audit, _: &Context) -> ::Result { + async fn handle(&mut self, _: Audit, reply: spawned_concurrency::reply::PendingReply<'_, ::Result>, _: &Context) { + let __reply_value: ::Result = { spawned_rt::tasks::sleep(Duration::from_millis(1)).await; ( std::thread::current().name().unwrap_or_default().into(), spawned_rt::tasks::affinity::current_cpu().ok(), ) + }; + let _ = reply.reply(__reply_value); } } struct StartPeer; @@ -35,12 +38,11 @@ impl Message for StartPeer { type Result = Result, SystemError>; } impl Handler for Probe { - async fn handle( - &mut self, - _: StartPeer, - ctx: &Context, - ) -> ::Result { + async fn handle(&mut self, _: StartPeer, reply: spawned_concurrency::reply::PendingReply<'_, ::Result>, ctx: &Context) { + let __reply_value: ::Result = { ctx.execution_group().unwrap().start(Probe) + }; + let _ = reply.reply(__reply_value); } } // Not simulated: native execution groups need OS threads. @@ -83,21 +85,27 @@ impl Message for Block { type Result = (); } impl Handler for Probe { - async fn handle(&mut self, msg: Block, _: &Context) { + async fn handle(&mut self, msg: Block, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { tokio::task::block_in_place(|| { let _ = msg.entered.send(()); msg.release.recv_timeout(LIMIT).unwrap(); }); } + let _ = reply.reply(()); + } } struct Crash; impl Message for Crash { type Result = (); } impl Handler for Probe { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("profile recovery injection"); } + let _ = reply.reply(()); + } } // Not simulated: native execution groups need OS threads. #[cfg(not(spawned_sim))] @@ -344,9 +352,12 @@ mod tls_join { type Result = (); } impl Handler for Probe { - async fn handle(&mut self, msg: Install, _: &Context) { + async fn handle(&mut self, msg: Install, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { PROBE.with(|p| *p.borrow_mut() = Some(msg.0)); } + let _ = reply.reply(()); + } } #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] async fn shutdown_reports_wait_for_actual_pinned_worker_tls_destruction() { diff --git a/concurrency/tests/exit_signal_matrix.rs b/concurrency/tests/exit_signal_matrix.rs index 559b96c4..d32ad6cb 100644 --- a/concurrency/tests/exit_signal_matrix.rs +++ b/concurrency/tests/exit_signal_matrix.rs @@ -28,7 +28,9 @@ macro_rules! actors { struct Idle; impl Actor for Idle {} impl Handler for Idle { - $($asynchronous)? fn handle(&mut self, _: Ping, _: &Context) {} + $($asynchronous)? fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } struct Observer(mpsc::Sender); impl Actor for Observer { @@ -37,12 +39,17 @@ macro_rules! actors { } } impl Handler for Observer { - $($asynchronous)? fn handle(&mut self, _: Ping, _: &Context) {} + $($asynchronous)? fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } impl Handler for Observer { - $($asynchronous)? fn handle(&mut self, down: Down, _: &Context) { + $($asynchronous)? fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.send(Notice::Down(down.reason)).unwrap(); } + let _ = reply.reply(()); + } } }; } diff --git a/concurrency/tests/failpoints.rs b/concurrency/tests/failpoints.rs index 5142132b..f86cda9c 100644 --- a/concurrency/tests/failpoints.rs +++ b/concurrency/tests/failpoints.rs @@ -93,12 +93,17 @@ impl Actor for Worker { } } impl Handler for Worker { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("injected child failure"); } + let _ = reply.reply(()); + } } impl Handler for Worker { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } // ---------------------------------------------------------------- child.factory @@ -648,22 +653,33 @@ struct Linker { } impl Actor for Linker {} impl Handler for Linker { - async fn handle(&mut self, msg: LinkTo, ctx: &Context) { + async fn handle(&mut self, msg: LinkTo, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.link(&msg.0); } + let _ = reply.reply(()); + } } impl Handler for Linker { - async fn handle(&mut self, msg: Watch, ctx: &Context) { + async fn handle(&mut self, msg: Watch, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.monitor(&msg.0); } + let _ = reply.reply(()); + } } impl Handler for Linker { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } impl Handler for Linker { - async fn handle(&mut self, msg: Down, _: &Context) { + async fn handle(&mut self, msg: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { let _ = self.downs.send(msg.reason); } + let _ = reply.reply(()); + } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/concurrency/tests/foreign_panics.rs b/concurrency/tests/foreign_panics.rs index f9c4e66f..74e26dd5 100644 --- a/concurrency/tests/foreign_panics.rs +++ b/concurrency/tests/foreign_panics.rs @@ -182,25 +182,37 @@ mod task_actors { } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.on_crash(); } + let _ = reply.reply(()); + } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Ping, _: &Context) -> u64 { + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { 7 + }; + let _ = reply.reply(__reply_value); } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Gift, _: &Context) -> Bomb { + async fn handle(&mut self, _: Gift, reply: spawned_concurrency::reply::PendingReply<'_, Bomb>, _: &Context) { + let __reply_value: Bomb = { self.0.gate.wait(); Bomb(Some(REPLY_PANIC)) + }; + let _ = reply.reply(__reply_value); } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Stash, _: &Context) { + async fn handle(&mut self, _: Stash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.stash.push(Bomb(Some(REPLY_PANIC))); } + let _ = reply.reply(()); + } } } @@ -217,25 +229,37 @@ mod thread_actors { } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Crash, _: &Context) { + fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.on_crash(); } + let _ = reply.reply(()); + } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Ping, _: &Context) -> u64 { + fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { 7 + }; + let _ = reply.reply(__reply_value); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Gift, _: &Context) -> Bomb { + fn handle(&mut self, _: Gift, reply: spawned_concurrency::reply::PendingReply<'_, Bomb>, _: &Context) { + let __reply_value: Bomb = { self.0.gate.wait(); Bomb(Some(REPLY_PANIC)) + }; + let _ = reply.reply(__reply_value); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Stash, _: &Context) { + fn handle(&mut self, _: Stash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.stash.push(Bomb(Some(REPLY_PANIC))); } + let _ = reply.reply(()); + } } } diff --git a/concurrency/tests/full_actor_shards.rs b/concurrency/tests/full_actor_shards.rs index ba6bd413..09bbe042 100644 --- a/concurrency/tests/full_actor_shards.rs +++ b/concurrency/tests/full_actor_shards.rs @@ -33,11 +33,14 @@ impl Message for Thread { type Result = String; } impl Handler for Worker { - async fn handle(&mut self, _: Thread, _: &Context) -> String { + async fn handle(&mut self, _: Thread, reply: spawned_concurrency::reply::PendingReply<'_, String>, _: &Context) { + let __reply_value: String = { std::thread::current() .name() .unwrap_or("unnamed") .to_owned() + }; + let _ = reply.reply(__reply_value); } } struct Hold { @@ -48,10 +51,13 @@ impl Message for Hold { type Result = u64; } impl Handler for Worker { - async fn handle(&mut self, msg: Hold, _: &Context) -> u64 { + async fn handle(&mut self, msg: Hold, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { let _ = msg.started.send(()); msg.release.await.unwrap(); 42 + }; + let _ = reply.reply(__reply_value); } } struct Nested(std::net::SocketAddr); @@ -59,7 +65,8 @@ impl Message for Nested { type Result = u64; } impl Handler for Worker { - async fn handle(&mut self, msg: Nested, _: &Context) -> u64 { + async fn handle(&mut self, msg: Nested, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { use tokio::io::{AsyncReadExt, AsyncWriteExt}; let child = Worker(Default::default()).start(); let _ = child.request(Thread).await.unwrap(); @@ -77,6 +84,8 @@ impl Handler for Worker { let mut reply = [0; 8]; socket.read_exact(&mut reply).await.unwrap(); u64::from_le_bytes(reply) + }; + let _ = reply.reply(__reply_value); } } fn runtime() -> tokio::runtime::Runtime { @@ -239,10 +248,13 @@ impl Message for Freeze { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, msg: Freeze, _: &Context) { + async fn handle(&mut self, msg: Freeze, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { let _ = msg.started.send(()); msg.release.recv_timeout(LIMIT).unwrap(); } + let _ = reply.reply(()); + } } #[test] fn blocking_handler_stalls_its_shard_while_another_shard_progresses() { @@ -315,9 +327,12 @@ impl Message for Tick { type Result = u64; } impl Handler for Worker { - async fn handle(&mut self, _: Tick, _: &Context) -> u64 { + async fn handle(&mut self, _: Tick, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { tokio::time::sleep(Duration::from_millis(1)).await; 42 + }; + let _ = reply.reply(__reply_value); } } struct SpawnBlockingChild; @@ -325,12 +340,11 @@ impl Message for SpawnBlockingChild { type Result = spawned_concurrency::tasks::ActorRef; } impl Handler for Worker { - async fn handle( - &mut self, - _: SpawnBlockingChild, - _: &Context, - ) -> spawned_concurrency::tasks::ActorRef { + async fn handle(&mut self, _: SpawnBlockingChild, reply: spawned_concurrency::reply::PendingReply<'_, spawned_concurrency::tasks::ActorRef>, _: &Context) { + let __reply_value: spawned_concurrency::tasks::ActorRef = { Worker(Default::default()).start_with_backend(spawned_concurrency::tasks::Backend::Blocking) + }; + let _ = reply.reply(__reply_value); } } struct Helper(tokio::sync::oneshot::Receiver<()>); @@ -338,12 +352,15 @@ impl Message for Helper { type Result = tokio::task::JoinHandle; } impl Handler for Worker { - async fn handle(&mut self, msg: Helper, _: &Context) -> tokio::task::JoinHandle { + async fn handle(&mut self, msg: Helper, reply: spawned_concurrency::reply::PendingReply<'_, tokio::task::JoinHandle>, _: &Context) { + let __reply_value: tokio::task::JoinHandle = { tokio::spawn(async move { msg.0.await.unwrap(); tokio::time::sleep(Duration::from_millis(1)).await; 42 }) + }; + let _ = reply.reply(__reply_value); } } #[test] @@ -382,22 +399,31 @@ impl Message for InstallMonitor { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, msg: InstallMonitor, _: &Context) { + async fn handle(&mut self, msg: InstallMonitor, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { msg.observer.monitor(&msg.target); } + let _ = reply.reply(()); + } } impl Handler for Worker { - async fn handle(&mut self, _: spawned_concurrency::Down, _: &Context) { + async fn handle(&mut self, _: spawned_concurrency::Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.lock().unwrap().push("down"); } + let _ = reply.reply(()); + } } struct Events; impl Message for Events { type Result = Vec<&'static str>; } impl Handler for Worker { - async fn handle(&mut self, _: Events, _: &Context) -> Vec<&'static str> { + async fn handle(&mut self, _: Events, reply: spawned_concurrency::reply::PendingReply<'_, Vec<&'static str>>, _: &Context) { + let __reply_value: Vec<&'static str> = { self.0.lock().unwrap().clone() + }; + let _ = reply.reply(__reply_value); } } #[test] diff --git a/concurrency/tests/future_layout.rs b/concurrency/tests/future_layout.rs index 12fd0432..fc92932a 100644 --- a/concurrency/tests/future_layout.rs +++ b/concurrency/tests/future_layout.rs @@ -23,11 +23,16 @@ trait ProbeProtocol: Send + Sync { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: probe_protocol::Notify, _: &Context) {} + async fn handle(&mut self, _: probe_protocol::Notify, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } impl Handler for Worker { - async fn handle(&mut self, query: probe_protocol::Query, _: &Context) -> u64 { + async fn handle(&mut self, query: probe_protocol::Query, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { query.value + }; + let _ = reply.reply(__reply_value); } } const BASELINE_WORDS: usize = 1; diff --git a/concurrency/tests/kill_links.rs b/concurrency/tests/kill_links.rs index 0b77a6db..ebcc4cc1 100644 --- a/concurrency/tests/kill_links.rs +++ b/concurrency/tests/kill_links.rs @@ -22,8 +22,11 @@ fn native_propagated_kill_is_trappable_but_direct_kill_is_not() { } } impl Handler for Node { - fn handle(&mut self, _: Ping, _: &Context) -> u8 { + fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, u8>, _: &Context) { + let __reply_value: u8 = { 7 + }; + let _ = reply.reply(__reply_value); } } fn wait(actor: &ActorRef) -> ExitReason { @@ -78,8 +81,11 @@ async fn async_propagated_kill_is_trappable_but_direct_kill_is_not() { } } impl Handler for Node { - async fn handle(&mut self, _: Ping, _: &Context) -> u8 { + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, u8>, _: &Context) { + let __reply_value: u8 = { 7 + }; + let _ = reply.reply(__reply_value); } } let (send, events) = mpsc::channel(); diff --git a/concurrency/tests/leaks.rs b/concurrency/tests/leaks.rs index 4491a0ca..b44f6396 100644 --- a/concurrency/tests/leaks.rs +++ b/concurrency/tests/leaks.rs @@ -64,7 +64,9 @@ impl threads::Actor for NativeProbe { } } impl threads::Handler for NativeProbe { - fn handle(&mut self, _: Ping, _: &threads::Context) {} + fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context) { + let _ = reply.reply(()); + } } #[test] @@ -102,14 +104,20 @@ struct NativeB { impl threads::Actor for NativeA {} impl threads::Actor for NativeB {} impl threads::Handler>> for NativeA { - fn handle(&mut self, msg: SetPeer>, _: &threads::Context) { + fn handle(&mut self, msg: SetPeer>, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context) { + { self.peer = Some(msg.0); } + let _ = reply.reply(()); + } } impl threads::Handler>> for NativeB { - fn handle(&mut self, msg: SetPeer>, _: &threads::Context) { + fn handle(&mut self, msg: SetPeer>, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context) { + { self.peer = Some(msg.0); } + let _ = reply.reply(()); + } } #[test] @@ -199,19 +207,26 @@ mod async_leaks { } } impl Handler for Probe { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } struct Crash; impl Message for Crash { type Result = (); } impl Handler for Probe { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("injected failure for leak test"); } + let _ = reply.reply(()); + } } impl Handler for Probe { - async fn handle(&mut self, _: Down, _: &Context) {} + async fn handle(&mut self, _: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } // Not simulated: counts Tokio runtime tasks and OS threads. @@ -299,30 +314,32 @@ mod async_leaks { impl Actor for AsyncA {} impl Actor for AsyncB {} impl Handler for AsyncA { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } impl Handler for AsyncB { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } impl Handler, tasks::Recipient)>> for AsyncA { - async fn handle( - &mut self, - msg: SetPeer<(ActorRef, tasks::Recipient)>, - _: &Context, - ) { + async fn handle(&mut self, msg: SetPeer<(ActorRef, tasks::Recipient)>, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.peer = Some(msg.0 .0); self.recipient = Some(msg.0 .1); } + let _ = reply.reply(()); + } } impl Handler, tasks::Recipient)>> for AsyncB { - async fn handle( - &mut self, - msg: SetPeer<(ActorRef, tasks::Recipient)>, - _: &Context, - ) { + async fn handle(&mut self, msg: SetPeer<(ActorRef, tasks::Recipient)>, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.peer = Some(msg.0 .0); self.recipient = Some(msg.0 .1); } + let _ = reply.reply(()); + } } // Not simulated: counts Tokio runtime tasks and OS threads. @@ -569,10 +586,14 @@ mod sim_leaks { } impl Actor for Probe {} impl Handler for Probe { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } impl Handler for Probe { - async fn handle(&mut self, _: Down, _: &Context) {} + async fn handle(&mut self, _: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } /// A holds B's ref and recipient, B holds A's: a reference cycle through @@ -588,28 +609,30 @@ mod sim_leaks { impl Actor for A {} impl Actor for B {} impl Handler for A { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } impl Handler for B { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } impl Handler, tasks::Recipient)>> for A { - async fn handle( - &mut self, - msg: SetPeer<(tasks::ActorRef, tasks::Recipient)>, - _: &Context, - ) { + async fn handle(&mut self, msg: SetPeer<(tasks::ActorRef, tasks::Recipient)>, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.peer = Some(msg.0); } + let _ = reply.reply(()); + } } impl Handler, tasks::Recipient)>> for B { - async fn handle( - &mut self, - msg: SetPeer<(tasks::ActorRef, tasks::Recipient)>, - _: &Context, - ) { + async fn handle(&mut self, msg: SetPeer<(tasks::ActorRef, tasks::Recipient)>, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.peer = Some(msg.0); } + let _ = reply.reply(()); + } } #[test] @@ -739,9 +762,12 @@ mod sim_leaks { type Result = (); } impl Handler for Probe { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("injected failure for leak test"); } + let _ = reply.reply(()); + } } /// A monitor whose target exits delivers `Down` and releases its waiter. @@ -847,8 +873,11 @@ mod sim_leaks { } } impl Handler for Named { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("injected failure for leak test"); } + let _ = reply.reply(()); + } } } diff --git a/concurrency/tests/mailbox_drop_panic.rs b/concurrency/tests/mailbox_drop_panic.rs index baa64cc4..939c5f79 100644 --- a/concurrency/tests/mailbox_drop_panic.rs +++ b/concurrency/tests/mailbox_drop_panic.rs @@ -42,16 +42,22 @@ impl Drop for Victim { } impl Actor for Victim {} impl Handler for Victim { - async fn handle(&mut self, msg: Hold, _: &Context) { + async fn handle(&mut self, msg: Hold, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { msg.0.send(()).unwrap(); let _ = msg.1.await; assert!(!msg.2, "original handler failure"); } + let _ = reply.reply(()); + } } impl Handler for Victim { - async fn handle(&mut self, _: Bomb, _: &Context) { + async fn handle(&mut self, _: Bomb, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.fetch_add(1, Ordering::SeqCst); } + let _ = reply.reply(()); + } } struct Ping; impl Message for Ping { @@ -64,12 +70,17 @@ impl Actor for Observer { } } impl Handler for Observer { - async fn handle(&mut self, down: Down, _: &Context) { + async fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.send((false, down.reason)).unwrap(); } + let _ = reply.reply(()); + } } impl Handler for Observer { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } // Not simulated: builds its own multi-thread Tokio runtime. diff --git a/concurrency/tests/metrics.rs b/concurrency/tests/metrics.rs index a3615e4e..b3383304 100644 --- a/concurrency/tests/metrics.rs +++ b/concurrency/tests/metrics.rs @@ -39,23 +39,34 @@ impl Actor for Probe { } } impl Handler for Probe { - async fn handle(&mut self, _: metrics_protocol::Query, _: &Context) -> u64 { + async fn handle(&mut self, _: metrics_protocol::Query, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { 9 + }; + let _ = reply.reply(__reply_value); } } impl Handler for Probe { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } impl Handler for Probe { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("metric panic regression"); } + let _ = reply.reply(()); + } } impl Handler for Probe { - async fn handle(&mut self, hold: Hold, _: &Context) { + async fn handle(&mut self, hold: Hold, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { let _ = hold.entered.send(()); let _ = hold.release.await; } + let _ = reply.reply(()); + } } fn runtime() -> tokio::runtime::Runtime { tokio::runtime::Builder::new_current_thread() diff --git a/concurrency/tests/metrics_composable.rs b/concurrency/tests/metrics_composable.rs index c6f25309..fcf98653 100644 --- a/concurrency/tests/metrics_composable.rs +++ b/concurrency/tests/metrics_composable.rs @@ -13,7 +13,9 @@ impl Message for Ping { } impl Actor for Probe {} impl Handler for Probe { - fn handle(&mut self, _: Ping, _: &Context) {} + fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } #[test] diff --git a/concurrency/tests/metrics_counts_only.rs b/concurrency/tests/metrics_counts_only.rs index 7108f5a7..00575813 100644 --- a/concurrency/tests/metrics_counts_only.rs +++ b/concurrency/tests/metrics_counts_only.rs @@ -11,7 +11,9 @@ impl Message for Ping { } impl Actor for Probe {} impl Handler for Probe { - fn handle(&mut self, _: Ping, _: &Context) {} + fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } #[test] fn zero_timing_rate_retains_counters_without_timing_samples() { diff --git a/concurrency/tests/metrics_sampling.rs b/concurrency/tests/metrics_sampling.rs index 23e4b1f4..ba74acf8 100644 --- a/concurrency/tests/metrics_sampling.rs +++ b/concurrency/tests/metrics_sampling.rs @@ -12,7 +12,9 @@ impl Message for Ping { } impl Actor for Probe {} impl Handler for Probe { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } #[test] fn sampling_retains_every_counter_and_scales_only_timing_observations() { diff --git a/concurrency/tests/native_dispatcher.rs b/concurrency/tests/native_dispatcher.rs index d8803972..8fbc4057 100644 --- a/concurrency/tests/native_dispatcher.rs +++ b/concurrency/tests/native_dispatcher.rs @@ -16,9 +16,12 @@ impl Message for Ping { struct Healthy; impl Actor for Healthy {} impl Handler for Healthy { - async fn handle(&mut self, _: Ping, _: &Context) -> u64 { + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { tokio::task::yield_now().await; 42 + }; + let _ = reply.reply(__reply_value); } } struct DropPanic; diff --git a/concurrency/tests/native_mailbox_drop.rs b/concurrency/tests/native_mailbox_drop.rs index 6b987c0b..25c7e2f2 100644 --- a/concurrency/tests/native_mailbox_drop.rs +++ b/concurrency/tests/native_mailbox_drop.rs @@ -40,9 +40,12 @@ impl Drop for Victim { } } impl Handler for Victim { - fn handle(&mut self, _: Bomb, _: &Context) { + fn handle(&mut self, _: Bomb, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("queued handler ran"); } + let _ = reply.reply(()); + } } struct Crash; @@ -50,9 +53,12 @@ impl Message for Crash { type Result = (); } impl Handler for Victim { - fn handle(&mut self, _: Crash, _: &Context) { + fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("native handler failure"); } + let _ = reply.reply(()); + } } #[test] diff --git a/concurrency/tests/nested_supervision.rs b/concurrency/tests/nested_supervision.rs index 004c6312..b56b1256 100644 --- a/concurrency/tests/nested_supervision.rs +++ b/concurrency/tests/nested_supervision.rs @@ -44,15 +44,21 @@ impl Message for Crash { struct Counter(usize); impl Actor for Counter {} impl Handler for Counter { - async fn handle(&mut self, _: Next, _: &Context) -> usize { + async fn handle(&mut self, _: Next, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { + let __reply_value: usize = { self.0 += 1; self.0 + }; + let _ = reply.reply(__reply_value); } } impl Handler for Counter { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("nested leaf failed"); } + let _ = reply.reply(()); + } } type Published = Arc>>>; @@ -517,9 +523,12 @@ impl Actor for MaybeStop { } } impl Handler for MaybeStop { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("nested leaf failed"); } + let _ = reply.reply(()); + } } /// A Transient leaf's failed replacement (stops before readiness) is retried diff --git a/concurrency/tests/panic_safety.rs b/concurrency/tests/panic_safety.rs index 1b4fa519..bcbafdbf 100644 --- a/concurrency/tests/panic_safety.rs +++ b/concurrency/tests/panic_safety.rs @@ -195,30 +195,45 @@ mod tasks_actors { } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Poke, _: &Context) { + async fn handle(&mut self, _: Poke, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.fault(Fault::SendHandler, "injected handler panic"); } + let _ = reply.reply(()); + } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Ask, _: &Context) -> u64 { + async fn handle(&mut self, _: Ask, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { self.0.ask() + }; + let _ = reply.reply(__reply_value); } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Payload, _: &Context) {} + async fn handle(&mut self, _: Payload, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Gift, _: &Context) -> Bomb { + async fn handle(&mut self, _: Gift, reply: spawned_concurrency::reply::PendingReply<'_, Bomb>, _: &Context) { + let __reply_value: Bomb = { self.0.gift() + }; + let _ = reply.reply(__reply_value); } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Hold, _: &Context) { + async fn handle(&mut self, _: Hold, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.gate.wait(); } + let _ = reply.reply(()); + } } impl Handler for TaskVictim { - async fn handle(&mut self, setup: Setup, ctx: &Context) -> Option { + async fn handle(&mut self, setup: Setup, reply: spawned_concurrency::reply::PendingReply<'_, Option>, ctx: &Context) { + let __reply_value: Option = async { match setup { Setup::Trap => ctx.trap_exit(true), Setup::Link(target) => ctx.link(&target), @@ -226,11 +241,17 @@ mod tasks_actors { } None } + .await; + let _ = reply.reply(__reply_value); + } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Down, _: &Context) { + async fn handle(&mut self, _: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.fault(Fault::DownHandler, "injected Down panic"); } + let _ = reply.reply(()); + } } pub struct TaskObserver(pub Observer); @@ -243,14 +264,18 @@ mod tasks_actors { } } impl Handler for TaskObserver { - async fn handle(&mut self, down: Down, _: &Context) { + async fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0 .events .push(Event::Down(down.monitor_ref, down.reason)); } + let _ = reply.reply(()); + } } impl Handler for TaskObserver { - async fn handle(&mut self, setup: Setup, ctx: &Context) -> Option { + async fn handle(&mut self, setup: Setup, reply: spawned_concurrency::reply::PendingReply<'_, Option>, ctx: &Context) { + let __reply_value: Option = async { match setup { Setup::Trap => ctx.trap_exit(true), Setup::Link(target) => ctx.link(&target), @@ -258,10 +283,16 @@ mod tasks_actors { } None } + .await; + let _ = reply.reply(__reply_value); + } } impl Handler for TaskObserver { - async fn handle(&mut self, _: Ask, _: &Context) -> u64 { + async fn handle(&mut self, _: Ask, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { 7 + }; + let _ = reply.reply(__reply_value); } } } @@ -286,42 +317,62 @@ mod thread_actors { } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Poke, _: &Context) { + fn handle(&mut self, _: Poke, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.fault(Fault::SendHandler, "injected handler panic"); } + let _ = reply.reply(()); + } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Ask, _: &Context) -> u64 { + fn handle(&mut self, _: Ask, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { self.0.ask() + }; + let _ = reply.reply(__reply_value); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Payload, _: &Context) {} + fn handle(&mut self, _: Payload, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Gift, _: &Context) -> Bomb { + fn handle(&mut self, _: Gift, reply: spawned_concurrency::reply::PendingReply<'_, Bomb>, _: &Context) { + let __reply_value: Bomb = { self.0.gift() + }; + let _ = reply.reply(__reply_value); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Hold, _: &Context) { + fn handle(&mut self, _: Hold, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.gate.wait(); } + let _ = reply.reply(()); + } } impl Handler for ThreadVictim { - fn handle(&mut self, setup: Setup, ctx: &Context) -> Option { + fn handle(&mut self, setup: Setup, reply: spawned_concurrency::reply::PendingReply<'_, Option>, ctx: &Context) { + let __reply_value: Option = (|| { match setup { Setup::Trap => ctx.trap_exit(true), Setup::Link(target) => ctx.link(&target), Setup::Watch(target) => return Some(ctx.monitor(&target)), } None + })(); + let _ = reply.reply(__reply_value); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Down, _: &Context) { + fn handle(&mut self, _: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.fault(Fault::DownHandler, "injected Down panic"); } + let _ = reply.reply(()); + } } pub struct ThreadObserver(pub Observer); @@ -334,25 +385,34 @@ mod thread_actors { } } impl Handler for ThreadObserver { - fn handle(&mut self, down: Down, _: &Context) { + fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0 .events .push(Event::Down(down.monitor_ref, down.reason)); } + let _ = reply.reply(()); + } } impl Handler for ThreadObserver { - fn handle(&mut self, setup: Setup, ctx: &Context) -> Option { + fn handle(&mut self, setup: Setup, reply: spawned_concurrency::reply::PendingReply<'_, Option>, ctx: &Context) { + let __reply_value: Option = (|| { match setup { Setup::Trap => ctx.trap_exit(true), Setup::Link(target) => ctx.link(&target), Setup::Watch(target) => return Some(ctx.monitor(&target)), } None + })(); + let _ = reply.reply(__reply_value); } } impl Handler for ThreadObserver { - fn handle(&mut self, _: Ask, _: &Context) -> u64 { + fn handle(&mut self, _: Ask, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { 7 + }; + let _ = reply.reply(__reply_value); } } } diff --git a/concurrency/tests/process_failures.rs b/concurrency/tests/process_failures.rs index 64812b6c..fcd68314 100644 --- a/concurrency/tests/process_failures.rs +++ b/concurrency/tests/process_failures.rs @@ -30,7 +30,8 @@ impl Drop for Fixture { } } impl Handler for Fixture { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { std::fs::write( self.path.join("effect"), b"effect happened before process failure", @@ -43,6 +44,8 @@ impl Handler for Fixture { std::thread::park(); } } + let _ = reply.reply(()); + } } #[test] #[ignore = "only invoked by the isolated parent test"] diff --git a/concurrency/tests/protocol_timeouts.rs b/concurrency/tests/protocol_timeouts.rs index c9f49142..68ab15b0 100644 --- a/concurrency/tests/protocol_timeouts.rs +++ b/concurrency/tests/protocol_timeouts.rs @@ -58,21 +58,30 @@ mod asynchronous { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, msg: timed_protocol::Work, _: &Context) -> u64 { + async fn handle(&mut self, msg: timed_protocol::Work, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; msg.millis + }; + let _ = reply.reply(__reply_value); } } impl Handler for Worker { - async fn handle(&mut self, msg: timed_protocol::Brief, _: &Context) -> u64 { + async fn handle(&mut self, msg: timed_protocol::Brief, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; msg.millis + }; + let _ = reply.reply(__reply_value); } } impl Handler for Worker { - async fn handle(&mut self, msg: legacy_protocol::Legacy, _: &Context) -> u64 { + async fn handle(&mut self, msg: legacy_protocol::Legacy, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; msg.millis + }; + let _ = reply.reply(__reply_value); } } // Tokio's paused clock normally; the simulator's clock is always virtual. @@ -130,15 +139,21 @@ mod synchronous { struct Worker; impl Actor for Worker {} impl Handler for Worker { - fn handle(&mut self, msg: timed_protocol::Work, _: &Context) -> u64 { + fn handle(&mut self, msg: timed_protocol::Work, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { std::thread::sleep(Duration::from_millis(msg.millis)); msg.millis + }; + let _ = reply.reply(__reply_value); } } impl Handler for Worker { - fn handle(&mut self, msg: timed_protocol::Brief, _: &Context) -> u64 { + fn handle(&mut self, msg: timed_protocol::Brief, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { std::thread::sleep(Duration::from_millis(msg.millis)); msg.millis + }; + let _ = reply.reply(__reply_value); } } #[test] @@ -167,26 +182,40 @@ mod synchronous { } } impl Handler for Held { - fn handle(&mut self, _: zero_protocol::Zero, _: &Context) {} + fn handle(&mut self, _: zero_protocol::Zero, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } impl Handler for Worker { - fn handle(&mut self, _: naming_protocol::Timeout, _: &Context) -> u64 { + fn handle(&mut self, _: naming_protocol::Timeout, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { 31 + }; + let _ = reply.reply(__reply_value); } } impl Handler for Worker { - fn handle(&mut self, _: naming_protocol::ActorRef, _: &Context) -> u64 { + fn handle(&mut self, _: naming_protocol::ActorRef, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { 32 + }; + let _ = reply.reply(__reply_value); } } impl Handler for Worker { - fn handle(&mut self, _: naming_protocol::TimeoutOverride, _: &Context) -> u64 { + fn handle(&mut self, _: naming_protocol::TimeoutOverride, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { 33 + }; + let _ = reply.reply(__reply_value); } } impl Handler for Worker { - fn handle(&mut self, _: naming_protocol::Dispatch0, _: &Context) -> u64 { + fn handle(&mut self, _: naming_protocol::Dispatch0, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { 34 + }; + let _ = reply.reply(__reply_value); } } #[test] @@ -205,8 +234,11 @@ mod synchronous { actor.join(); } impl Handler for Worker { - fn handle(&mut self, msg: owned_protocol::Owned, _: &Context) -> u64 { + fn handle(&mut self, msg: owned_protocol::Owned, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { msg.__spawned_request_timeout.value + }; + let _ = reply.reply(__reply_value); } } #[test] diff --git a/concurrency/tests/resource_cleanup.rs b/concurrency/tests/resource_cleanup.rs index 1642777d..5f849034 100644 --- a/concurrency/tests/resource_cleanup.rs +++ b/concurrency/tests/resource_cleanup.rs @@ -15,15 +15,21 @@ struct Sink; impl tasks::Actor for Sink {} #[cfg(feature = "async")] impl tasks::Handler for Sink { - async fn handle(&mut self, msg: Probe, _: &tasks::Context) { + async fn handle(&mut self, msg: Probe, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &tasks::Context) { + { drop(msg.0); } + let _ = reply.reply(()); + } } impl threads::Actor for Sink {} impl threads::Handler for Sink { - fn handle(&mut self, msg: Probe, _: &threads::Context) { + fn handle(&mut self, msg: Probe, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context) { + { drop(msg.0); } + let _ = reply.reply(()); + } } fn released(weak: &Weak<()>) { let deadline = Instant::now() + Duration::from_secs(10); diff --git a/concurrency/tests/semantics_gaps.rs b/concurrency/tests/semantics_gaps.rs index 9a6b70ff..29b835fe 100644 --- a/concurrency/tests/semantics_gaps.rs +++ b/concurrency/tests/semantics_gaps.rs @@ -101,27 +101,37 @@ mod native { } } impl Handler for Observer { - fn handle(&mut self, watch: Watch, ctx: &Context) { + fn handle(&mut self, watch: Watch, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.trap_exit(true); ctx.link(&watch.0); ctx.monitor(&watch.0); } + let _ = reply.reply(()); + } } impl Handler for Observer { - fn handle(&mut self, ping: Ping, _: &Context) { + fn handle(&mut self, ping: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { let _ = self.0.send(Event::Ping(ping.0)); } + let _ = reply.reply(()); + } } impl Handler for Observer { - fn handle(&mut self, down: Down, _: &Context) { + fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { let _ = self.0.send(Event::Down(down.reason)); } + let _ = reply.reply(()); + } } struct Source; impl Actor for Source {} impl Handler>> for Source { - fn handle(&mut self, burst: Burst>, ctx: &Context) { + fn handle(&mut self, burst: Burst>, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + { for i in 0..burst.count { burst.to.send(Ping(i)).unwrap(); } @@ -130,6 +140,8 @@ mod native { None => panic!("source panicked"), } } + let _ = reply.reply(()); + } } #[test] @@ -163,8 +175,11 @@ mod native { struct Panicker; impl Actor for Panicker {} impl Handler for Panicker { - fn handle(&mut self, _: Boom, _: &Context) -> u64 { + fn handle(&mut self, _: Boom, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { panic!("request handler panic") + }; + let _ = reply.reply(__reply_value); } } @@ -228,27 +243,37 @@ mod tasks { } } impl Handler for Observer { - async fn handle(&mut self, watch: Watch, ctx: &Context) { + async fn handle(&mut self, watch: Watch, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.trap_exit(true); ctx.link(&watch.0); ctx.monitor(&watch.0); } + let _ = reply.reply(()); + } } impl Handler for Observer { - async fn handle(&mut self, ping: Ping, _: &Context) { + async fn handle(&mut self, ping: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { let _ = self.0.send(Event::Ping(ping.0)); } + let _ = reply.reply(()); + } } impl Handler for Observer { - async fn handle(&mut self, down: Down, _: &Context) { + async fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { let _ = self.0.send(Event::Down(down.reason)); } + let _ = reply.reply(()); + } } struct Source; impl Actor for Source {} impl Handler>> for Source { - async fn handle(&mut self, burst: Burst>, ctx: &Context) { + async fn handle(&mut self, burst: Burst>, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + { for i in 0..burst.count { burst.to.send(Ping(i)).unwrap(); } @@ -257,6 +282,8 @@ mod tasks { None => panic!("source panicked"), } } + let _ = reply.reply(()); + } } const BACKENDS: [Backend; 3] = [Backend::Async, Backend::Blocking, Backend::Thread]; @@ -300,9 +327,12 @@ mod tasks { struct Ticker(UnboundedSender); impl Actor for Ticker {} impl Handler for Ticker { - async fn handle(&mut self, _: Tick, _: &Context) { + async fn handle(&mut self, _: Tick, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { let _ = self.0.send(Instant::now()); } + let _ = reply.reply(()); + } } const PERIOD: Duration = Duration::from_millis(100); @@ -384,14 +414,20 @@ mod tasks { } } impl Handler for Clocked { - async fn handle(&mut self, stamp: Stamp, _: &Context) { + async fn handle(&mut self, stamp: Stamp, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { let _ = self.log.send((self.generation, stamp.0)); } + let _ = reply.reply(()); + } } impl Handler for Clocked { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("injected crash"); } + let _ = reply.reply(()); + } } /// T3: the first generation's timer never delivers to its replacement. @@ -465,8 +501,11 @@ mod tasks { struct Panicker; impl Actor for Panicker {} impl Handler for Panicker { - async fn handle(&mut self, _: Boom, _: &Context) -> u64 { + async fn handle(&mut self, _: Boom, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { panic!("request handler panic") + }; + let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/shutdown_publication.rs b/concurrency/tests/shutdown_publication.rs index 041c7eb8..80a5d3c2 100644 --- a/concurrency/tests/shutdown_publication.rs +++ b/concurrency/tests/shutdown_publication.rs @@ -66,7 +66,8 @@ impl Actor for Worker { } } impl Handler for Worker { - async fn handle(&mut self, msg: Command, _: &Context) { + async fn handle(&mut self, msg: Command, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { match msg { Command::Hold { entered, release } => { let _ = entered.send(()); @@ -75,6 +76,8 @@ impl Handler for Worker { Command::Work => panic!("queued work must be discarded after stop"), } } + let _ = reply.reply(()); + } } struct Cleanup(ChildHandle); impl Drop for Cleanup { diff --git a/concurrency/tests/sim.rs b/concurrency/tests/sim.rs index cac1ec64..1c2b776d 100644 --- a/concurrency/tests/sim.rs +++ b/concurrency/tests/sim.rs @@ -85,9 +85,12 @@ impl Actor for Node { } } impl Handler for Node { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("injected crash after readiness"); } + let _ = reply.reply(()); + } } /// One simulated run: outcome, lifecycle log and the scheduler's trace. diff --git a/concurrency/tests/sim_explore.rs b/concurrency/tests/sim_explore.rs index f6af8020..ba9eaf16 100644 --- a/concurrency/tests/sim_explore.rs +++ b/concurrency/tests/sim_explore.rs @@ -56,20 +56,29 @@ impl Message for Log { type Result = Vec; } impl Handler for Store { - async fn handle(&mut self, _: Get, _: &Context) -> u64 { + async fn handle(&mut self, _: Get, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { self.log.push(Access::Read(self.value)); self.value + }; + let _ = reply.reply(__reply_value); } } impl Handler for Store { - async fn handle(&mut self, Set(value): Set, _: &Context) { + async fn handle(&mut self, Set(value): Set, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.log.push(Access::Write(value)); self.value = value; } + let _ = reply.reply(()); + } } impl Handler for Store { - async fn handle(&mut self, _: Log, _: &Context) -> Vec { + async fn handle(&mut self, _: Log, reply: spawned_concurrency::reply::PendingReply<'_, Vec>, _: &Context) { + let __reply_value: Vec = { self.log.clone() + }; + let _ = reply.reply(__reply_value); } } @@ -87,13 +96,16 @@ impl Message for Bump { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _: Bump, _: &Context) { + async fn handle(&mut self, _: Bump, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { for _ in 0..self.noise { spawned_rt::tasks::yield_now().await; } let value = self.store.request(Get).await.unwrap(); self.store.request(Set(value + 1)).await.unwrap(); } + let _ = reply.reply(()); + } } /// Workers with the given noise bump the store once each, concurrently. @@ -200,9 +212,12 @@ impl Actor for Node { } } impl Handler for Node { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("injected crash after readiness"); } + let _ = reply.reply(()); + } } /// States (`Debug` of the state, generation) of three OneForAll children diff --git a/concurrency/tests/stack_budget.rs b/concurrency/tests/stack_budget.rs index e7c423eb..13eaf7f3 100644 --- a/concurrency/tests/stack_budget.rs +++ b/concurrency/tests/stack_budget.rs @@ -117,13 +117,16 @@ mod tasks_chain { } impl Actor for Hop {} impl Handler for Hop { - async fn handle(&mut self, msg: relay::Forward, _: &Context) -> Block { + async fn handle(&mut self, msg: relay::Forward, reply: spawned_concurrency::reply::PendingReply<'_, Block>, _: &Context) { + let __reply_value: Block = { let mut block = msg.block; stamp(&mut block, self.index); match &self.next { Some(next) => next.forward(block).await.expect("downstream hop"), None => block, } + }; + let _ = reply.reply(__reply_value); } } @@ -215,15 +218,21 @@ mod tasks_chain { } impl Actor for Flaky {} impl Handler for Flaky { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { if self.generation == 1 { panic!("injected stack-budget crash"); } } + let _ = reply.reply(()); + } } impl Handler for Flaky { - async fn handle(&mut self, msg: relay::Forward, _: &Context) -> Block { + async fn handle(&mut self, msg: relay::Forward, reply: spawned_concurrency::reply::PendingReply<'_, Block>, _: &Context) { + let __reply_value: Block = { msg.block + }; + let _ = reply.reply(__reply_value); } } @@ -295,13 +304,16 @@ mod threads_chain { } impl Actor for Hop {} impl Handler for Hop { - fn handle(&mut self, msg: relay::Forward, _: &Context) -> Block { + fn handle(&mut self, msg: relay::Forward, reply: spawned_concurrency::reply::PendingReply<'_, Block>, _: &Context) { + let __reply_value: Block = { let mut block = msg.block; stamp(&mut block, self.index); match &self.next { Some(next) => next.forward(block).unwrap(), None => block, } + }; + let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/startup_admission.rs b/concurrency/tests/startup_admission.rs index 10d2f7e2..bb767e77 100644 --- a/concurrency/tests/startup_admission.rs +++ b/concurrency/tests/startup_admission.rs @@ -33,9 +33,12 @@ impl Drop for Queued { } } impl spawned_concurrency::tasks::Handler for Probe { - async fn handle(&mut self, _: Queued, _: &Context) { + async fn handle(&mut self, _: Queued, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("unstarted handler ran"); } + let _ = reply.reply(()); + } } impl Actor for Probe { async fn started(&mut self, _: &Context) { diff --git a/concurrency/tests/supervision.rs b/concurrency/tests/supervision.rs index c152594d..3caf4b7d 100644 --- a/concurrency/tests/supervision.rs +++ b/concurrency/tests/supervision.rs @@ -40,15 +40,21 @@ impl Message for Crash { struct Counter(usize); impl Actor for Counter {} impl Handler for Counter { - async fn handle(&mut self, _: Next, _: &Context) -> usize { + async fn handle(&mut self, _: Next, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { + let __reply_value: usize = { self.0 += 1; self.0 + }; + let _ = reply.reply(__reply_value); } } impl Handler for Counter { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("injected child failure"); } + let _ = reply.reply(()); + } } #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] @@ -275,9 +281,12 @@ impl Actor for SlowHelper { } } impl Handler for SlowHelper { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("injected failure with active helper"); } + let _ = reply.reply(()); + } } // Not simulated: a helper blocks an OS worker on purpose. #[cfg(not(spawned_sim))] @@ -342,9 +351,12 @@ impl Actor for SlowStop { } } impl Handler for SlowStop { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("failure before stuck stopped hook"); } + let _ = reply.reply(()); + } } // Not simulated: a helper blocks an OS worker on purpose. #[cfg(not(spawned_sim))] @@ -527,14 +539,20 @@ impl Drop for RaceActor { } impl Actor for RaceActor {} impl Handler for RaceActor { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("racing child failure"); } + let _ = reply.reply(()); + } } impl Handler for RaceActor { - async fn handle(&mut self, _: Next, _: &Context) -> usize { + async fn handle(&mut self, _: Next, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { + let __reply_value: usize = { self.count += 1; self.count + }; + let _ = reply.reply(__reply_value); } } #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] @@ -817,9 +835,12 @@ impl Actor for ExitOnceDuringRestart { } } impl Handler for ExitOnceDuringRestart { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("injected child failure"); } + let _ = reply.reply(()); + } } /// A Transient child's *replacement* that stops `Normal` or `Shutdown` @@ -1286,9 +1307,12 @@ impl Actor for Named { } } impl Handler for Named { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("crash a registered child"); } + let _ = reply.reply(()); + } } /// A generation's owned name registration is released before its diff --git a/concurrency/tests/supervision_strategies.rs b/concurrency/tests/supervision_strategies.rs index 1cf5842f..7c79985c 100644 --- a/concurrency/tests/supervision_strategies.rs +++ b/concurrency/tests/supervision_strategies.rs @@ -70,21 +70,30 @@ impl Drop for Probe { } } impl Handler for Probe { - async fn handle(&mut self, _: Next, _: &Context) -> usize { + async fn handle(&mut self, _: Next, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { + let __reply_value: usize = { self.value += 1; self.value + }; + let _ = reply.reply(__reply_value); } } impl Handler for Probe { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("injected cohort failure"); } + let _ = reply.reply(()); + } } impl Handler for Probe { - async fn handle(&mut self, msg: CrashTogether, _: &Context) { + async fn handle(&mut self, msg: CrashTogether, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { msg.0.wait().await; panic!("simultaneous cohort failure"); } + let _ = reply.reply(()); + } } #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] @@ -705,7 +714,10 @@ impl Actor for MaybeStop { } } impl Handler for MaybeStop { - async fn handle(&mut self, _: CrashPlain, _: &Context) { + async fn handle(&mut self, _: CrashPlain, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("injected cohort failure"); } + let _ = reply.reply(()); + } } diff --git a/concurrency/tests/supervision_trace.rs b/concurrency/tests/supervision_trace.rs index 171e4a29..a9bef362 100644 --- a/concurrency/tests/supervision_trace.rs +++ b/concurrency/tests/supervision_trace.rs @@ -98,10 +98,13 @@ impl Message for HoldForDrain { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, msg: HoldForDrain, _: &Context) { + async fn handle(&mut self, msg: HoldForDrain, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { msg.0.send(()).unwrap(); let _ = msg.1.await; } + let _ = reply.reply(()); + } } struct DropBomb; impl Message for DropBomb { @@ -113,9 +116,12 @@ impl Drop for DropBomb { } } impl Handler for Worker { - async fn handle(&mut self, _: DropBomb, _: &Context) { + async fn handle(&mut self, _: DropBomb, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("discarded message must not be handled"); } + let _ = reply.reply(()); + } } #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] @@ -197,19 +203,28 @@ async fn trace_queued_destructor_failure_preserves_shutdown() { assert!(system.shutdown(LIMIT).await.complete); } impl Handler for Worker { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("injected child failure"); } + let _ = reply.reply(()); + } } impl Handler for Worker { - async fn handle(&mut self, _: Quit, ctx: &Context) { + async fn handle(&mut self, _: Quit, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.stop(); } + let _ = reply.reply(()); + } } impl Handler for Worker { - async fn handle(&mut self, msg: ExitWith, ctx: &Context) { + async fn handle(&mut self, msg: ExitWith, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.stop_with_reason(msg.0); } + let _ = reply.reply(()); + } } #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] @@ -498,9 +513,12 @@ impl Actor for ExitOnceDuringRestart { } } impl Handler for ExitOnceDuringRestart { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("injected child failure"); } + let _ = reply.reply(()); + } } /// A replacement that stops before readiness while the supervisor is open: diff --git a/concurrency/tests/supervisor_policies.rs b/concurrency/tests/supervisor_policies.rs index ea810f94..3c0a0b2b 100644 --- a/concurrency/tests/supervisor_policies.rs +++ b/concurrency/tests/supervisor_policies.rs @@ -25,14 +25,20 @@ impl Message for Crash { struct Probe(u64); impl Actor for Probe {} impl Handler for Probe { - async fn handle(&mut self, _: Generation, _: &Context) -> u64 { + async fn handle(&mut self, _: Generation, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { self.0 + }; + let _ = reply.reply(__reply_value); } } impl Handler for Probe { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("generated supervisor failure"); } + let _ = reply.reply(()); + } } #[derive(Clone, Debug)] struct Expected { diff --git a/concurrency/tests/support/failure_actor.rs b/concurrency/tests/support/failure_actor.rs index a7db6149..0dda354d 100644 --- a/concurrency/tests/support/failure_actor.rs +++ b/concurrency/tests/support/failure_actor.rs @@ -156,11 +156,8 @@ impl spawned_concurrency::tasks::Actor for AsyncWorker { } #[cfg(feature = "async")] impl spawned_concurrency::tasks::Handler for AsyncWorker { - async fn handle( - &mut self, - command: Command, - _: &spawned_concurrency::tasks::Context, - ) -> Reply { + async fn handle(&mut self, command: Command, reply: spawned_concurrency::reply::PendingReply<'_, Reply>, _: &spawned_concurrency::tasks::Context) { + let __reply_value: Reply = { match command { Command::Payload { id, payload } => { assert_eq!(payload.bytes[0], (id % 251) as u8); @@ -185,15 +182,20 @@ impl spawned_concurrency::tasks::Handler for AsyncWorker { result } } + }; + let _ = reply.reply(__reply_value); } } #[cfg(feature = "async")] impl spawned_concurrency::tasks::Handler for AsyncWorker { - async fn handle(&mut self, down: Down, _: &spawned_concurrency::tasks::Context) { + async fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &spawned_concurrency::tasks::Context) { + { if let Some(sender) = &self.0.down { let _ = sender.send(down); } } + let _ = reply.reply(()); + } } pub struct NativeWorker(pub Worker); impl spawned_concurrency::threads::Actor for NativeWorker { @@ -210,11 +212,8 @@ impl spawned_concurrency::threads::Actor for NativeWorker { } } impl spawned_concurrency::threads::Handler for NativeWorker { - fn handle( - &mut self, - command: Command, - _: &spawned_concurrency::threads::Context, - ) -> Reply { + fn handle(&mut self, command: Command, reply: spawned_concurrency::reply::PendingReply<'_, Reply>, _: &spawned_concurrency::threads::Context) { + let __reply_value: Reply = { match command { Command::Payload { id, payload } => { assert_eq!(payload.bytes[0], (id % 251) as u8); @@ -239,14 +238,19 @@ impl spawned_concurrency::threads::Handler for NativeWorker { result } } + }; + let _ = reply.reply(__reply_value); } } impl spawned_concurrency::threads::Handler for NativeWorker { - fn handle(&mut self, down: Down, _: &spawned_concurrency::threads::Context) { + fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &spawned_concurrency::threads::Context) { + { if let Some(sender) = &self.0.down { let _ = sender.send(down); } } + let _ = reply.reply(()); + } } #[derive(Clone)] pub enum Handle { diff --git a/concurrency/tests/support/store_actor.rs b/concurrency/tests/support/store_actor.rs index 4d5b3091..6ca90d2e 100644 --- a/concurrency/tests/support/store_actor.rs +++ b/concurrency/tests/support/store_actor.rs @@ -112,16 +112,15 @@ impl spawned_concurrency::tasks::Actor for AsyncStore {} #[cfg(feature = "async")] impl spawned_concurrency::tasks::Handler for AsyncStore { - async fn handle( - &mut self, - op: StoreOp, - _: &spawned_concurrency::tasks::Context, - ) -> Option { + async fn handle(&mut self, op: StoreOp, reply: spawned_concurrency::reply::PendingReply<'_, Option>, _: &spawned_concurrency::tasks::Context) { + let __reply_value: Option = { // Yield inside the turn so concurrent requests really queue up. if op.key().1.is_multiple_of(2) { tokio::task::yield_now().await; } self.0.apply(op) + }; + let _ = reply.reply(__reply_value); } } @@ -130,12 +129,11 @@ pub struct NativeStore(pub Store); impl spawned_concurrency::threads::Actor for NativeStore {} impl spawned_concurrency::threads::Handler for NativeStore { - fn handle( - &mut self, - op: StoreOp, - _: &spawned_concurrency::threads::Context, - ) -> Option { + fn handle(&mut self, op: StoreOp, reply: spawned_concurrency::reply::PendingReply<'_, Option>, _: &spawned_concurrency::threads::Context) { + let __reply_value: Option = { self.0.apply(op) + }; + let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/tree_trace.rs b/concurrency/tests/tree_trace.rs index a9f8d323..4c17a5a3 100644 --- a/concurrency/tests/tree_trace.rs +++ b/concurrency/tests/tree_trace.rs @@ -20,9 +20,12 @@ impl Message for Crash { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("nested trace failure"); } + let _ = reply.reply(()); + } } fn config(strategy: SupervisorStrategy) -> SupervisorConfig { SupervisorConfig { diff --git a/examples/capabilities/src/bin/byte_budget_pipeline.rs b/examples/capabilities/src/bin/byte_budget_pipeline.rs index 86447c6c..dab3eced 100644 --- a/examples/capabilities/src/bin/byte_budget_pipeline.rs +++ b/examples/capabilities/src/bin/byte_budget_pipeline.rs @@ -18,13 +18,16 @@ impl Message for Packet { struct Sink(usize); impl Actor for Sink {} impl Handler for Sink { - async fn handle(&mut self, packet: Packet, _: &Context) -> usize { + async fn handle(&mut self, packet: Packet, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { + let __reply_value: usize = { let Packet { bytes, credit } = packet; let size = bytes.len(); self.0 += size; drop(bytes); drop(credit); // Capacity returns after the payload is released. size + }; + let _ = reply.reply(__reply_value); } } struct Total; @@ -32,15 +35,21 @@ impl Message for Total { type Result = usize; } impl Handler for Sink { - async fn handle(&mut self, _: Total, _: &Context) -> usize { + async fn handle(&mut self, _: Total, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { + let __reply_value: usize = { self.0 + }; + let _ = reply.reply(__reply_value); } } struct Forwarder(ActorRef); impl Actor for Forwarder {} impl Handler for Forwarder { - async fn handle(&mut self, packet: Packet, _: &Context) -> usize { + async fn handle(&mut self, packet: Packet, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { + let __reply_value: usize = { self.0.request(packet).await.unwrap() + }; + let _ = reply.reply(__reply_value); } } diff --git a/examples/capabilities/src/bin/coordinated_shutdown.rs b/examples/capabilities/src/bin/coordinated_shutdown.rs index bfc1bffe..95abdd59 100644 --- a/examples/capabilities/src/bin/coordinated_shutdown.rs +++ b/examples/capabilities/src/bin/coordinated_shutdown.rs @@ -36,13 +36,19 @@ impl Actor for Peers { } } impl Handler for Peers { - async fn handle(&mut self, msg: Peer, _: &Context) { + async fn handle(&mut self, msg: Peer, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.peers.push(msg.0); } + let _ = reply.reply(()); + } } impl Handler for Peers { - async fn handle(&mut self, _: Snapshot, _: &Context) -> Vec { + async fn handle(&mut self, _: Snapshot, reply: spawned_concurrency::reply::PendingReply<'_, Vec>, _: &Context) { + let __reply_value: Vec = { self.peers.clone() + }; + let _ = reply.reply(__reply_value); } } struct Importer { @@ -55,12 +61,17 @@ impl Actor for Importer { } } impl Handler for Importer { - async fn handle(&mut self, msg: Import, _: &Context) { + async fn handle(&mut self, msg: Import, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.peers.send(Peer(msg.0)).unwrap(); } + let _ = reply.reply(()); + } } impl Handler for Importer { - async fn handle(&mut self, _: Fence, _: &Context) {} + async fn handle(&mut self, _: Fence, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } #[tokio::main(flavor = "current_thread")] async fn main() { diff --git a/examples/capabilities/src/bin/dedicated_thread.rs b/examples/capabilities/src/bin/dedicated_thread.rs index 1604e3aa..d2cc05b5 100644 --- a/examples/capabilities/src/bin/dedicated_thread.rs +++ b/examples/capabilities/src/bin/dedicated_thread.rs @@ -12,7 +12,8 @@ impl Message for Check { type Result = (usize, u64); } impl Handler for Worker { - async fn handle(&mut self, _: Check, _: &Context) -> (usize, u64) { + async fn handle(&mut self, _: Check, reply: spawned_concurrency::reply::PendingReply<'_, (usize, u64)>, _: &Context) { + let __reply_value: (usize, u64) = { let root = thread::current().id(); let workers = tokio::runtime::Handle::current().metrics().num_workers(); let (tx, rx) = mpsc::channel(); @@ -27,6 +28,8 @@ impl Handler for Worker { helper.await.unwrap(); assert_eq!(root, thread::current().id()); (workers, answer) + }; + let _ = reply.reply(__reply_value); } } diff --git a/examples/capabilities/src/bin/execution_groups.rs b/examples/capabilities/src/bin/execution_groups.rs index eb0af643..5ce92750 100644 --- a/examples/capabilities/src/bin/execution_groups.rs +++ b/examples/capabilities/src/bin/execution_groups.rs @@ -12,15 +12,21 @@ impl Message for Double { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, msg: Double, _: &Context) -> u64 { + async fn handle(&mut self, msg: Double, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { msg.0 * 2 + }; + let _ = reply.reply(__reply_value); } } struct Router(ActorRef); impl Actor for Router {} impl Handler for Router { - async fn handle(&mut self, msg: Double, _: &Context) -> u64 { + async fn handle(&mut self, msg: Double, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { self.0.request(msg).await.unwrap() + }; + let _ = reply.reply(__reply_value); } } #[tokio::main(flavor = "current_thread")] diff --git a/examples/capabilities/src/bin/execution_profiles.rs b/examples/capabilities/src/bin/execution_profiles.rs index 493b169d..03e06fe7 100644 --- a/examples/capabilities/src/bin/execution_profiles.rs +++ b/examples/capabilities/src/bin/execution_profiles.rs @@ -12,11 +12,14 @@ impl Message for Location { type Result = (String, Option); } impl Handler for Worker { - async fn handle(&mut self, _: Location, _: &Context) -> ::Result { + async fn handle(&mut self, _: Location, reply: spawned_concurrency::reply::PendingReply<'_, ::Result>, _: &Context) { + let __reply_value: ::Result = { ( std::thread::current().name().unwrap_or_default().to_owned(), spawned_rt::tasks::affinity::current_cpu().ok(), ) + }; + let _ = reply.reply(__reply_value); } } #[tokio::main(flavor = "multi_thread", worker_threads = 2)] diff --git a/examples/capabilities/src/bin/framework_metrics.rs b/examples/capabilities/src/bin/framework_metrics.rs index 089e7c47..e679276f 100644 --- a/examples/capabilities/src/bin/framework_metrics.rs +++ b/examples/capabilities/src/bin/framework_metrics.rs @@ -11,7 +11,9 @@ impl Message for Ping { } impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } #[tokio::main(flavor = "multi_thread", worker_threads = 2)] async fn main() { diff --git a/examples/capabilities/src/bin/graceful_drain.rs b/examples/capabilities/src/bin/graceful_drain.rs index 022902d4..7e2e3acd 100644 --- a/examples/capabilities/src/bin/graceful_drain.rs +++ b/examples/capabilities/src/bin/graceful_drain.rs @@ -10,17 +10,23 @@ impl Message for Add { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _: Add, _: &Context) { + async fn handle(&mut self, _: Add, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0 += 1; } + let _ = reply.reply(()); + } } struct Fence; impl Message for Fence { type Result = usize; } impl Handler for Worker { - async fn handle(&mut self, _: Fence, _: &Context) -> usize { + async fn handle(&mut self, _: Fence, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { + let __reply_value: usize = { self.0 + }; + let _ = reply.reply(__reply_value); } } #[tokio::main(flavor = "multi_thread", worker_threads = 2)] diff --git a/examples/capabilities/src/bin/native_lifecycle.rs b/examples/capabilities/src/bin/native_lifecycle.rs index 5772fee2..b3d037c0 100644 --- a/examples/capabilities/src/bin/native_lifecycle.rs +++ b/examples/capabilities/src/bin/native_lifecycle.rs @@ -19,18 +19,24 @@ impl Actor for Observer { } } impl Handler for Observer { - fn handle(&mut self, event: Down, _: &Context) { + fn handle(&mut self, event: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.counts.1 += 1; self.downs.send(event).unwrap(); } + let _ = reply.reply(()); + } } struct Counts; impl Message for Counts { type Result = (usize, usize); } impl Handler for Observer { - fn handle(&mut self, _: Counts, _: &Context) -> (usize, usize) { + fn handle(&mut self, _: Counts, reply: spawned_concurrency::reply::PendingReply<'_, (usize, usize)>, _: &Context) { + let __reply_value: (usize, usize) = { self.counts + }; + let _ = reply.reply(__reply_value); } } struct Failing; diff --git a/examples/capabilities/src/bin/protocol_timeouts.rs b/examples/capabilities/src/bin/protocol_timeouts.rs index f62b08fa..c6c2960a 100644 --- a/examples/capabilities/src/bin/protocol_timeouts.rs +++ b/examples/capabilities/src/bin/protocol_timeouts.rs @@ -23,15 +23,21 @@ mod asynchronous { } } impl Handler for Worker { - async fn handle(&mut self, _: lookup_protocol::Value, _: &Context) -> u64 { + async fn handle(&mut self, _: lookup_protocol::Value, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { self.1.fetch_add(1, Ordering::SeqCst); 42 + }; + let _ = reply.reply(__reply_value); } } impl Handler for Worker { - async fn handle(&mut self, _: lookup_protocol::Quick, _: &Context) -> u64 { + async fn handle(&mut self, _: lookup_protocol::Quick, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { self.1.fetch_add(1, Ordering::SeqCst); 42 + }; + let _ = reply.reply(__reply_value); } } pub async fn run() { @@ -63,15 +69,21 @@ mod synchronous { } } impl Handler for Worker { - fn handle(&mut self, _: lookup_protocol::Value, _: &Context) -> u64 { + fn handle(&mut self, _: lookup_protocol::Value, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { self.1.fetch_add(1, Ordering::SeqCst); 42 + }; + let _ = reply.reply(__reply_value); } } impl Handler for Worker { - fn handle(&mut self, _: lookup_protocol::Quick, _: &Context) -> u64 { + fn handle(&mut self, _: lookup_protocol::Quick, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { self.1.fetch_add(1, Ordering::SeqCst); 42 + }; + let _ = reply.reply(__reply_value); } } pub fn run() { diff --git a/examples/capabilities/src/bin/registrations_and_fakes.rs b/examples/capabilities/src/bin/registrations_and_fakes.rs index 7becca84..6675f95a 100644 --- a/examples/capabilities/src/bin/registrations_and_fakes.rs +++ b/examples/capabilities/src/bin/registrations_and_fakes.rs @@ -37,9 +37,12 @@ impl spawned_concurrency::message::Message for Crash { type Result = (); } impl Handler for Service { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("intentional named-service replacement"); } + let _ = reply.reply(()); + } } #[tokio::main(flavor = "current_thread")] async fn main() { diff --git a/examples/capabilities/src/bin/runtime_isolation.rs b/examples/capabilities/src/bin/runtime_isolation.rs index b0b39c80..b9bc67db 100644 --- a/examples/capabilities/src/bin/runtime_isolation.rs +++ b/examples/capabilities/src/bin/runtime_isolation.rs @@ -11,16 +11,22 @@ impl Message for Ping { } impl Actor for Critical {} impl Handler for Critical { - async fn handle(&mut self, _: Ping, _: &Context) -> String { + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, String>, _: &Context) { + let __reply_value: String = { std::thread::current().name().unwrap().to_owned() + }; + let _ = reply.reply(__reply_value); } } struct Router(ActorRef); impl Actor for Router {} impl Handler for Router { - async fn handle(&mut self, _: Ping, _: &Context) -> String { + async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, String>, _: &Context) { + let __reply_value: String = { assert_eq!(std::thread::current().name(), Some("background-worker")); self.0.request(Ping).await.unwrap() + }; + let _ = reply.reply(__reply_value); } } diff --git a/examples/capabilities/src/bin/shutdown_diagnostics.rs b/examples/capabilities/src/bin/shutdown_diagnostics.rs index cda151fe..e017b424 100644 --- a/examples/capabilities/src/bin/shutdown_diagnostics.rs +++ b/examples/capabilities/src/bin/shutdown_diagnostics.rs @@ -16,20 +16,26 @@ impl Message for Hold { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, msg: Hold, _: &Context) { + async fn handle(&mut self, msg: Hold, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { let _ = msg.0.send(()); let _ = msg.1.await; } + let _ = reply.reply(()); + } } struct Unhealthy; impl Message for Unhealthy { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _: Unhealthy, ctx: &Context) { + async fn handle(&mut self, _: Unhealthy, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + { // Cooperative abnormal exit, without a panic or forced interruption. ctx.stop_with_reason(ExitReason::Panic("reported unhealthy worker".into())); } + let _ = reply.reply(()); + } } #[tokio::main(flavor = "current_thread")] diff --git a/examples/capabilities/src/bin/streams_and_recipients.rs b/examples/capabilities/src/bin/streams_and_recipients.rs index d9a84b14..177ebb19 100644 --- a/examples/capabilities/src/bin/streams_and_recipients.rs +++ b/examples/capabilities/src/bin/streams_and_recipients.rs @@ -19,13 +19,19 @@ mod asynchronous { use spawned_concurrency::tasks::{self, Actor, ActorStart, Context, Handler, Recipient}; impl Actor for Counter {} impl Handler for Counter { - async fn handle(&mut self, msg: Item, _: &Context) { + async fn handle(&mut self, msg: Item, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0 += N * msg.0; } + let _ = reply.reply(()); + } } impl Handler for Counter { - async fn handle(&mut self, _: Total, _: &Context) -> u64 { + async fn handle(&mut self, _: Total, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { self.0 + }; + let _ = reply.reply(__reply_value); } } pub async fn run() { @@ -73,13 +79,19 @@ mod synchronous { use spawned_concurrency::threads::{self, Actor, ActorStart, Context, Handler, Recipient}; impl Actor for Counter {} impl Handler for Counter { - fn handle(&mut self, msg: Item, _: &Context) { + fn handle(&mut self, msg: Item, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0 += N * msg.0; } + let _ = reply.reply(()); + } } impl Handler for Counter { - fn handle(&mut self, _: Total, _: &Context) -> u64 { + fn handle(&mut self, _: Total, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { self.0 + }; + let _ = reply.reply(__reply_value); } } struct Source { diff --git a/examples/capabilities/src/bin/supervision_recovery.rs b/examples/capabilities/src/bin/supervision_recovery.rs index 967dc711..516ac62d 100644 --- a/examples/capabilities/src/bin/supervision_recovery.rs +++ b/examples/capabilities/src/bin/supervision_recovery.rs @@ -17,15 +17,21 @@ impl Message for Crash { struct Counter(u64); impl Actor for Counter {} impl Handler for Counter { - async fn handle(&mut self, _: Next, _: &Context) -> u64 { + async fn handle(&mut self, _: Next, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + let __reply_value: u64 = { self.0 += 1; self.0 + }; + let _ = reply.reply(__reply_value); } } impl Handler for Counter { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("intentional recovery example failure"); } + let _ = reply.reply(()); + } } #[tokio::main(flavor = "current_thread")] async fn main() { diff --git a/examples/capabilities/src/bin/supervision_tree.rs b/examples/capabilities/src/bin/supervision_tree.rs index 4a6bf2d6..846ce89c 100644 --- a/examples/capabilities/src/bin/supervision_tree.rs +++ b/examples/capabilities/src/bin/supervision_tree.rs @@ -19,9 +19,12 @@ impl Message for Fail { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _: Fail, _: &Context) { + async fn handle(&mut self, _: Fail, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { panic!("example failure"); } + let _ = reply.reply(()); + } } #[tokio::main(flavor = "multi_thread", worker_threads = 2)] async fn main() { diff --git a/examples/capabilities/src/bin/timeout_outcomes.rs b/examples/capabilities/src/bin/timeout_outcomes.rs index ac10dd20..32423d6a 100644 --- a/examples/capabilities/src/bin/timeout_outcomes.rs +++ b/examples/capabilities/src/bin/timeout_outcomes.rs @@ -17,19 +17,25 @@ impl Message for Apply { type Result = (); } impl Handler for Counter { - async fn handle(&mut self, msg: Apply, _: &Context) { + async fn handle(&mut self, msg: Apply, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0 += 1; // Effect happens before the reply. msg.entered.send(()).unwrap(); let _ = msg.release.await; } + let _ = reply.reply(()); + } } struct Read; impl Message for Read { type Result = usize; } impl Handler for Counter { - async fn handle(&mut self, _: Read, _: &Context) -> usize { + async fn handle(&mut self, _: Read, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { + let __reply_value: usize = { self.0 + }; + let _ = reply.reply(__reply_value); } } diff --git a/examples/exit_reason/src/main.rs b/examples/exit_reason/src/main.rs index e5e8f935..93214c5d 100644 --- a/examples/exit_reason/src/main.rs +++ b/examples/exit_reason/src/main.rs @@ -94,7 +94,9 @@ fn main() { } } impl Handler for PanicOnStart { - async fn handle(&mut self, _msg: Noop, _ctx: &Context) {} + async fn handle(&mut self, _msg: Noop, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _ctx: &Context) { + let _ = reply.reply(()); + } } let worker3 = PanicOnStart.start(); let reason = worker3.wait_exit().await; @@ -170,17 +172,24 @@ fn main() { } impl Actor for Observer {} impl Handler for Observer { - async fn handle(&mut self, msg: StartMonitor, ctx: &Context) -> MonitorRef { + async fn handle(&mut self, msg: StartMonitor, reply: spawned_concurrency::reply::PendingReply<'_, MonitorRef>, ctx: &Context) { + let __reply_value: MonitorRef = { ctx.monitor(&msg.0) + }; + let _ = reply.reply(__reply_value); } } impl Handler for Observer { - async fn handle(&mut self, msg: StopMonitor, ctx: &Context) { + async fn handle(&mut self, msg: StopMonitor, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.demonitor(msg.0); } + let _ = reply.reply(()); + } } impl Handler for Observer { - async fn handle(&mut self, msg: Down, _ctx: &Context) { + async fn handle(&mut self, msg: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _ctx: &Context) { + { tracing::info!( "[observer] received Down for {} ({})", msg.monitor_ref, @@ -188,6 +197,8 @@ fn main() { ); self.log.lock().unwrap().push(msg); } + let _ = reply.reply(()); + } } let log = Arc::new(Mutex::new(Vec::new())); @@ -259,9 +270,12 @@ fn main() { } } impl Handler for Supervisor { - async fn handle(&mut self, msg: LinkChild, ctx: &Context) { + async fn handle(&mut self, msg: LinkChild, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + { ctx.link(&msg.0); } + let _ = reply.reply(()); + } } let sup_log = Arc::new(Mutex::new(Vec::new())); diff --git a/examples/thread_only/src/main.rs b/examples/thread_only/src/main.rs index a8226371..0e5ec5ed 100644 --- a/examples/thread_only/src/main.rs +++ b/examples/thread_only/src/main.rs @@ -32,14 +32,20 @@ impl spawned_concurrency::message::Message for Tick { struct Observer(Arc); impl Actor for Observer {} impl Handler for Observer { - fn handle(&mut self, _: Down, _: &Context) { + fn handle(&mut self, _: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.fetch_add(1, Ordering::SeqCst); } + let _ = reply.reply(()); + } } impl Handler for Observer { - fn handle(&mut self, _: Tick, _: &Context) { + fn handle(&mut self, _: Tick, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.fetch_add(1, Ordering::SeqCst); } + let _ = reply.reply(()); + } } fn main() { let counter = Counter(0).start(); diff --git a/testing/src/faults.rs b/testing/src/faults.rs index 4ff2629a..a09ddd14 100644 --- a/testing/src/faults.rs +++ b/testing/src/faults.rs @@ -61,9 +61,12 @@ impl Message for Detonate { type Result = (); } impl threads::Handler for Fuse { - fn handle(&mut self, msg: Detonate, _: &threads::Context) { + fn handle(&mut self, msg: Detonate, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context) { + { panic!("{}", msg.0); } + let _ = reply.reply(()); + } } fn panic_linked(target: &ChildHandle, msg: String) { diff --git a/testing/tests/tasks_kit.rs b/testing/tests/tasks_kit.rs index 97102509..569ebb75 100644 --- a/testing/tests/tasks_kit.rs +++ b/testing/tests/tasks_kit.rs @@ -225,13 +225,19 @@ impl Message for Get { } impl Actor for Counter {} impl Handler for Counter { - async fn handle(&mut self, _: Bump, _: &Context) { + async fn handle(&mut self, _: Bump, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.fetch_add(1, Ordering::SeqCst); } + let _ = reply.reply(()); + } } impl Handler for Counter { - async fn handle(&mut self, _: Get, _: &Context) -> usize { + async fn handle(&mut self, _: Get, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { + let __reply_value: usize = { self.0.load(Ordering::SeqCst) + }; + let _ = reply.reply(__reply_value); } } @@ -330,11 +336,14 @@ impl Message for Hang { } impl Actor for Stuck {} impl Handler for Stuck { - async fn handle(&mut self, _: Hang, _: &Context) { + async fn handle(&mut self, _: Hang, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + { self.0.store(true, Ordering::SeqCst); // Blocking code is never preempted, so shutdown cannot finish in time. std::thread::sleep(Duration::from_millis(500)); } + let _ = reply.reply(()); + } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/testing/tests/threads_kit.rs b/testing/tests/threads_kit.rs index db50159c..4a90ffc9 100644 --- a/testing/tests/threads_kit.rs +++ b/testing/tests/threads_kit.rs @@ -150,7 +150,9 @@ impl Message for Ping { type Result = (); } impl Handler for Idle { - fn handle(&mut self, _: Ping, _: &Context) {} + fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + let _ = reply.reply(()); + } } #[test] From 0d4386e0026a302c4d64e34b48c6bc7fc86b85b2 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:22:02 -0300 Subject: [PATCH 009/208] wip --- concurrency/src/tasks/actor.rs | 14 ++++++++------ concurrency/src/threads/actor.rs | 10 ++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index ec3eb8bd..0d77500e 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -7,9 +7,10 @@ use crate::link::{ use crate::mailbox::tasks as mpsc; use crate::message::Message; use crate::monitor::{Down, MonitorCleanup, MonitorRef, MonitorTable}; -use crate::reply::{registry::Registry, reply_slot, Deadline, SlotHandle}; +use crate::reply::{registry::Registry, reply_slot, SlotHandle}; +#[doc(no_inline)] pub use crate::reply::{ - DeferError, Delivery, PendingReply, ReplyReceiver, ReplyTo, DEFAULT_MAX_OUTSTANDING, + Deadline, DeferError, Delivery, PendingReply, ReplyReceiver, ReplyTo, DEFAULT_MAX_OUTSTANDING, }; use core::pin::pin; use futures::future::{self, FutureExt as _}; @@ -172,13 +173,14 @@ where observation.reject(); } } - fn handle<'a>(self: Box, actor: &'a mut A, reply: crate::reply::PendingReply<'_, Pin + Send + 'a>>>, ctx: &'a Context) { - let __reply_value: Pin + Send + 'a>> = { + fn handle<'a>( + self: Box, + actor: &'a mut A, + ctx: &'a Context, + ) -> Pin + Send + 'a>> { let mut this = *self; let reply = PendingReply::new(this.slot.take(), &ctx.shared.replies); Box::pin(actor.handle(this.msg, reply, ctx)) - }; - let _ = reply.reply(__reply_value); } } diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 9714e773..75828115 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -16,9 +16,10 @@ use crate::link::{ }; use crate::message::Message; use crate::monitor::{Down, MonitorCleanup, MonitorRef, MonitorTable}; -use crate::reply::{registry::Registry, reply_slot, Deadline, SlotHandle}; +use crate::reply::{registry::Registry, reply_slot, SlotHandle}; +#[doc(no_inline)] pub use crate::reply::{ - DeferError, Delivery, PendingReply, ReplyReceiver, ReplyTo, DEFAULT_MAX_OUTSTANDING, + Deadline, DeferError, Delivery, PendingReply, ReplyReceiver, ReplyTo, DEFAULT_MAX_OUTSTANDING, }; use std::sync::atomic::Ordering; @@ -101,14 +102,11 @@ where observation.reject(); } } - fn handle(self: Box, actor: &mut A, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { - { + fn handle(self: Box, actor: &mut A, ctx: &Context) { let mut this = *self; let reply = PendingReply::new(this.slot.take(), &ctx.replies); actor.handle(this.msg, reply, ctx); } - let _ = reply.reply(()); - } } fn enqueue( From 997558e02e9394d2951dcd3141762f07e5069472 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:22:28 -0300 Subject: [PATCH 010/208] style: rustfmt --- concurrency/benches/baseline.rs | 22 +- concurrency/benches/designs.rs | 94 ++-- concurrency/benches/lifecycle_soak.rs | 99 ++-- concurrency/benches/metrics_overhead.rs | 22 +- concurrency/benches/pinned_pipeline.rs | 75 +-- concurrency/benches/runtime_isolation.rs | 54 +- concurrency/benches/workload_research.rs | 98 ++-- concurrency/examples/panic_abort.rs | 22 +- concurrency/src/child_handle.rs | 44 +- concurrency/src/link_regression_tests.rs | 54 +- concurrency/src/reply/mod.rs | 20 +- concurrency/src/reply/slot.rs | 22 +- concurrency/src/tasks/actor.rs | 489 ++++++++++++------ concurrency/src/tasks/stream_tests.rs | 37 +- concurrency/src/tasks/timer_tests.rs | 70 ++- concurrency/src/threads/actor.rs | 346 +++++++++---- concurrency/src/threads/stream_tests.rs | 24 +- concurrency/src/threads/timer_tests.rs | 70 ++- concurrency/tests/actor_scenarios.rs | 100 ++-- concurrency/tests/backend_experiments.rs | 85 +-- concurrency/tests/backend_resource_limits.rs | 7 +- concurrency/tests/borrowed_runtime.rs | 22 +- concurrency/tests/canary_feedback.rs | 35 +- concurrency/tests/cancellation_safety.rs | 82 +-- concurrency/tests/cfg_gated_protocol.rs | 24 +- concurrency/tests/cohort_trace.rs | 22 +- concurrency/tests/context_exit.rs | 26 +- concurrency/tests/coverage_trace.rs | 22 +- concurrency/tests/cross_backend.rs | 386 ++++++++------ concurrency/tests/dedicated_runtime.rs | 168 +++--- concurrency/tests/execution_groups.rs | 48 +- concurrency/tests/execution_profiles.rs | 70 ++- concurrency/tests/failpoints.rs | 58 ++- concurrency/tests/foreign_panics.rs | 92 +++- concurrency/tests/full_actor_shards.rs | 168 +++--- concurrency/tests/future_layout.rs | 18 +- concurrency/tests/kill_links.rs | 22 +- concurrency/tests/leaks.rs | 166 ++++-- concurrency/tests/mailbox_drop_panic.rs | 44 +- concurrency/tests/metrics.rs | 42 +- concurrency/tests/metrics_composable.rs | 7 +- concurrency/tests/metrics_counts_only.rs | 7 +- concurrency/tests/metrics_sampling.rs | 7 +- concurrency/tests/native_dispatcher.rs | 13 +- concurrency/tests/native_mailbox_drop.rs | 22 +- concurrency/tests/nested_supervision.rs | 35 +- concurrency/tests/panic_safety.rs | 256 ++++++--- concurrency/tests/process_failures.rs | 29 +- concurrency/tests/protocol_timeouts.rs | 127 +++-- concurrency/tests/reply_slot_loom.rs | 19 +- concurrency/tests/resource_cleanup.rs | 22 +- concurrency/tests/semantics_gaps.rs | 171 ++++-- concurrency/tests/shutdown_publication.rs | 19 +- concurrency/tests/sim.rs | 11 +- concurrency/tests/sim_explore.rs | 65 ++- concurrency/tests/stack_budget.rs | 66 ++- concurrency/tests/startup_admission.rs | 11 +- concurrency/tests/supervision.rs | 92 +++- concurrency/tests/supervision_strategies.rs | 48 +- concurrency/tests/supervision_trace.rs | 68 ++- concurrency/tests/supervisor_policies.rs | 22 +- concurrency/tests/support/failure_actor.rs | 128 +++-- concurrency/tests/support/store_actor.rs | 30 +- concurrency/tests/tree_trace.rs | 11 +- .../src/bin/byte_budget_pipeline.rs | 43 +- .../src/bin/coordinated_shutdown.rs | 40 +- .../capabilities/src/bin/dedicated_thread.rs | 37 +- .../capabilities/src/bin/execution_groups.rs | 22 +- .../src/bin/execution_profiles.rs | 17 +- .../capabilities/src/bin/framework_metrics.rs | 7 +- .../capabilities/src/bin/graceful_drain.rs | 22 +- .../capabilities/src/bin/native_lifecycle.rs | 24 +- .../capabilities/src/bin/protocol_timeouts.rs | 52 +- .../src/bin/registrations_and_fakes.rs | 11 +- .../capabilities/src/bin/runtime_isolation.rs | 24 +- .../src/bin/shutdown_diagnostics.rs | 26 +- .../src/bin/streams_and_recipients.rs | 44 +- .../src/bin/supervision_recovery.rs | 24 +- .../capabilities/src/bin/supervision_tree.rs | 11 +- .../capabilities/src/bin/timeout_outcomes.rs | 26 +- examples/exit_reason/src/main.rs | 61 ++- macros/src/lib.rs | 11 +- rt/src/tasks/mod.rs | 4 +- testing/src/faults.rs | 11 +- testing/src/tasks.rs | 2 +- testing/src/threads.rs | 2 +- testing/tests/tasks_kit.rs | 37 +- testing/tests/threads_kit.rs | 7 +- 88 files changed, 3463 insertions(+), 1757 deletions(-) diff --git a/concurrency/benches/baseline.rs b/concurrency/benches/baseline.rs index f65e4065..05e70b76 100644 --- a/concurrency/benches/baseline.rs +++ b/concurrency/benches/baseline.rs @@ -98,19 +98,25 @@ fn process(work: Work) -> Ack { struct Sink; impl tasks::Actor for Sink {} impl tasks::Handler for Sink { - async fn handle(&mut self, work: Work, reply: spawned_concurrency::reply::PendingReply<'_, Ack>, _: &tasks::Context) { - let __reply_value: Ack = { - process(work) - }; + async fn handle( + &mut self, + work: Work, + reply: spawned_concurrency::reply::PendingReply<'_, Ack>, + _: &tasks::Context, + ) { + let __reply_value: Ack = { process(work) }; let _ = reply.reply(__reply_value); } } impl threads::Actor for Sink {} impl threads::Handler for Sink { - fn handle(&mut self, work: Work, reply: spawned_concurrency::reply::PendingReply<'_, Ack>, _: &threads::Context) { - let __reply_value: Ack = { - process(work) - }; + fn handle( + &mut self, + work: Work, + reply: spawned_concurrency::reply::PendingReply<'_, Ack>, + _: &threads::Context, + ) { + let __reply_value: Ack = { process(work) }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/benches/designs.rs b/concurrency/benches/designs.rs index 8db4d352..64532435 100644 --- a/concurrency/benches/designs.rs +++ b/concurrency/benches/designs.rs @@ -122,46 +122,54 @@ impl Sink { } impl tasks::Actor for Sink {} impl tasks::Handler for Sink { - async fn handle(&mut self, work: Work, reply: spawned_concurrency::reply::PendingReply<'_, Ack>, _: &tasks::Context) { + async fn handle( + &mut self, + work: Work, + reply: spawned_concurrency::reply::PendingReply<'_, Ack>, + _: &tasks::Context, + ) { let __reply_value: Ack = { - let start = if work.timing { - Instant::now() - } else { - work.enqueued - }; - match self.work.as_str() { - "timer" => tokio::time::sleep(self.wait).await, - "blocking" => std::thread::sleep(self.wait), - "tcp" => { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - if self.socket.is_none() { - let socket = tokio::net::TcpStream::connect(self.address.unwrap()) - .await - .unwrap(); - socket.set_nodelay(true).unwrap(); - self.socket = Some(socket); + let start = if work.timing { + Instant::now() + } else { + work.enqueued + }; + match self.work.as_str() { + "timer" => tokio::time::sleep(self.wait).await, + "blocking" => std::thread::sleep(self.wait), + "tcp" => { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + if self.socket.is_none() { + let socket = tokio::net::TcpStream::connect(self.address.unwrap()) + .await + .unwrap(); + socket.set_nodelay(true).unwrap(); + self.socket = Some(socket); + } + let socket = self.socket.as_mut().unwrap(); + let mut frame = [0u8; 64]; + frame[..8].copy_from_slice(&work.id.to_le_bytes()); + let expected = frame; + socket.write_all(&frame).await.unwrap(); + socket.read_exact(&mut frame).await.unwrap(); + assert_eq!(frame, expected); } - let socket = self.socket.as_mut().unwrap(); - let mut frame = [0u8; 64]; - frame[..8].copy_from_slice(&work.id.to_le_bytes()); - let expected = frame; - socket.write_all(&frame).await.unwrap(); - socket.read_exact(&mut frame).await.unwrap(); - assert_eq!(frame, expected); + _ => {} } - _ => {} - } - process_at(work, start) - }; + process_at(work, start) + }; let _ = reply.reply(__reply_value); } } impl threads::Actor for Sink {} impl threads::Handler for Sink { - fn handle(&mut self, work: Work, reply: spawned_concurrency::reply::PendingReply<'_, Ack>, _: &threads::Context) { - let __reply_value: Ack = { - process(work) - }; + fn handle( + &mut self, + work: Work, + reply: spawned_concurrency::reply::PendingReply<'_, Ack>, + _: &threads::Context, + ) { + let __reply_value: Ack = { process(work) }; let _ = reply.reply(__reply_value); } } @@ -193,18 +201,28 @@ impl Message for Probe { type Result = (); } impl tasks::Handler for Sink { - async fn handle(&mut self, probe: Probe, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &tasks::Context) { + async fn handle( + &mut self, + probe: Probe, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &tasks::Context, + ) { { - probe.reply.send(process(probe.work)); - } + probe.reply.send(process(probe.work)); + } let _ = reply.reply(()); } } impl threads::Handler for Sink { - fn handle(&mut self, probe: Probe, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context) { + fn handle( + &mut self, + probe: Probe, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &threads::Context, + ) { { - probe.reply.send(process(probe.work)); - } + probe.reply.send(process(probe.work)); + } let _ = reply.reply(()); } } diff --git a/concurrency/benches/lifecycle_soak.rs b/concurrency/benches/lifecycle_soak.rs index 1a2864a3..e01e82cc 100644 --- a/concurrency/benches/lifecycle_soak.rs +++ b/concurrency/benches/lifecycle_soak.rs @@ -61,25 +61,30 @@ impl Message for Step { type Result = u64; } impl Handler for Worker { - async fn handle(&mut self, msg: Step, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + async fn handle( + &mut self, + msg: Step, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - assert_eq!(self.state, msg.expected); - assert_eq!(self.bytes[0], 13); - if msg.helper { - assert_eq!( - tokio::spawn(async { - tokio::time::sleep(Duration::from_millis(1)).await; + assert_eq!(self.state, msg.expected); + assert_eq!(self.bytes[0], 13); + if msg.helper { + assert_eq!( + tokio::spawn(async { + tokio::time::sleep(Duration::from_millis(1)).await; + 42 + }) + .await + .unwrap(), 42 - }) - .await - .unwrap(), - 42 - ); - } - self.state += 1; - self.counts.requests.fetch_add(1, Ordering::Relaxed); - self.state - }; + ); + } + self.state += 1; + self.counts.requests.fetch_add(1, Ordering::Relaxed); + self.state + }; let _ = reply.reply(__reply_value); } } @@ -88,10 +93,15 @@ impl Message for Crash { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("soak injected handler failure"); - } + panic!("soak injected handler failure"); + } let _ = reply.reply(()); } } @@ -105,28 +115,38 @@ impl Message for Watch { type Result = (); } impl Handler for Watcher { - async fn handle(&mut self, msg: Watch, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + msg: Watch, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - for (child, panic) in msg.0 { - assert!(self.pending.insert(ctx.monitor(&child), panic).is_none()); + for (child, panic) in msg.0 { + assert!(self.pending.insert(ctx.monitor(&child), panic).is_none()); + } } - } let _ = reply.reply(()); } } impl Handler for Watcher { - async fn handle(&mut self, msg: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - let panic = self - .pending - .remove(&msg.monitor_ref) - .expect("duplicate/unknown monitor notification"); - assert_eq!(matches!(msg.reason, ExitReason::Panic(_)), panic); - if !panic { - assert_eq!(msg.reason, ExitReason::Normal); + let panic = self + .pending + .remove(&msg.monitor_ref) + .expect("duplicate/unknown monitor notification"); + assert_eq!(matches!(msg.reason, ExitReason::Panic(_)), panic); + if !panic { + assert_eq!(msg.reason, ExitReason::Normal); + } + self.counts.down.fetch_add(1, Ordering::Release); } - self.counts.down.fetch_add(1, Ordering::Release); - } let _ = reply.reply(()); } } @@ -135,10 +155,13 @@ impl Message for Remaining { type Result = usize; } impl Handler for Watcher { - async fn handle(&mut self, _: Remaining, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { - let __reply_value: usize = { - self.pending.len() - }; + async fn handle( + &mut self, + _: Remaining, + reply: spawned_concurrency::reply::PendingReply<'_, usize>, + _: &Context, + ) { + let __reply_value: usize = { self.pending.len() }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/benches/metrics_overhead.rs b/concurrency/benches/metrics_overhead.rs index 99f94825..793a0d56 100644 --- a/concurrency/benches/metrics_overhead.rs +++ b/concurrency/benches/metrics_overhead.rs @@ -18,19 +18,25 @@ fn compute(work: Work) -> u64 { } impl tasks::Actor for Sink {} impl tasks::Handler for Sink { - async fn handle(&mut self, work: Work, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &tasks::Context) { - let __reply_value: u64 = { - compute(work) - }; + async fn handle( + &mut self, + work: Work, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &tasks::Context, + ) { + let __reply_value: u64 = { compute(work) }; let _ = reply.reply(__reply_value); } } impl threads::Actor for Sink {} impl threads::Handler for Sink { - fn handle(&mut self, work: Work, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &threads::Context) { - let __reply_value: u64 = { - compute(work) - }; + fn handle( + &mut self, + work: Work, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &threads::Context, + ) { + let __reply_value: u64 = { compute(work) }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/benches/pinned_pipeline.rs b/concurrency/benches/pinned_pipeline.rs index 04c4f590..a86b754e 100644 --- a/concurrency/benches/pinned_pipeline.rs +++ b/concurrency/benches/pinned_pipeline.rs @@ -49,10 +49,15 @@ mod linux { type Result = (); } impl Handler for Stage { - async fn handle(&mut self, m: Wire, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + m: Wire, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.next = m.0; - } + self.next = m.0; + } let _ = reply.reply(()); } } @@ -66,21 +71,26 @@ mod linux { type Result = (); } impl Handler for Stage { - async fn handle(&mut self, mut m: Work, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + mut m: Work, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - for _ in 0..self.iterations { - m.value = black_box(m.value.wrapping_mul(MUL).wrapping_add(1)); - } - m.hops += 1; - self.handled += 1; - if let Some(next) = &self.next { - next.send(m).unwrap(); - } else { - m.reply - .send((m.id, m.value, m.hops, Instant::now())) - .unwrap(); + for _ in 0..self.iterations { + m.value = black_box(m.value.wrapping_mul(MUL).wrapping_add(1)); + } + m.hops += 1; + self.handled += 1; + if let Some(next) = &self.next { + next.send(m).unwrap(); + } else { + m.reply + .send((m.id, m.value, m.hops, Instant::now())) + .unwrap(); + } } - } let _ = reply.reply(()); } } @@ -89,21 +99,26 @@ mod linux { type Result = (String, String, u64); } impl Handler for Stage { - async fn handle(&mut self, _: Audit, reply: spawned_concurrency::reply::PendingReply<'_, (String, String, u64)>, _: &Context) { + async fn handle( + &mut self, + _: Audit, + reply: spawned_concurrency::reply::PendingReply<'_, (String, String, u64)>, + _: &Context, + ) { let __reply_value: (String, String, u64) = { - let status = fs::read_to_string("/proc/thread-self/status").unwrap(); - let affinity = status - .lines() - .find_map(|s| s.strip_prefix("Cpus_allowed_list:")) - .unwrap() - .trim() - .to_owned(); - ( - std::thread::current().name().unwrap_or("main").to_owned(), - affinity, - self.handled, - ) - }; + let status = fs::read_to_string("/proc/thread-self/status").unwrap(); + let affinity = status + .lines() + .find_map(|s| s.strip_prefix("Cpus_allowed_list:")) + .unwrap() + .trim() + .to_owned(); + ( + std::thread::current().name().unwrap_or("main").to_owned(), + affinity, + self.handled, + ) + }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/benches/runtime_isolation.rs b/concurrency/benches/runtime_isolation.rs index 9b11d9c0..fcc1eaff 100644 --- a/concurrency/benches/runtime_isolation.rs +++ b/concurrency/benches/runtime_isolation.rs @@ -46,18 +46,23 @@ mod linux { type Result = (u64, u64, Instant); } impl Handler for Stage { - async fn handle(&mut self, m: Work, reply: spawned_concurrency::reply::PendingReply<'_, (u64, u64, Instant)>, _: &Context) { + async fn handle( + &mut self, + m: Work, + reply: spawned_concurrency::reply::PendingReply<'_, (u64, u64, Instant)>, + _: &Context, + ) { let __reply_value: (u64, u64, Instant) = async { - self.handled += 1; - if let Some(next) = &self.next { - return next.request_raw(m).unwrap().await.unwrap(); - } - let mut value = m.0; - for _ in 0..self.iterations { - value = black_box(value.wrapping_mul(MUL).wrapping_add(1)); + self.handled += 1; + if let Some(next) = &self.next { + return next.request_raw(m).unwrap().await.unwrap(); + } + let mut value = m.0; + for _ in 0..self.iterations { + value = black_box(value.wrapping_mul(MUL).wrapping_add(1)); + } + (m.0, value, Instant::now()) } - (m.0, value, Instant::now()) - } .await; let _ = reply.reply(__reply_value); } @@ -67,19 +72,24 @@ mod linux { type Result = (String, u64); } impl Handler for Stage { - async fn handle(&mut self, _: Audit, reply: spawned_concurrency::reply::PendingReply<'_, (String, u64)>, _: &Context) { + async fn handle( + &mut self, + _: Audit, + reply: spawned_concurrency::reply::PendingReply<'_, (String, u64)>, + _: &Context, + ) { let __reply_value: (String, u64) = { - let status = fs::read_to_string("/proc/thread-self/status").unwrap(); - ( - status - .lines() - .find_map(|s| s.strip_prefix("Cpus_allowed_list:")) - .unwrap() - .trim() - .into(), - self.handled, - ) - }; + let status = fs::read_to_string("/proc/thread-self/status").unwrap(); + ( + status + .lines() + .find_map(|s| s.strip_prefix("Cpus_allowed_list:")) + .unwrap() + .trim() + .into(), + self.handled, + ) + }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/benches/workload_research.rs b/concurrency/benches/workload_research.rs index 941b44b1..c4a71002 100644 --- a/concurrency/benches/workload_research.rs +++ b/concurrency/benches/workload_research.rs @@ -230,16 +230,21 @@ struct Node { } impl Actor for Node {} impl Handler for Node { - async fn handle(&mut self, msg: Wire, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: Wire, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - if std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").as_deref() == Ok("sharded") { - assert_eq!( - std::thread::current().name().unwrap(), - format!("spawned-shard-{}", msg.expected_shard) - ); + if std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").as_deref() == Ok("sharded") { + assert_eq!( + std::thread::current().name().unwrap(), + format!("spawned-shard-{}", msg.expected_shard) + ); + } + self.children = msg.children; } - self.children = msg.children; - } let _ = reply.reply(()); } } @@ -251,43 +256,48 @@ fn cpu_work(mut value: u64, iterations: u32) -> u64 { value } impl Handler for Node { - async fn handle(&mut self, msg: Route, reply: spawned_concurrency::reply::PendingReply<'_, Ack>, _: &Context) { + async fn handle( + &mut self, + msg: Route, + reply: spawned_concurrency::reply::PendingReply<'_, Ack>, + _: &Context, + ) { let __reply_value: Ack = { - let started = Instant::now(); - if msg.measured { - self.meter.started.fetch_add(1, Ordering::Relaxed); - } - let value = cpu_work(msg.id ^ (self.id as u64), msg.iterations); - // Start all fan-out requests before awaiting any; chains have one child. - let replies: Vec<_> = self - .children - .iter() - .map(|c| c.request_raw(msg.clone()).unwrap()) - .collect(); - let mut nodes = 1; - let mut tag = self.id as u64 + 1; - let mut checksum = value; - for reply in futures::future::join_all(replies).await { - let ack = reply.unwrap(); - assert_eq!(ack.id, msg.id); - nodes += ack.nodes; - tag += ack.tag; - checksum ^= ack.checksum; - } - let completed = Instant::now(); - if msg.measured { - self.meter.completed.fetch_add(1, Ordering::Relaxed); - } - Ack { - id: msg.id, - nodes, - tag, - checksum, - completed, - queue_ns: started.duration_since(msg.enqueued).as_nanos() as u64, - service_ns: completed.duration_since(started).as_nanos() as u64, - } - }; + let started = Instant::now(); + if msg.measured { + self.meter.started.fetch_add(1, Ordering::Relaxed); + } + let value = cpu_work(msg.id ^ (self.id as u64), msg.iterations); + // Start all fan-out requests before awaiting any; chains have one child. + let replies: Vec<_> = self + .children + .iter() + .map(|c| c.request_raw(msg.clone()).unwrap()) + .collect(); + let mut nodes = 1; + let mut tag = self.id as u64 + 1; + let mut checksum = value; + for reply in futures::future::join_all(replies).await { + let ack = reply.unwrap(); + assert_eq!(ack.id, msg.id); + nodes += ack.nodes; + tag += ack.tag; + checksum ^= ack.checksum; + } + let completed = Instant::now(); + if msg.measured { + self.meter.completed.fetch_add(1, Ordering::Relaxed); + } + Ack { + id: msg.id, + nodes, + tag, + checksum, + completed, + queue_ns: started.duration_since(msg.enqueued).as_nanos() as u64, + service_ns: completed.duration_since(started).as_nanos() as u64, + } + }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/examples/panic_abort.rs b/concurrency/examples/panic_abort.rs index 1e62efa2..ac92f789 100644 --- a/concurrency/examples/panic_abort.rs +++ b/concurrency/examples/panic_abort.rs @@ -43,10 +43,13 @@ mod native { } } impl Handler for Victim { - fn handle(&mut self, _: Boom, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - panic!("{PANIC_TEXT}") - }; + fn handle( + &mut self, + _: Boom, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { panic!("{PANIC_TEXT}") }; let _ = reply.reply(__reply_value); } } @@ -69,10 +72,13 @@ mod tasks { } } impl Handler for Victim { - async fn handle(&mut self, _: Boom, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - panic!("{PANIC_TEXT}") - }; + async fn handle( + &mut self, + _: Boom, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { panic!("{PANIC_TEXT}") }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/src/child_handle.rs b/concurrency/src/child_handle.rs index 1d2fb827..fd4f99f7 100644 --- a/concurrency/src/child_handle.rs +++ b/concurrency/src/child_handle.rs @@ -472,10 +472,15 @@ mod tests { } impl Actor for Worker {} impl Handler for Worker { - fn handle(&mut self, _msg: Stop, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + fn handle( + &mut self, + _msg: Stop, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.stop(); - } + ctx.stop(); + } let _ = reply.reply(()); } } @@ -506,10 +511,15 @@ mod tests { } impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _msg: Stop, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + _msg: Stop, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.stop(); - } + ctx.stop(); + } let _ = reply.reply(()); } } @@ -626,10 +636,15 @@ mod tests { } impl Actor for Boomer {} impl Handler for Boomer { - fn handle(&mut self, _msg: Boom, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + fn handle( + &mut self, + _msg: Boom, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - panic!("intentional panic in handler"); - } + panic!("intentional panic in handler"); + } let _ = reply.reply(()); } } @@ -661,10 +676,15 @@ mod tests { } impl Actor for Boomer {} impl Handler for Boomer { - async fn handle(&mut self, _msg: Boom, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: Boom, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - panic!("intentional panic in handler"); - } + panic!("intentional panic in handler"); + } let _ = reply.reply(()); } } diff --git a/concurrency/src/link_regression_tests.rs b/concurrency/src/link_regression_tests.rs index e5dbfd8a..6a203d7c 100644 --- a/concurrency/src/link_regression_tests.rs +++ b/concurrency/src/link_regression_tests.rs @@ -119,7 +119,12 @@ impl crate::message::Message for Ping { type Result = (); } impl crate::tasks::Handler for Idler { - async fn handle(&mut self, _: Ping, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } @@ -159,10 +164,15 @@ fn polled_actor_propagates_runtime_shutdown_to_peer() { struct MonitorObserver(Option>); impl Actor for MonitorObserver {} impl crate::tasks::Handler for MonitorObserver { - async fn handle(&mut self, down: crate::Down, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + down: crate::Down, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0.take().unwrap().send(down).unwrap(); - } + self.0.take().unwrap().send(down).unwrap(); + } let _ = reply.reply(()); } } @@ -210,10 +220,15 @@ impl crate::message::Message for Crash { type Result = (); } impl crate::tasks::Handler for Idler { - async fn handle(&mut self, _: Crash, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("linked failure during another actor's stopped hook"); - } + panic!("linked failure during another actor's stopped hook"); + } let _ = reply.reply(()); } } @@ -228,10 +243,15 @@ impl Actor for Stopping { } } impl crate::tasks::Handler for Stopping { - async fn handle(&mut self, _: Stop, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + _: Stop, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.stop(); - } + ctx.stop(); + } let _ = reply.reply(()); } } @@ -477,7 +497,12 @@ fn linked_runtime_abort_cancels_an_idle_threads_actor() { struct ThreadPeer; impl ThreadActor for ThreadPeer {} impl crate::threads::Handler for ThreadPeer { - fn handle(&mut self, _: Ping, reply: crate::reply::PendingReply<'_, ()>, _: &crate::threads::Context) { + fn handle( + &mut self, + _: Ping, + reply: crate::reply::PendingReply<'_, ()>, + _: &crate::threads::Context, + ) { let _ = reply.reply(()); } } @@ -521,7 +546,12 @@ fn dropping_runtime_with_large_polled_link_cycle_reclaims_every_actor() { struct Node(Arc); impl Actor for Node {} impl crate::tasks::Handler for Node { - async fn handle(&mut self, _: Ping, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 05d93171..d8e545e0 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -242,7 +242,9 @@ impl<'t, R: Send + 'static> PendingReply<'t, R> { /// The request's deadline, or `None` for [`Deadline::Never`] and sends. pub fn deadline(&self) -> Option { - self.slot.as_ref().and_then(|s| Deadline::from_ticks(s.deadline())) + self.slot + .as_ref() + .and_then(|s| Deadline::from_ticks(s.deadline())) } /// Whether the caller can no longer be answered (the deadline passed or @@ -305,7 +307,10 @@ impl std::fmt::Debug for ReplyTo { impl ReplyTo { /// Answer the caller. Returns how the reply was delivered. pub fn reply(mut self, value: R) -> Delivery { - let slot = self.slot.take().expect("ReplyTo holds its slot until consumed"); + let slot = self + .slot + .take() + .expect("ReplyTo holds its slot until consumed"); let (delivery, unsent) = slot.reply(value, &now_ticks); drop(unsent); record(delivery); @@ -326,7 +331,8 @@ impl ReplyTo { /// Shorten the deadline to `at` (never extends it). pub fn with_deadline(&self, at: Instant) { - self.slot().shorten(spawned_rt::time::to_ticks(at), &now_ticks); + self.slot() + .shorten(spawned_rt::time::to_ticks(at), &now_ticks); self.registry.track_deadline(self.slot()); } @@ -358,7 +364,9 @@ impl ReplyTo { } fn slot(&self) -> &Arc> { - self.slot.as_ref().expect("ReplyTo holds its slot until consumed") + self.slot + .as_ref() + .expect("ReplyTo holds its slot until consumed") } pub(crate) fn slot_arc(&self) -> &Arc> { @@ -498,7 +506,9 @@ impl std::future::Future for ReplyReceiver { /// a manual `Handler::handle` directly in tests (`spawned_test::reply_pair`). /// Await the receiver only after driving the turns the reply depends on. #[doc(hidden)] -pub fn reply_pair(deadline: Deadline) -> (PendingReply<'static, R>, ReplyReceiver) { +pub fn reply_pair( + deadline: Deadline, +) -> (PendingReply<'static, R>, ReplyReceiver) { let (mut handle, receiver) = reply_slot(deadline); let reply = PendingReply::new(handle.take(), Registry::detached()); (reply, receiver) diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 6a7824cc..1200eda3 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -292,7 +292,11 @@ impl Slot { } let mut inner = lock(&self.inner); Wake { - pool: if release > 0 { inner.pool.clone() } else { None }, + pool: if release > 0 { + inner.pool.clone() + } else { + None + }, release, waker: if terminal { inner.waker.take() } else { None }, watcher: if terminal { inner.watcher.take() } else { None }, @@ -356,14 +360,20 @@ impl Slot { pub(crate) fn shorten(&self, at: u64, now: &impl Fn() -> u64) { let _ = self .deadline - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |d| (at < d).then_some(at)); + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |d| { + (at < d).then_some(at) + }); self.check_deadline(now); } /// `defer()`: `OPEN, TOKEN=0 -> OPEN, TOKEN=1, REG=1`, reserving the /// token's credit when no linked job holds one. At the bound the caller /// is answered `Overloaded` in the same step. - pub(crate) fn defer(&self, pool: &Arc, now: &impl Fn() -> u64) -> Result<(), Refused> { + pub(crate) fn defer( + &self, + pool: &Arc, + now: &impl Fn() -> u64, + ) -> Result<(), Refused> { if self.check_deadline(now) { return Err(Refused::Closed); } @@ -461,7 +471,11 @@ impl Slot { let (old, new) = self.resolve_writing(REPLIED); let release = credits(old).saturating_sub(credits(new)); Wake { - pool: if release > 0 { inner.pool.clone() } else { None }, + pool: if release > 0 { + inner.pool.clone() + } else { + None + }, release, waker: inner.waker.take(), watcher: inner.watcher.take(), diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 0d77500e..3bc7a68b 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -1601,30 +1601,43 @@ mod tests { impl Actor for Counter {} impl Handler for Counter { - async fn handle(&mut self, _msg: GetCount, reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context) { - let __reply_value: u64 = { - self.count - }; + async fn handle( + &mut self, + _msg: GetCount, + reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + let __reply_value: u64 = { self.count }; let _ = reply.reply(__reply_value); } } impl Handler for Counter { - async fn handle(&mut self, _msg: Increment, reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: Increment, + reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { let __reply_value: u64 = { - self.count += 1; - self.count - }; + self.count += 1; + self.count + }; let _ = reply.reply(__reply_value); } } impl Handler for Counter { - async fn handle(&mut self, _msg: StopCounter, reply: crate::reply::PendingReply<'_, u64>, ctx: &Context) { + async fn handle( + &mut self, + _msg: StopCounter, + reply: crate::reply::PendingReply<'_, u64>, + ctx: &Context, + ) { let __reply_value: u64 = { - ctx.stop(); - self.count - }; + ctx.stop(); + self.count + }; let _ = reply.reply(__reply_value); } } @@ -1766,12 +1779,17 @@ mod tests { } impl Actor for SlowActor {} impl Handler for SlowActor { - async fn handle(&mut self, _msg: SlowOp, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: SlowOp, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - // Never replies, so the timeout is the only possible outcome - // however late the runtime polls it. - std::future::pending::<()>().await; - } + // Never replies, so the timeout is the only possible outcome + // however late the runtime polls it. + std::future::pending::<()>().await; + } let _ = reply.reply(()); } } @@ -1819,10 +1837,15 @@ mod tests { } impl Handler for SlowShutdownActor { - async fn handle(&mut self, _msg: StopSlow, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + _msg: StopSlow, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.stop(); - } + ctx.stop(); + } let _ = reply.reply(()); } } @@ -1862,20 +1885,30 @@ mod tests { } impl Handler for HeldThread { - async fn handle(&mut self, _msg: StopSlow, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + _msg: StopSlow, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.stop(); - } + ctx.stop(); + } let _ = reply.reply(()); } } impl Handler for HeldThread { - async fn handle(&mut self, _msg: DoBlock, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + _msg: DoBlock, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - self.hold(); - ctx.stop(); - } + self.hold(); + ctx.stop(); + } let _ = reply.reply(()); } } @@ -1949,12 +1982,17 @@ mod tests { impl Actor for BadlyBehavedTask {} impl Handler for BadlyBehavedTask { - async fn handle(&mut self, _msg: DoBlock, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + _msg: DoBlock, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - rt::sleep(Duration::from_millis(20)).await; - thread::sleep(Duration::from_secs(2)); - ctx.stop(); - } + rt::sleep(Duration::from_millis(20)).await; + thread::sleep(Duration::from_secs(2)); + ctx.stop(); + } let _ = reply.reply(()); } } @@ -1971,31 +2009,44 @@ mod tests { impl Actor for WellBehavedTask {} impl Handler for WellBehavedTask { - async fn handle(&mut self, _msg: GetCount, reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context) { - let __reply_value: u64 = { - self.count - }; + async fn handle( + &mut self, + _msg: GetCount, + reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + let __reply_value: u64 = { self.count }; let _ = reply.reply(__reply_value); } } impl Handler for WellBehavedTask { - async fn handle(&mut self, _msg: StopCounter, reply: crate::reply::PendingReply<'_, u64>, ctx: &Context) { + async fn handle( + &mut self, + _msg: StopCounter, + reply: crate::reply::PendingReply<'_, u64>, + ctx: &Context, + ) { let __reply_value: u64 = { - ctx.stop(); - self.count - }; + ctx.stop(); + self.count + }; let _ = reply.reply(__reply_value); } } impl Handler for WellBehavedTask { - async fn handle(&mut self, _msg: IncrementWell, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + _msg: IncrementWell, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - self.count += 1; - use crate::tasks::send_after; - send_after(Duration::from_millis(100), ctx.clone(), IncrementWell); - } + self.count += 1; + use crate::tasks::send_after; + send_after(Duration::from_millis(100), ctx.clone(), IncrementWell); + } let _ = reply.reply(()); } } @@ -2086,7 +2137,12 @@ mod tests { } } impl Handler for PanicOnStart { - async fn handle(&mut self, _msg: Ping, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: Ping, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { let _ = reply.reply(()); } } @@ -2113,18 +2169,26 @@ mod tests { } impl Actor for PanicOnMsg {} impl Handler for PanicOnMsg { - async fn handle(&mut self, _msg: Explode, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: Explode, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - panic!("boom in handler"); - } + panic!("boom in handler"); + } let _ = reply.reply(()); } } impl Handler for PanicOnMsg { - async fn handle(&mut self, _msg: Check, reply: crate::reply::PendingReply<'_, u32>, _ctx: &Context) { - let __reply_value: u32 = { - 42 - }; + async fn handle( + &mut self, + _msg: Check, + reply: crate::reply::PendingReply<'_, u32>, + _ctx: &Context, + ) { + let __reply_value: u32 = { 42 }; let _ = reply.reply(__reply_value); } } @@ -2152,10 +2216,15 @@ mod tests { } } impl Handler for PanicOnStop { - async fn handle(&mut self, _msg: StopMe, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + _msg: StopMe, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.stop(); - } + ctx.stop(); + } let _ = reply.reply(()); } } @@ -2223,7 +2292,12 @@ mod tests { } } impl Handler for PanicStart { - async fn handle(&mut self, _msg: Ping, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: Ping, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { let _ = reply.reply(()); } } @@ -2247,10 +2321,15 @@ mod tests { } impl Actor for PanicHandler {} impl Handler for PanicHandler { - async fn handle(&mut self, _msg: Explode, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: Explode, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - panic!("boom in handler"); - } + panic!("boom in handler"); + } let _ = reply.reply(()); } } @@ -2303,37 +2382,53 @@ mod tests { impl Actor for Watcher {} impl Handler for Watcher { - async fn handle(&mut self, msg: StartMonitor, reply: crate::reply::PendingReply<'_, crate::monitor::MonitorRef>, ctx: &Context) { - let __reply_value: crate::monitor::MonitorRef = { - ctx.monitor(&msg.0) - }; + async fn handle( + &mut self, + msg: StartMonitor, + reply: crate::reply::PendingReply<'_, crate::monitor::MonitorRef>, + ctx: &Context, + ) { + let __reply_value: crate::monitor::MonitorRef = { ctx.monitor(&msg.0) }; let _ = reply.reply(__reply_value); } } impl Handler for Watcher { - async fn handle(&mut self, msg: CallDemonitor, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + msg: CallDemonitor, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.demonitor(msg.0); - } + ctx.demonitor(msg.0); + } let _ = reply.reply(()); } } impl Handler for Watcher { - async fn handle(&mut self, msg: crate::monitor::Down, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + async fn handle( + &mut self, + msg: crate::monitor::Down, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - self.downs.lock().unwrap().push(msg); - } + self.downs.lock().unwrap().push(msg); + } let _ = reply.reply(()); } } impl Handler for Watcher { - async fn handle(&mut self, _msg: GetDowns, reply: crate::reply::PendingReply<'_, Vec>, _ctx: &Context) { - let __reply_value: Vec = { - self.downs.lock().unwrap().clone() - }; + async fn handle( + &mut self, + _msg: GetDowns, + reply: crate::reply::PendingReply<'_, Vec>, + _ctx: &Context, + ) { + let __reply_value: Vec = { self.downs.lock().unwrap().clone() }; let _ = reply.reply(__reply_value); } } @@ -2462,15 +2557,20 @@ mod tests { type Result = usize; } impl Handler for Watcher { - async fn handle(&mut self, _msg: Inspect, reply: crate::reply::PendingReply<'_, usize>, ctx: &Context) { + async fn handle( + &mut self, + _msg: Inspect, + reply: crate::reply::PendingReply<'_, usize>, + ctx: &Context, + ) { let __reply_value: usize = { - ctx.shared - .monitors - .lock() - .unwrap() - .as_ref() - .map_or(0, HashMap::len) - }; + ctx.shared + .monitors + .lock() + .unwrap() + .as_ref() + .map_or(0, HashMap::len) + }; let _ = reply.reply(__reply_value); } } @@ -2513,10 +2613,15 @@ mod tests { struct PanicMe; impl Actor for PanicMe {} impl Handler for PanicMe { - async fn handle(&mut self, _msg: PanicMsg, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: PanicMsg, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - panic!("intentional panic"); - } + panic!("intentional panic"); + } let _ = reply.reply(()); } } @@ -2604,19 +2709,27 @@ mod tests { } impl Handler for TrapActor { - async fn handle(&mut self, msg: LinkTo, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + msg: LinkTo, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.link(&msg.0); - } + ctx.link(&msg.0); + } let _ = reply.reply(()); } } impl Handler for TrapActor { - async fn handle(&mut self, _msg: GetExits, reply: crate::reply::PendingReply<'_, Vec>, _ctx: &Context) { - let __reply_value: Vec = { - self.exits.lock().unwrap().clone() - }; + async fn handle( + &mut self, + _msg: GetExits, + reply: crate::reply::PendingReply<'_, Vec>, + _ctx: &Context, + ) { + let __reply_value: Vec = { self.exits.lock().unwrap().clone() }; let _ = reply.reply(__reply_value); } } @@ -2645,10 +2758,15 @@ mod tests { type Result = (); } impl Handler for TrapActor { - async fn handle(&mut self, _msg: Boom, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: Boom, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - panic!("boom"); - } + panic!("boom"); + } let _ = reply.reply(()); } } @@ -2676,10 +2794,15 @@ mod tests { type Result = (); } impl Handler for TrapActor { - async fn handle(&mut self, _msg: Boom2, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: Boom2, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - panic!("boom2"); - } + panic!("boom2"); + } let _ = reply.reply(()); } } @@ -2738,10 +2861,15 @@ mod tests { type Result = (); } impl Handler for TrapActor { - async fn handle(&mut self, _msg: Boom3, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: Boom3, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - panic!("boom3"); - } + panic!("boom3"); + } let _ = reply.reply(()); } } @@ -2785,10 +2913,15 @@ mod tests { type Result = (); } impl Handler for TrapActor { - async fn handle(&mut self, msg: UnlinkFrom, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + msg: UnlinkFrom, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.unlink(&msg.0); - } + ctx.unlink(&msg.0); + } let _ = reply.reply(()); } } @@ -2800,10 +2933,15 @@ mod tests { type Result = u64; } impl Handler for Counter { - async fn handle(&mut self, _msg: Boom4, reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: Boom4, + reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { let __reply_value: u64 = { - panic!("boom4"); - }; + panic!("boom4"); + }; let _ = reply.reply(__reply_value); } } @@ -2836,10 +2974,15 @@ mod tests { type Result = u64; } impl Handler for Counter { - async fn handle(&mut self, _msg: Boom5, reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: Boom5, + reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { let __reply_value: u64 = { - panic!("boom5"); - }; + panic!("boom5"); + }; let _ = reply.reply(__reply_value); } } @@ -2882,10 +3025,15 @@ mod tests { type Result = (); } impl Handler for TrapActor { - async fn handle(&mut self, _msg: Boom6, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: Boom6, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - panic!("boom6"); - } + panic!("boom6"); + } let _ = reply.reply(()); } } @@ -2918,10 +3066,15 @@ mod tests { type Result = (); } impl Handler for TrapActor { - async fn handle(&mut self, _msg: Boom7, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: Boom7, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - panic!("boom7"); - } + panic!("boom7"); + } let _ = reply.reply(()); } } @@ -2969,10 +3122,15 @@ mod tests { } impl Actor for Parent {} impl Handler for Parent { - async fn handle(&mut self, _msg: PanicParent, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: PanicParent, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - panic!("parent boom"); - } + panic!("parent boom"); + } let _ = reply.reply(()); } } @@ -3002,7 +3160,12 @@ mod monitor_cleanup_regressions { struct Idle; impl Actor for Idle {} impl Handler for Idle { - async fn handle(&mut self, _: Down, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Down, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } @@ -3145,10 +3308,15 @@ mod turn_regressions { type Result = (); } impl Handler for Count { - async fn handle(&mut self, _: Inc, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Inc, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0.fetch_add(1, Ordering::SeqCst); - } + self.0.fetch_add(1, Ordering::SeqCst); + } let _ = reply.reply(()); } } @@ -3214,10 +3382,15 @@ mod idle_stop_regression { type Result = (); } impl Handler for Counter { - async fn handle(&mut self, _: Increment, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Increment, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0.fetch_add(1, Ordering::SeqCst); - } + self.0.fetch_add(1, Ordering::SeqCst); + } let _ = reply.reply(()); } } @@ -3300,35 +3473,51 @@ mod api_coverage { } impl Actor for Echo {} impl Handler for Echo { - async fn handle(&mut self, msg: Ask, reply: crate::reply::PendingReply<'_, u32>, _: &Context) { - let __reply_value: u32 = { - msg.0 - }; + async fn handle( + &mut self, + msg: Ask, + reply: crate::reply::PendingReply<'_, u32>, + _: &Context, + ) { + let __reply_value: u32 = { msg.0 }; let _ = reply.reply(__reply_value); } } impl Handler for Echo { - async fn handle(&mut self, msg: StopAfter, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + msg: StopAfter, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - let _ = msg.0.await; - ctx.stop(); - } + let _ = msg.0.await; + ctx.stop(); + } let _ = reply.reply(()); } } impl Handler for Echo { - async fn handle(&mut self, _: OwnRecipient, reply: crate::reply::PendingReply<'_, (Recipient, ActorId)>, ctx: &Context) { - let __reply_value: (Recipient, ActorId) = { - (ctx.recipient::(), ctx.id()) - }; + async fn handle( + &mut self, + _: OwnRecipient, + reply: crate::reply::PendingReply<'_, (Recipient, ActorId)>, + ctx: &Context, + ) { + let __reply_value: (Recipient, ActorId) = { (ctx.recipient::(), ctx.id()) }; let _ = reply.reply(__reply_value); } } impl Handler for Echo { - async fn handle(&mut self, _: Boom, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Boom, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("boom"); - } + panic!("boom"); + } let _ = reply.reply(()); } } @@ -3410,10 +3599,13 @@ mod api_coverage { } } impl Handler for Trapper { - async fn handle(&mut self, msg: Ask, reply: crate::reply::PendingReply<'_, u32>, _: &Context) { - let __reply_value: u32 = { - msg.0 - }; + async fn handle( + &mut self, + msg: Ask, + reply: crate::reply::PendingReply<'_, u32>, + _: &Context, + ) { + let __reply_value: u32 = { msg.0 }; let _ = reply.reply(__reply_value); } } @@ -3465,7 +3657,12 @@ mod shared_state_release { struct Plain; impl Actor for Plain {} impl Handler for Plain { - async fn handle(&mut self, _: Down, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Down, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } diff --git a/concurrency/src/tasks/stream_tests.rs b/concurrency/src/tasks/stream_tests.rs index 26622e4c..167515d9 100644 --- a/concurrency/src/tasks/stream_tests.rs +++ b/concurrency/src/tasks/stream_tests.rs @@ -43,31 +43,44 @@ impl Summatory { impl Actor for Summatory {} impl Handler for Summatory { - async fn handle(&mut self, msg: StreamMsg, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + msg: StreamMsg, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - match msg { - StreamMsg::Add(val) => self.count += val, - StreamMsg::Error => ctx.stop(), + match msg { + StreamMsg::Add(val) => self.count += val, + StreamMsg::Error => ctx.stop(), + } } - } let _ = reply.reply(()); } } impl Handler for Summatory { - async fn handle(&mut self, _msg: StopSum, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + _msg: StopSum, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.stop(); - } + ctx.stop(); + } let _ = reply.reply(()); } } impl Handler for Summatory { - async fn handle(&mut self, _msg: GetValue, reply: crate::reply::PendingReply<'_, u16>, _ctx: &Context) { - let __reply_value: u16 = { - self.count - }; + async fn handle( + &mut self, + _msg: GetValue, + reply: crate::reply::PendingReply<'_, u16>, + _ctx: &Context, + ) { + let __reply_value: u16 = { self.count }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/src/tasks/timer_tests.rs b/concurrency/src/tasks/timer_tests.rs index 42ad58a6..b542a2d8 100644 --- a/concurrency/src/tasks/timer_tests.rs +++ b/concurrency/src/tasks/timer_tests.rs @@ -46,30 +46,43 @@ impl Actor for Repeater { } impl Handler for Repeater { - async fn handle(&mut self, _msg: Inc, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: Inc, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - self.count += 1; - } + self.count += 1; + } let _ = reply.reply(()); } } impl Handler for Repeater { - async fn handle(&mut self, _msg: StopTimer, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: StopTimer, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - if let Some(ct) = self.cancellation_token.clone() { - ct.cancel(); + if let Some(ct) = self.cancellation_token.clone() { + ct.cancel(); + } } - } let _ = reply.reply(()); } } impl Handler for Repeater { - async fn handle(&mut self, _msg: GetRepCount, reply: crate::reply::PendingReply<'_, i32>, _ctx: &Context) { - let __reply_value: i32 = { - self.count - }; + async fn handle( + &mut self, + _msg: GetRepCount, + reply: crate::reply::PendingReply<'_, i32>, + _ctx: &Context, + ) { + let __reply_value: i32 = { self.count }; let _ = reply.reply(__reply_value); } } @@ -131,29 +144,42 @@ impl Delayed { impl Actor for Delayed {} impl Handler for Delayed { - async fn handle(&mut self, _msg: Inc, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: Inc, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - self.count += 1; - } + self.count += 1; + } let _ = reply.reply(()); } } impl Handler for Delayed { - async fn handle(&mut self, _msg: GetDelCount, reply: crate::reply::PendingReply<'_, i32>, _ctx: &Context) { - let __reply_value: i32 = { - self.count - }; + async fn handle( + &mut self, + _msg: GetDelCount, + reply: crate::reply::PendingReply<'_, i32>, + _ctx: &Context, + ) { + let __reply_value: i32 = { self.count }; let _ = reply.reply(__reply_value); } } impl Handler for Delayed { - async fn handle(&mut self, _msg: StopDelayed, reply: crate::reply::PendingReply<'_, i32>, ctx: &Context) { + async fn handle( + &mut self, + _msg: StopDelayed, + reply: crate::reply::PendingReply<'_, i32>, + ctx: &Context, + ) { let __reply_value: i32 = { - ctx.stop(); - self.count - }; + ctx.stop(); + self.count + }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 75828115..f4ffda20 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -1094,30 +1094,43 @@ mod tests { impl Actor for Counter {} impl Handler for Counter { - fn handle(&mut self, _msg: GetCount, reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context) { - let __reply_value: u64 = { - self.count - }; + fn handle( + &mut self, + _msg: GetCount, + reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + let __reply_value: u64 = { self.count }; let _ = reply.reply(__reply_value); } } impl Handler for Counter { - fn handle(&mut self, _msg: Increment, reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context) { + fn handle( + &mut self, + _msg: Increment, + reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { let __reply_value: u64 = { - self.count += 1; - self.count - }; + self.count += 1; + self.count + }; let _ = reply.reply(__reply_value); } } impl Handler for Counter { - fn handle(&mut self, _msg: StopCounter, reply: crate::reply::PendingReply<'_, u64>, ctx: &Context) { + fn handle( + &mut self, + _msg: StopCounter, + reply: crate::reply::PendingReply<'_, u64>, + ctx: &Context, + ) { let __reply_value: u64 = { - ctx.stop(); - self.count - }; + ctx.stop(); + self.count + }; let _ = reply.reply(__reply_value); } } @@ -1146,10 +1159,15 @@ mod tests { } } impl Handler for SlowStop { - fn handle(&mut self, _msg: StopSlow, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + fn handle( + &mut self, + _msg: StopSlow, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.stop(); - } + ctx.stop(); + } let _ = reply.reply(()); } } @@ -1173,10 +1191,15 @@ mod tests { } } impl Handler for SlowStop2 { - fn handle(&mut self, _msg: StopSlow2, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + fn handle( + &mut self, + _msg: StopSlow2, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.stop(); - } + ctx.stop(); + } let _ = reply.reply(()); } } @@ -1210,7 +1233,12 @@ mod tests { } } impl Handler for PanicOnStart { - fn handle(&mut self, _msg: PingThread, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + fn handle( + &mut self, + _msg: PingThread, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { let _ = reply.reply(()); } } @@ -1234,18 +1262,26 @@ mod tests { } impl Actor for PanicOnMsg {} impl Handler for PanicOnMsg { - fn handle(&mut self, _msg: ExplodeThread, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + fn handle( + &mut self, + _msg: ExplodeThread, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - panic!("boom in handler"); - } + panic!("boom in handler"); + } let _ = reply.reply(()); } } impl Handler for PanicOnMsg { - fn handle(&mut self, _msg: CheckThread, reply: crate::reply::PendingReply<'_, u32>, _ctx: &Context) { - let __reply_value: u32 = { - 42 - }; + fn handle( + &mut self, + _msg: CheckThread, + reply: crate::reply::PendingReply<'_, u32>, + _ctx: &Context, + ) { + let __reply_value: u32 = { 42 }; let _ = reply.reply(__reply_value); } } @@ -1270,10 +1306,15 @@ mod tests { } } impl Handler for PanicOnStop { - fn handle(&mut self, _msg: StopMeThread, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + fn handle( + &mut self, + _msg: StopMeThread, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.stop(); - } + ctx.stop(); + } let _ = reply.reply(()); } } @@ -1324,7 +1365,12 @@ mod tests { } } impl Handler for PanicStartThread { - fn handle(&mut self, _msg: PingThread2, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + fn handle( + &mut self, + _msg: PingThread2, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { let _ = reply.reply(()); } } @@ -1343,10 +1389,15 @@ mod tests { } impl Actor for PanicHandlerThread {} impl Handler for PanicHandlerThread { - fn handle(&mut self, _msg: ExplodeThread2, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + fn handle( + &mut self, + _msg: ExplodeThread2, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - panic!("boom in handler"); - } + panic!("boom in handler"); + } let _ = reply.reply(()); } } @@ -1389,37 +1440,53 @@ mod tests { impl Actor for Watcher {} impl Handler for Watcher { - fn handle(&mut self, msg: StartMonitor, reply: crate::reply::PendingReply<'_, crate::monitor::MonitorRef>, ctx: &Context) { - let __reply_value: crate::monitor::MonitorRef = { - ctx.monitor(&msg.0) - }; + fn handle( + &mut self, + msg: StartMonitor, + reply: crate::reply::PendingReply<'_, crate::monitor::MonitorRef>, + ctx: &Context, + ) { + let __reply_value: crate::monitor::MonitorRef = { ctx.monitor(&msg.0) }; let _ = reply.reply(__reply_value); } } impl Handler for Watcher { - fn handle(&mut self, msg: CallDemonitor, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + fn handle( + &mut self, + msg: CallDemonitor, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.demonitor(msg.0); - } + ctx.demonitor(msg.0); + } let _ = reply.reply(()); } } impl Handler for Watcher { - fn handle(&mut self, msg: crate::monitor::Down, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + fn handle( + &mut self, + msg: crate::monitor::Down, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - self.downs.lock().unwrap().push(msg); - } + self.downs.lock().unwrap().push(msg); + } let _ = reply.reply(()); } } impl Handler for Watcher { - fn handle(&mut self, _msg: GetDowns, reply: crate::reply::PendingReply<'_, Vec>, _ctx: &Context) { - let __reply_value: Vec = { - self.downs.lock().unwrap().clone() - }; + fn handle( + &mut self, + _msg: GetDowns, + reply: crate::reply::PendingReply<'_, Vec>, + _ctx: &Context, + ) { + let __reply_value: Vec = { self.downs.lock().unwrap().clone() }; let _ = reply.reply(__reply_value); } } @@ -1519,10 +1586,15 @@ mod tests { struct PanicMe; impl Actor for PanicMe {} impl Handler for PanicMe { - fn handle(&mut self, _msg: PanicMsg, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + fn handle( + &mut self, + _msg: PanicMsg, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - panic!("intentional panic"); - } + panic!("intentional panic"); + } let _ = reply.reply(()); } } @@ -1569,19 +1641,27 @@ mod tests { } impl Handler for TrapActor { - fn handle(&mut self, msg: LinkTo, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + fn handle( + &mut self, + msg: LinkTo, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.link(&msg.0); - } + ctx.link(&msg.0); + } let _ = reply.reply(()); } } impl Handler for TrapActor { - fn handle(&mut self, _msg: GetExits, reply: crate::reply::PendingReply<'_, Vec>, _ctx: &Context) { - let __reply_value: Vec = { - self.exits.lock().unwrap().clone() - }; + fn handle( + &mut self, + _msg: GetExits, + reply: crate::reply::PendingReply<'_, Vec>, + _ctx: &Context, + ) { + let __reply_value: Vec = { self.exits.lock().unwrap().clone() }; let _ = reply.reply(__reply_value); } } @@ -1601,10 +1681,15 @@ mod tests { type Result = (); } impl Handler for TrapActor { - fn handle(&mut self, _msg: Boom, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + fn handle( + &mut self, + _msg: Boom, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - panic!("boom"); - } + panic!("boom"); + } let _ = reply.reply(()); } } @@ -1627,10 +1712,15 @@ mod tests { type Result = (); } impl Handler for TrapActor { - fn handle(&mut self, _msg: Boom, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + fn handle( + &mut self, + _msg: Boom, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - panic!("boom"); - } + panic!("boom"); + } let _ = reply.reply(()); } } @@ -1665,10 +1755,15 @@ mod tests { } impl Actor for Parent {} impl Handler for Parent { - fn handle(&mut self, _msg: PanicParent, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + fn handle( + &mut self, + _msg: PanicParent, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - panic!("parent boom"); - } + panic!("parent boom"); + } let _ = reply.reply(()); } } @@ -1692,7 +1787,12 @@ mod monitor_cleanup_regressions { struct Idle; impl Actor for Idle {} impl Handler for Idle { - fn handle(&mut self, _: Down, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + _: Down, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } @@ -1804,10 +1904,15 @@ mod idle_stop_regression { type Result = (); } impl Handler for Counter { - fn handle(&mut self, _: Increment, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + _: Increment, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0.fetch_add(1, Ordering::SeqCst); - } + self.0.fetch_add(1, Ordering::SeqCst); + } let _ = reply.reply(()); } } @@ -1895,7 +2000,12 @@ mod stop_wakeup_regression { type Result = (); } impl Handler for Idle { - fn handle(&mut self, _: Noop, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + _: Noop, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } @@ -1975,10 +2085,15 @@ mod stop_wakeup_regression { struct Failing; impl Actor for Failing {} impl Handler for Failing { - fn handle(&mut self, _: Crash, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + _: Crash, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("linked wakeup regression"); - } + panic!("linked wakeup regression"); + } let _ = reply.reply(()); } } @@ -2034,12 +2149,17 @@ mod stop_wakeup_regression { type Result = (); } impl Handler for Busy { - fn handle(&mut self, _: Work, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + _: Work, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.handled.fetch_add(1, Ordering::SeqCst); - let _ = self.entered.send(()); - self.release.recv_timeout(Duration::from_secs(5)).unwrap(); - } + self.handled.fetch_add(1, Ordering::SeqCst); + let _ = self.entered.send(()); + self.release.recv_timeout(Duration::from_secs(5)).unwrap(); + } let _ = reply.reply(()); } } @@ -2113,35 +2233,51 @@ mod api_coverage { } impl Actor for Echo {} impl Handler for Echo { - fn handle(&mut self, msg: Ask, reply: crate::reply::PendingReply<'_, u32>, _: &Context) { - let __reply_value: u32 = { - msg.0 - }; + fn handle( + &mut self, + msg: Ask, + reply: crate::reply::PendingReply<'_, u32>, + _: &Context, + ) { + let __reply_value: u32 = { msg.0 }; let _ = reply.reply(__reply_value); } } impl Handler for Echo { - fn handle(&mut self, msg: StopAfter, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + fn handle( + &mut self, + msg: StopAfter, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - let _ = msg.0.recv_timeout(WAIT); - ctx.stop(); - } + let _ = msg.0.recv_timeout(WAIT); + ctx.stop(); + } let _ = reply.reply(()); } } impl Handler for Echo { - fn handle(&mut self, _: OwnRecipient, reply: crate::reply::PendingReply<'_, (Recipient, ActorId)>, ctx: &Context) { - let __reply_value: (Recipient, ActorId) = { - (ctx.recipient::(), ctx.id()) - }; + fn handle( + &mut self, + _: OwnRecipient, + reply: crate::reply::PendingReply<'_, (Recipient, ActorId)>, + ctx: &Context, + ) { + let __reply_value: (Recipient, ActorId) = { (ctx.recipient::(), ctx.id()) }; let _ = reply.reply(__reply_value); } } impl Handler for Echo { - fn handle(&mut self, _: Boom, reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + _: Boom, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("boom"); - } + panic!("boom"); + } let _ = reply.reply(()); } } @@ -2220,18 +2356,26 @@ mod api_coverage { } } impl Handler for Trapper { - fn handle(&mut self, msg: Ask, reply: crate::reply::PendingReply<'_, u32>, _: &Context) { - let __reply_value: u32 = { - msg.0 - }; + fn handle( + &mut self, + msg: Ask, + reply: crate::reply::PendingReply<'_, u32>, + _: &Context, + ) { + let __reply_value: u32 = { msg.0 }; let _ = reply.reply(__reply_value); } } impl Handler for Trapper { - fn handle(&mut self, msg: Unlink, reply: crate::reply::PendingReply<'_, ()>, ctx: &Context) { + fn handle( + &mut self, + msg: Unlink, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.unlink(&msg.0); - } + ctx.unlink(&msg.0); + } let _ = reply.reply(()); } } diff --git a/concurrency/src/threads/stream_tests.rs b/concurrency/src/threads/stream_tests.rs index 8ef180f0..b059da36 100644 --- a/concurrency/src/threads/stream_tests.rs +++ b/concurrency/src/threads/stream_tests.rs @@ -28,20 +28,28 @@ impl Message for GetItems { impl Actor for Collector {} impl Handler for Collector { - fn handle(&mut self, msg: Push, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + fn handle( + &mut self, + msg: Push, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - self.items.push(msg.value); - self.done.fetch_add(1, Ordering::SeqCst); - } + self.items.push(msg.value); + self.done.fetch_add(1, Ordering::SeqCst); + } let _ = reply.reply(()); } } impl Handler for Collector { - fn handle(&mut self, _msg: GetItems, reply: crate::reply::PendingReply<'_, Vec>, _ctx: &Context) { - let __reply_value: Vec = { - self.items.clone() - }; + fn handle( + &mut self, + _msg: GetItems, + reply: crate::reply::PendingReply<'_, Vec>, + _ctx: &Context, + ) { + let __reply_value: Vec = { self.items.clone() }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/src/threads/timer_tests.rs b/concurrency/src/threads/timer_tests.rs index 04a76dab..ecfa34f9 100644 --- a/concurrency/src/threads/timer_tests.rs +++ b/concurrency/src/threads/timer_tests.rs @@ -46,30 +46,43 @@ impl Actor for Repeater { } impl Handler for Repeater { - fn handle(&mut self, _msg: Inc, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + fn handle( + &mut self, + _msg: Inc, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - self.count += 1; - } + self.count += 1; + } let _ = reply.reply(()); } } impl Handler for Repeater { - fn handle(&mut self, _msg: StopTimer, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + fn handle( + &mut self, + _msg: StopTimer, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - if let Some(ct) = self.cancellation_token.clone() { - ct.cancel(); + if let Some(ct) = self.cancellation_token.clone() { + ct.cancel(); + } } - } let _ = reply.reply(()); } } impl Handler for Repeater { - fn handle(&mut self, _msg: GetRepCount, reply: crate::reply::PendingReply<'_, i32>, _ctx: &Context) { - let __reply_value: i32 = { - self.count - }; + fn handle( + &mut self, + _msg: GetRepCount, + reply: crate::reply::PendingReply<'_, i32>, + _ctx: &Context, + ) { + let __reply_value: i32 = { self.count }; let _ = reply.reply(__reply_value); } } @@ -128,29 +141,42 @@ impl Delayed { impl Actor for Delayed {} impl Handler for Delayed { - fn handle(&mut self, _msg: Inc, reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context) { + fn handle( + &mut self, + _msg: Inc, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - self.count += 1; - } + self.count += 1; + } let _ = reply.reply(()); } } impl Handler for Delayed { - fn handle(&mut self, _msg: GetDelCount, reply: crate::reply::PendingReply<'_, i32>, _ctx: &Context) { - let __reply_value: i32 = { - self.count - }; + fn handle( + &mut self, + _msg: GetDelCount, + reply: crate::reply::PendingReply<'_, i32>, + _ctx: &Context, + ) { + let __reply_value: i32 = { self.count }; let _ = reply.reply(__reply_value); } } impl Handler for Delayed { - fn handle(&mut self, _msg: StopDelayed, reply: crate::reply::PendingReply<'_, i32>, ctx: &Context) { + fn handle( + &mut self, + _msg: StopDelayed, + reply: crate::reply::PendingReply<'_, i32>, + ctx: &Context, + ) { let __reply_value: i32 = { - ctx.stop(); - self.count - }; + ctx.stop(); + self.count + }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/actor_scenarios.rs b/concurrency/tests/actor_scenarios.rs index f923cfe4..59cd899c 100644 --- a/concurrency/tests/actor_scenarios.rs +++ b/concurrency/tests/actor_scenarios.rs @@ -37,33 +37,43 @@ mod tasks { pub struct TestActor(pub Worker); impl Actor for TestActor {} impl Handler for TestActor { - async fn handle(&mut self, command: Command, reply: spawned_concurrency::reply::PendingReply<'_, Reply>, ctx: &Context) { + async fn handle( + &mut self, + command: Command, + reply: spawned_concurrency::reply::PendingReply<'_, Reply>, + ctx: &Context, + ) { let __reply_value: Reply = { - match command { - Command::Work(id) => { - self.0 - .processed - .lock() - .unwrap_or_else(|p| p.into_inner()) - .push(id); - Reply::Work(id) - } - Command::Watch(target) => Reply::Watch(ctx.monitor(&target)), - Command::Unwatch(id) => { - ctx.demonitor(id); - Reply::Unwatched + match command { + Command::Work(id) => { + self.0 + .processed + .lock() + .unwrap_or_else(|p| p.into_inner()) + .push(id); + Reply::Work(id) + } + Command::Watch(target) => Reply::Watch(ctx.monitor(&target)), + Command::Unwatch(id) => { + ctx.demonitor(id); + Reply::Unwatched + } + Command::Boom => panic!("scenario injected panic"), } - Command::Boom => panic!("scenario injected panic"), - } - }; + }; let _ = reply.reply(__reply_value); } } impl Handler for TestActor { - async fn handle(&mut self, event: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + event: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - let _ = self.0.events.send(event); - } + let _ = self.0.events.send(event); + } let _ = reply.reply(()); } } @@ -75,33 +85,43 @@ mod threads { pub struct TestActor(pub Worker); impl Actor for TestActor {} impl Handler for TestActor { - fn handle(&mut self, command: Command, reply: spawned_concurrency::reply::PendingReply<'_, Reply>, ctx: &Context) { + fn handle( + &mut self, + command: Command, + reply: spawned_concurrency::reply::PendingReply<'_, Reply>, + ctx: &Context, + ) { let __reply_value: Reply = { - match command { - Command::Work(id) => { - self.0 - .processed - .lock() - .unwrap_or_else(|p| p.into_inner()) - .push(id); - Reply::Work(id) - } - Command::Watch(target) => Reply::Watch(ctx.monitor(&target)), - Command::Unwatch(id) => { - ctx.demonitor(id); - Reply::Unwatched + match command { + Command::Work(id) => { + self.0 + .processed + .lock() + .unwrap_or_else(|p| p.into_inner()) + .push(id); + Reply::Work(id) + } + Command::Watch(target) => Reply::Watch(ctx.monitor(&target)), + Command::Unwatch(id) => { + ctx.demonitor(id); + Reply::Unwatched + } + Command::Boom => panic!("scenario injected panic"), } - Command::Boom => panic!("scenario injected panic"), - } - }; + }; let _ = reply.reply(__reply_value); } } impl Handler for TestActor { - fn handle(&mut self, event: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + event: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - let _ = self.0.events.send(event); - } + let _ = self.0.events.send(event); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/backend_experiments.rs b/concurrency/tests/backend_experiments.rs index 7421262e..087786a5 100644 --- a/concurrency/tests/backend_experiments.rs +++ b/concurrency/tests/backend_experiments.rs @@ -17,27 +17,37 @@ impl Message for Nested { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: Probe, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + async fn handle( + &mut self, + _: Probe, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - let child = tokio::spawn(async { - tokio::time::sleep(Duration::from_millis(1)).await; - 42 - }); - child.await.unwrap() - }; + let child = tokio::spawn(async { + tokio::time::sleep(Duration::from_millis(1)).await; + 42 + }); + child.await.unwrap() + }; let _ = reply.reply(__reply_value); } } impl Handler for Worker { - async fn handle(&mut self, _: Nested, reply: spawned_concurrency::reply::PendingReply<'_, bool>, _: &Context) { + async fn handle( + &mut self, + _: Nested, + reply: spawned_concurrency::reply::PendingReply<'_, bool>, + _: &Context, + ) { let __reply_value: bool = { - let (tx, rx) = mpsc::channel(); - tokio::spawn(async move { - let _ = tx.send(()); - }); - // Characterizes progress of a detached helper while its actor blocks. - rx.recv_timeout(Duration::from_millis(100)).is_ok() - }; + let (tx, rx) = mpsc::channel(); + tokio::spawn(async move { + let _ = tx.send(()); + }); + // Characterizes progress of a detached helper while its actor blocks. + rx.recv_timeout(Duration::from_millis(100)).is_ok() + }; let _ = reply.reply(__reply_value); } } @@ -69,28 +79,33 @@ impl Message for Network { type Result = u64; } impl Handler for Worker { - async fn handle(&mut self, _: Network, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + async fn handle( + &mut self, + _: Network, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - use std::io::{Read, Write}; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - let addr = listener.local_addr().unwrap(); - let server = std::thread::spawn(move || { - let (mut socket, _) = listener.accept().unwrap(); - socket - .set_read_timeout(Some(Duration::from_secs(3))) - .unwrap(); + use std::io::{Read, Write}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut socket, _) = listener.accept().unwrap(); + socket + .set_read_timeout(Some(Duration::from_secs(3))) + .unwrap(); + let mut data = [0; 8]; + socket.read_exact(&mut data).unwrap(); + socket.write_all(&data).unwrap(); + }); + let mut socket = tokio::net::TcpStream::connect(addr).await.unwrap(); + socket.write_all(&42u64.to_le_bytes()).await.unwrap(); let mut data = [0; 8]; - socket.read_exact(&mut data).unwrap(); - socket.write_all(&data).unwrap(); - }); - let mut socket = tokio::net::TcpStream::connect(addr).await.unwrap(); - socket.write_all(&42u64.to_le_bytes()).await.unwrap(); - let mut data = [0; 8]; - socket.read_exact(&mut data).await.unwrap(); - server.join().unwrap(); - u64::from_le_bytes(data) - }; + socket.read_exact(&mut data).await.unwrap(); + server.join().unwrap(); + u64::from_le_bytes(data) + }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/backend_resource_limits.rs b/concurrency/tests/backend_resource_limits.rs index facaf343..d802965f 100644 --- a/concurrency/tests/backend_resource_limits.rs +++ b/concurrency/tests/backend_resource_limits.rs @@ -30,7 +30,12 @@ impl Actor for Worker { } } impl Handler for Worker { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } diff --git a/concurrency/tests/borrowed_runtime.rs b/concurrency/tests/borrowed_runtime.rs index 3b53d59d..3557d7b3 100644 --- a/concurrency/tests/borrowed_runtime.rs +++ b/concurrency/tests/borrowed_runtime.rs @@ -20,18 +20,24 @@ impl Message for Ambient { } impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: Identity, reply: spawned_concurrency::reply::PendingReply<'_, ThreadId>, _: &Context) { - let __reply_value: ThreadId = { - std::thread::current().id() - }; + async fn handle( + &mut self, + _: Identity, + reply: spawned_concurrency::reply::PendingReply<'_, ThreadId>, + _: &Context, + ) { + let __reply_value: ThreadId = { std::thread::current().id() }; let _ = reply.reply(__reply_value); } } impl Handler for Worker { - async fn handle(&mut self, _: Ambient, reply: spawned_concurrency::reply::PendingReply<'_, ActorRef>, _: &Context) { - let __reply_value: ActorRef = { - Worker.start() - }; + async fn handle( + &mut self, + _: Ambient, + reply: spawned_concurrency::reply::PendingReply<'_, ActorRef>, + _: &Context, + ) { + let __reply_value: ActorRef = { Worker.start() }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/canary_feedback.rs b/concurrency/tests/canary_feedback.rs index 94ca3e29..32dbdfc2 100644 --- a/concurrency/tests/canary_feedback.rs +++ b/concurrency/tests/canary_feedback.rs @@ -33,19 +33,29 @@ impl Message for Quit { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, hold: Hold, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + hold: Hold, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - let _ = hold.entered.send(()); - let _ = hold.release.await; - } + let _ = hold.entered.send(()); + let _ = hold.release.await; + } let _ = reply.reply(()); } } impl Handler for Worker { - async fn handle(&mut self, quit: Quit, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + quit: Quit, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.stop_with_reason(quit.0); - } + ctx.stop_with_reason(quit.0); + } let _ = reply.reply(()); } } @@ -400,10 +410,15 @@ async fn panicked_turn_is_cleared_before_stopped_hook() { } } impl Handler for Panicking { - async fn handle(&mut self, _: Quit, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Quit, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("diagnostic turn cleanup regression"); - } + panic!("diagnostic turn cleanup regression"); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/cancellation_safety.rs b/concurrency/tests/cancellation_safety.rs index 648a3891..e09e8efd 100644 --- a/concurrency/tests/cancellation_safety.rs +++ b/concurrency/tests/cancellation_safety.rs @@ -189,43 +189,53 @@ impl Message for Cmd { type Result = u64; } impl Handler for Probe { - async fn handle(&mut self, cmd: Cmd, reply: spawned_concurrency::reply::PendingReply<'_, u64>, ctx: &Context) { + async fn handle( + &mut self, + cmd: Cmd, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + ctx: &Context, + ) { let __reply_value: u64 = { - let c = &self.counters; - match cmd { - Cmd::Bump => c.effects.fetch_add(1, Ordering::SeqCst) as u64 + 1, - Cmd::Read => c.effects.load(Ordering::SeqCst) as u64, - Cmd::Watch(target) => { - ctx.monitor(&target); - c.watched.fetch_add(1, Ordering::SeqCst) as u64 - } - Cmd::Link(target) => { - ctx.link(&target); - c.linked.fetch_add(1, Ordering::SeqCst) as u64 - } - Cmd::Trap => { - ctx.trap_exit(true); - 0 - } - Cmd::Boom => panic!("cancellation fixture crash"), - Cmd::Helper(gate) => { - let task = ctx - .spawn_helper(async move { gate.notified().await }) - .unwrap(); - // Drop the ScopedTask: the helper stays owned by the actor scope. - drop(task); - 0 + let c = &self.counters; + match cmd { + Cmd::Bump => c.effects.fetch_add(1, Ordering::SeqCst) as u64 + 1, + Cmd::Read => c.effects.load(Ordering::SeqCst) as u64, + Cmd::Watch(target) => { + ctx.monitor(&target); + c.watched.fetch_add(1, Ordering::SeqCst) as u64 + } + Cmd::Link(target) => { + ctx.link(&target); + c.linked.fetch_add(1, Ordering::SeqCst) as u64 + } + Cmd::Trap => { + ctx.trap_exit(true); + 0 + } + Cmd::Boom => panic!("cancellation fixture crash"), + Cmd::Helper(gate) => { + let task = ctx + .spawn_helper(async move { gate.notified().await }) + .unwrap(); + // Drop the ScopedTask: the helper stays owned by the actor scope. + drop(task); + 0 + } } - } - }; + }; let _ = reply.reply(__reply_value); } } impl Handler for Probe { - async fn handle(&mut self, _: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.counters.downs.fetch_add(1, Ordering::SeqCst); - } + self.counters.downs.fetch_add(1, Ordering::SeqCst); + } let _ = reply.reply(()); } } @@ -235,10 +245,14 @@ pub trait Bumper: Send + Sync { fn bump(&self) -> Response; } impl Handler for Probe { - async fn handle(&mut self, _: bumper::Bump, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - self.counters.effects.fetch_add(1, Ordering::SeqCst) as u64 + 1 - }; + async fn handle( + &mut self, + _: bumper::Bump, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = + { self.counters.effects.fetch_add(1, Ordering::SeqCst) as u64 + 1 }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/cfg_gated_protocol.rs b/concurrency/tests/cfg_gated_protocol.rs index 57ebf5f8..25f6f309 100644 --- a/concurrency/tests/cfg_gated_protocol.rs +++ b/concurrency/tests/cfg_gated_protocol.rs @@ -23,21 +23,29 @@ struct MixedActor { impl Actor for MixedActor {} impl Handler for MixedActor { - async fn handle(&mut self, _msg: mixed_protocol::GetValue, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _ctx: &Context) { - let __reply_value: u64 = { - self.value - }; + async fn handle( + &mut self, + _msg: mixed_protocol::GetValue, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + let __reply_value: u64 = { self.value }; let _ = reply.reply(__reply_value); } } #[cfg(test)] impl Handler for MixedActor { - async fn handle(&mut self, _msg: mixed_protocol::TestOnlyIncrement, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: mixed_protocol::TestOnlyIncrement, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { let __reply_value: u64 = { - self.value += 1; - self.value - }; + self.value += 1; + self.value + }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/cohort_trace.rs b/concurrency/tests/cohort_trace.rs index 5bfaeca6..84d7d12a 100644 --- a/concurrency/tests/cohort_trace.rs +++ b/concurrency/tests/cohort_trace.rs @@ -15,10 +15,15 @@ impl Message for Crash { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("trace cohort failure"); - } + panic!("trace cohort failure"); + } let _ = reply.reply(()); } } @@ -271,10 +276,15 @@ impl Actor for MaybeStop { } } impl Handler for MaybeStop { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("trace cohort failure"); - } + panic!("trace cohort failure"); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/context_exit.rs b/concurrency/tests/context_exit.rs index c6954c7b..a5e540ee 100644 --- a/concurrency/tests/context_exit.rs +++ b/concurrency/tests/context_exit.rs @@ -20,11 +20,16 @@ fn native_explicit_exit_is_cooperative_and_first_reason_wins() { } } impl Handler for Worker { - fn handle(&mut self, exit: Exit, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + fn handle( + &mut self, + exit: Exit, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.stop_with_reason(exit.0); - ctx.stop(); - } + ctx.stop_with_reason(exit.0); + ctx.stop(); + } let _ = reply.reply(()); } } @@ -53,11 +58,16 @@ async fn async_explicit_exit_is_cooperative_and_first_reason_wins() { } } impl Handler for Worker { - async fn handle(&mut self, exit: Exit, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + exit: Exit, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.stop_with_reason(exit.0); - ctx.stop(); - } + ctx.stop_with_reason(exit.0); + ctx.stop(); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/coverage_trace.rs b/concurrency/tests/coverage_trace.rs index 42c41448..896e6d80 100644 --- a/concurrency/tests/coverage_trace.rs +++ b/concurrency/tests/coverage_trace.rs @@ -91,10 +91,15 @@ impl Message for Crash { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("injected child failure"); - } + panic!("injected child failure"); + } let _ = reply.reply(()); } } @@ -293,10 +298,15 @@ impl Actor for QuitInStarted { } } impl Handler for QuitInStarted { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("injected child failure"); - } + panic!("injected child failure"); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/cross_backend.rs b/concurrency/tests/cross_backend.rs index e437e201..2b2e13ae 100644 --- a/concurrency/tests/cross_backend.rs +++ b/concurrency/tests/cross_backend.rs @@ -156,116 +156,135 @@ mod tasks_node { } } impl Handler for Node { - async fn handle(&mut self, cmd: Cmd, reply: spawned_concurrency::reply::PendingReply<'_, Reply>, ctx: &Context) { + async fn handle( + &mut self, + cmd: Cmd, + reply: spawned_concurrency::reply::PendingReply<'_, Reply>, + ctx: &Context, + ) { let __reply_value: Reply = async { - match cmd { - Cmd::Echo(v) => self.state.echo(v), - Cmd::Chain(v) => match self.peers.first() { - None => self.state.leaf(v), - Some(next) => next - .request_with_timeout(Cmd::Chain(chain_arg(v, self.state.index)), LIMIT) - .await - .unwrap_or(Reply::Failed), - }, - Cmd::FanOut(v) => { - let calls = - self.peers.iter().enumerate().map(|(i, peer)| { + match cmd { + Cmd::Echo(v) => self.state.echo(v), + Cmd::Chain(v) => match self.peers.first() { + None => self.state.leaf(v), + Some(next) => next + .request_with_timeout(Cmd::Chain(chain_arg(v, self.state.index)), LIMIT) + .await + .unwrap_or(Reply::Failed), + }, + Cmd::FanOut(v) => { + let calls = self.peers.iter().enumerate().map(|(i, peer)| { peer.request_with_timeout(Cmd::Echo(v + i as u64), LIMIT) }); - let mut values = Vec::new(); - for reply in futures::future::join_all(calls).await { - match reply { - Ok(Reply::Value(v)) => values.push(v), - _ => return Reply::Failed, + let mut values = Vec::new(); + for reply in futures::future::join_all(calls).await { + match reply { + Ok(Reply::Value(v)) => values.push(v), + _ => return Reply::Failed, + } } + Reply::Values(values) } - Reply::Values(values) - } - Cmd::Work { sender, seq } => { - self.state.work.push((sender, seq)); - Reply::Unit - } - Cmd::Log => Reply::Log(self.state.work.clone()), - Cmd::Watch(target) => Reply::Monitor(ctx.monitor(&target)), - Cmd::Link(target) => { - ctx.link(&target); - Reply::Unit - } - Cmd::Trap => { - ctx.trap_exit(true); - Reply::Unit - } - Cmd::Boom => panic!("cross-backend injected panic"), - Cmd::Stop => { - ctx.stop(); - Reply::Unit - } - Cmd::Timers { after, pulses } => { - self.state.ticks_wanted = after; - self.state.pulses_wanted = pulses; - for n in 0..after { - send_after(TICK_STEP * (n + 1), ctx.clone(), Tick(n)); + Cmd::Work { sender, seq } => { + self.state.work.push((sender, seq)); + Reply::Unit } - if pulses > 0 { - self.interval = Some(send_interval(TICK_STEP, ctx.clone(), Pulse)); + Cmd::Log => Reply::Log(self.state.work.clone()), + Cmd::Watch(target) => Reply::Monitor(ctx.monitor(&target)), + Cmd::Link(target) => { + ctx.link(&target); + Reply::Unit } - if self.state.timers_done() { + Cmd::Trap => { + ctx.trap_exit(true); + Reply::Unit + } + Cmd::Boom => panic!("cross-backend injected panic"), + Cmd::Stop => { ctx.stop(); + Reply::Unit + } + Cmd::Timers { after, pulses } => { + self.state.ticks_wanted = after; + self.state.pulses_wanted = pulses; + for n in 0..after { + send_after(TICK_STEP * (n + 1), ctx.clone(), Tick(n)); + } + if pulses > 0 { + self.interval = Some(send_interval(TICK_STEP, ctx.clone(), Pulse)); + } + if self.state.timers_done() { + ctx.stop(); + } + Reply::Unit } - Reply::Unit } } - } .await; let _ = reply.reply(__reply_value); } } impl Handler for Node { - async fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.state.log.push(Obs::Down { - observer: self.state.index, - monitor: down.monitor_ref, - reason: down.reason, - }); - } + self.state.log.push(Obs::Down { + observer: self.state.index, + monitor: down.monitor_ref, + reason: down.reason, + }); + } let _ = reply.reply(()); } } impl Handler for Node { - async fn handle(&mut self, Tick(n): Tick, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + Tick(n): Tick, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - self.state.ticks_seen += 1; - self.state.log.push(Obs::Tick { - node: self.state.index, - n, - }); - if self.state.timers_done() { - ctx.stop(); + self.state.ticks_seen += 1; + self.state.log.push(Obs::Tick { + node: self.state.index, + n, + }); + if self.state.timers_done() { + ctx.stop(); + } } - } let _ = reply.reply(()); } } impl Handler for Node { - async fn handle(&mut self, _: Pulse, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + _: Pulse, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { let __reply_value: () = async { - if self.state.pulses_seen == self.state.pulses_wanted { - return; // A tick already in flight when the interval was cancelled. - } - self.state.pulses_seen += 1; - self.state.log.push(Obs::Pulse { - node: self.state.index, - }); - if self.state.pulses_seen == self.state.pulses_wanted { - if let Some(timer) = self.interval.take() { - timer.cancellation_token.cancel(); + if self.state.pulses_seen == self.state.pulses_wanted { + return; // A tick already in flight when the interval was cancelled. + } + self.state.pulses_seen += 1; + self.state.log.push(Obs::Pulse { + node: self.state.index, + }); + if self.state.pulses_seen == self.state.pulses_wanted { + if let Some(timer) = self.interval.take() { + timer.cancellation_token.cancel(); + } + } + if self.state.timers_done() { + ctx.stop(); } } - if self.state.timers_done() { - ctx.stop(); - } - } .await; let _ = reply.reply(__reply_value); } @@ -295,117 +314,137 @@ mod threads_node { } } impl Handler for Node { - fn handle(&mut self, cmd: Cmd, reply: spawned_concurrency::reply::PendingReply<'_, Reply>, ctx: &Context) { + fn handle( + &mut self, + cmd: Cmd, + reply: spawned_concurrency::reply::PendingReply<'_, Reply>, + ctx: &Context, + ) { let __reply_value: Reply = (|| { - match cmd { - Cmd::Echo(v) => self.state.echo(v), - Cmd::Chain(v) => match self.peers.first() { - None => self.state.leaf(v), - Some(next) => next - .request_with_timeout(Cmd::Chain(chain_arg(v, self.state.index)), LIMIT) - .unwrap_or(Reply::Failed), - }, - Cmd::FanOut(v) => { - // Fan out every request before collecting any reply. - let pending: Vec<_> = self - .peers - .iter() - .enumerate() - .map(|(i, peer)| peer.request_raw(Cmd::Echo(v + i as u64))) - .collect(); - let mut values = Vec::new(); - for rx in pending { - match rx.map(|rx| rx.recv_timeout(LIMIT)) { - Ok(Ok(Reply::Value(v))) => values.push(v), - _ => return Reply::Failed, + match cmd { + Cmd::Echo(v) => self.state.echo(v), + Cmd::Chain(v) => match self.peers.first() { + None => self.state.leaf(v), + Some(next) => next + .request_with_timeout(Cmd::Chain(chain_arg(v, self.state.index)), LIMIT) + .unwrap_or(Reply::Failed), + }, + Cmd::FanOut(v) => { + // Fan out every request before collecting any reply. + let pending: Vec<_> = self + .peers + .iter() + .enumerate() + .map(|(i, peer)| peer.request_raw(Cmd::Echo(v + i as u64))) + .collect(); + let mut values = Vec::new(); + for rx in pending { + match rx.map(|rx| rx.recv_timeout(LIMIT)) { + Ok(Ok(Reply::Value(v))) => values.push(v), + _ => return Reply::Failed, + } } + Reply::Values(values) } - Reply::Values(values) - } - Cmd::Work { sender, seq } => { - self.state.work.push((sender, seq)); - Reply::Unit - } - Cmd::Log => Reply::Log(self.state.work.clone()), - Cmd::Watch(target) => Reply::Monitor(ctx.monitor(&target)), - Cmd::Link(target) => { - ctx.link(&target); - Reply::Unit - } - Cmd::Trap => { - ctx.trap_exit(true); - Reply::Unit - } - Cmd::Boom => panic!("cross-backend injected panic"), - Cmd::Stop => { - ctx.stop(); - Reply::Unit - } - Cmd::Timers { after, pulses } => { - self.state.ticks_wanted = after; - self.state.pulses_wanted = pulses; - for n in 0..after { - send_after(TICK_STEP * (n + 1), ctx.clone(), Tick(n)); + Cmd::Work { sender, seq } => { + self.state.work.push((sender, seq)); + Reply::Unit + } + Cmd::Log => Reply::Log(self.state.work.clone()), + Cmd::Watch(target) => Reply::Monitor(ctx.monitor(&target)), + Cmd::Link(target) => { + ctx.link(&target); + Reply::Unit } - if pulses > 0 { - self.interval = Some(send_interval(TICK_STEP, ctx.clone(), Pulse)); + Cmd::Trap => { + ctx.trap_exit(true); + Reply::Unit } - if self.state.timers_done() { + Cmd::Boom => panic!("cross-backend injected panic"), + Cmd::Stop => { ctx.stop(); + Reply::Unit + } + Cmd::Timers { after, pulses } => { + self.state.ticks_wanted = after; + self.state.pulses_wanted = pulses; + for n in 0..after { + send_after(TICK_STEP * (n + 1), ctx.clone(), Tick(n)); + } + if pulses > 0 { + self.interval = Some(send_interval(TICK_STEP, ctx.clone(), Pulse)); + } + if self.state.timers_done() { + ctx.stop(); + } + Reply::Unit } - Reply::Unit } - } - })(); + })(); let _ = reply.reply(__reply_value); } } impl Handler for Node { - fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.state.log.push(Obs::Down { - observer: self.state.index, - monitor: down.monitor_ref, - reason: down.reason, - }); - } + self.state.log.push(Obs::Down { + observer: self.state.index, + monitor: down.monitor_ref, + reason: down.reason, + }); + } let _ = reply.reply(()); } } impl Handler for Node { - fn handle(&mut self, Tick(n): Tick, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + fn handle( + &mut self, + Tick(n): Tick, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - self.state.ticks_seen += 1; - self.state.log.push(Obs::Tick { - node: self.state.index, - n, - }); - if self.state.timers_done() { - ctx.stop(); + self.state.ticks_seen += 1; + self.state.log.push(Obs::Tick { + node: self.state.index, + n, + }); + if self.state.timers_done() { + ctx.stop(); + } } - } let _ = reply.reply(()); } } impl Handler for Node { - fn handle(&mut self, _: Pulse, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + fn handle( + &mut self, + _: Pulse, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { let __reply_value: () = (|| { - if self.state.pulses_seen == self.state.pulses_wanted { - return; - } - self.state.pulses_seen += 1; - self.state.log.push(Obs::Pulse { - node: self.state.index, - }); - if self.state.pulses_seen == self.state.pulses_wanted { - if let Some(timer) = self.interval.take() { - timer.cancellation_token.cancel(); + if self.state.pulses_seen == self.state.pulses_wanted { + return; } - } - if self.state.timers_done() { - ctx.stop(); - } - })(); + self.state.pulses_seen += 1; + self.state.log.push(Obs::Pulse { + node: self.state.index, + }); + if self.state.pulses_seen == self.state.pulses_wanted { + if let Some(timer) = self.interval.take() { + timer.cancellation_token.cancel(); + } + } + if self.state.timers_done() { + ctx.stop(); + } + })(); let _ = reply.reply(__reply_value); } } @@ -1307,12 +1346,17 @@ mod early_stop { } } impl spawned_concurrency::tasks::Handler for Hooks { - async fn handle(&mut self, block: Block, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &spawned_concurrency::tasks::Context) { + async fn handle( + &mut self, + block: Block, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &spawned_concurrency::tasks::Context, + ) { { - // Deliberately blocks the only async worker of the group. - let _ = block.1.send(()); - let _ = block.0.lock().unwrap().recv_timeout(LIMIT); - } + // Deliberately blocks the only async worker of the group. + let _ = block.1.send(()); + let _ = block.0.lock().unwrap().recv_timeout(LIMIT); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/dedicated_runtime.rs b/concurrency/tests/dedicated_runtime.rs index 64dc2c3d..3b9b958f 100644 --- a/concurrency/tests/dedicated_runtime.rs +++ b/concurrency/tests/dedicated_runtime.rs @@ -92,53 +92,58 @@ impl Message for Inspect { type Result = Observation; } impl Handler for Probe { - async fn handle(&mut self, operation: Inspect, reply: spawned_concurrency::reply::PendingReply<'_, Observation>, _: &Context) { + async fn handle( + &mut self, + operation: Inspect, + reply: spawned_concurrency::reply::PendingReply<'_, Observation>, + _: &Context, + ) { let __reply_value: Observation = { - let root = self.assert_root(); - let handle = tokio::runtime::Handle::current(); - let workers = handle.metrics().num_workers(); - let flavor = handle.runtime_flavor(); - let worker = match operation { - Inspect::SpawnBlocking => tokio::time::timeout( - LIMIT, - tokio::task::spawn_blocking(|| thread::current().id()), - ) - .await - .unwrap() - .unwrap(), - Inspect::Yield => { - tokio::task::yield_now().await; - tokio::time::timeout(LIMIT, tokio::spawn(timed_helper())) - .await - .unwrap() - .unwrap() - } - Inspect::BlockingReceive | Inspect::BlockInPlace => { - let (tx, rx) = mpsc::channel(); - let helper = tokio::spawn(async move { - let worker = timed_helper().await; - tx.send(worker).unwrap(); - }); - // The dedicated root synchronously waits for an async helper. - // A current-thread runtime cannot make progress here, but the - // bounded receive still lets that regression terminate. - let worker = if matches!(operation, Inspect::BlockInPlace) { - tokio::task::block_in_place(|| rx.recv_timeout(LIMIT).unwrap()) - } else { - rx.recv_timeout(LIMIT).unwrap() - }; - tokio::time::timeout(LIMIT, helper).await.unwrap().unwrap(); - worker + let root = self.assert_root(); + let handle = tokio::runtime::Handle::current(); + let workers = handle.metrics().num_workers(); + let flavor = handle.runtime_flavor(); + let worker = match operation { + Inspect::SpawnBlocking => tokio::time::timeout( + LIMIT, + tokio::task::spawn_blocking(|| thread::current().id()), + ) + .await + .unwrap() + .unwrap(), + Inspect::Yield => { + tokio::task::yield_now().await; + tokio::time::timeout(LIMIT, tokio::spawn(timed_helper())) + .await + .unwrap() + .unwrap() + } + Inspect::BlockingReceive | Inspect::BlockInPlace => { + let (tx, rx) = mpsc::channel(); + let helper = tokio::spawn(async move { + let worker = timed_helper().await; + tx.send(worker).unwrap(); + }); + // The dedicated root synchronously waits for an async helper. + // A current-thread runtime cannot make progress here, but the + // bounded receive still lets that regression terminate. + let worker = if matches!(operation, Inspect::BlockInPlace) { + tokio::task::block_in_place(|| rx.recv_timeout(LIMIT).unwrap()) + } else { + rx.recv_timeout(LIMIT).unwrap() + }; + tokio::time::timeout(LIMIT, helper).await.unwrap().unwrap(); + worker + } + }; + self.assert_root(); + Observation { + root, + worker, + workers, + flavor, } }; - self.assert_root(); - Observation { - root, - worker, - workers, - flavor, - } - }; let _ = reply.reply(__reply_value); } } @@ -150,11 +155,16 @@ impl Message for Ping { struct Child; impl Actor for Child {} impl Handler for Child { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - tokio::time::sleep(Duration::from_millis(1)).await; - 42 - }; + tokio::time::sleep(Duration::from_millis(1)).await; + 42 + }; let _ = reply.reply(__reply_value); } } @@ -163,18 +173,23 @@ impl Message for Nested { type Result = u64; } impl Handler for Probe { - async fn handle(&mut self, _: Nested, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + async fn handle( + &mut self, + _: Nested, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - self.assert_root(); - // Default Async startup uses this actor's isolated runtime. - let child = Child.start(); - let _cleanup = StopOnDrop(child.child_handle()); - let answer = child.request_with_timeout(Ping, LIMIT).await.unwrap(); - child.child_handle().stop(); - tokio::time::timeout(LIMIT, child.join()).await.unwrap(); - self.assert_root(); - answer - }; + self.assert_root(); + // Default Async startup uses this actor's isolated runtime. + let child = Child.start(); + let _cleanup = StopOnDrop(child.child_handle()); + let answer = child.request_with_timeout(Ping, LIMIT).await.unwrap(); + child.child_handle().stop(); + tokio::time::timeout(LIMIT, child.join()).await.unwrap(); + self.assert_root(); + answer + }; let _ = reply.reply(__reply_value); } } @@ -184,22 +199,27 @@ impl Message for Exchange { type Result = u64; } impl Handler for Probe { - async fn handle(&mut self, message: Exchange, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + async fn handle( + &mut self, + message: Exchange, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - self.assert_root(); - let answer = tokio::time::timeout(LIMIT, async { - let mut stream = tokio::net::TcpStream::connect(message.0).await.unwrap(); - stream.write_all(&42u64.to_le_bytes()).await.unwrap(); - let mut bytes = [0; 8]; - stream.read_exact(&mut bytes).await.unwrap(); - u64::from_le_bytes(bytes) - }) - .await - .expect("dedicated runtime I/O stalled"); - self.assert_root(); - answer - }; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + self.assert_root(); + let answer = tokio::time::timeout(LIMIT, async { + let mut stream = tokio::net::TcpStream::connect(message.0).await.unwrap(); + stream.write_all(&42u64.to_le_bytes()).await.unwrap(); + let mut bytes = [0; 8]; + stream.read_exact(&mut bytes).await.unwrap(); + u64::from_le_bytes(bytes) + }) + .await + .expect("dedicated runtime I/O stalled"); + self.assert_root(); + answer + }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/execution_groups.rs b/concurrency/tests/execution_groups.rs index 01c55ddc..1502aebd 100644 --- a/concurrency/tests/execution_groups.rs +++ b/concurrency/tests/execution_groups.rs @@ -23,18 +23,28 @@ impl Message for Ping { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, String>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, String>, + _: &Context, + ) { let __reply_value: String = { - std::thread::current() - .name() - .unwrap_or_default() - .to_string() - }; + std::thread::current() + .name() + .unwrap_or_default() + .to_string() + }; let _ = reply.reply(__reply_value); } } impl Handler for Worker { - async fn handle(&mut self, _: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } @@ -343,11 +353,16 @@ impl Actor for FailedStartup { } } impl Handler for FailedStartup { - async fn handle(&mut self, msg: Queued, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: Queued, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.handled.fetch_add(1, Ordering::SeqCst); - drop(msg.0); - } + self.handled.fetch_add(1, Ordering::SeqCst); + drop(msg.0); + } let _ = reply.reply(()); } } @@ -570,10 +585,13 @@ impl Message for CheckAdmission { struct HelperAware; impl Actor for HelperAware {} impl Handler for HelperAware { - async fn handle(&mut self, _: CheckAdmission, reply: spawned_concurrency::reply::PendingReply<'_, bool>, ctx: &Context) { - let __reply_value: bool = { - ctx.spawn_helper(async {}).is_err() - }; + async fn handle( + &mut self, + _: CheckAdmission, + reply: spawned_concurrency::reply::PendingReply<'_, bool>, + ctx: &Context, + ) { + let __reply_value: bool = { ctx.spawn_helper(async {}).is_err() }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/execution_profiles.rs b/concurrency/tests/execution_profiles.rs index d08ef09b..93db6c79 100644 --- a/concurrency/tests/execution_profiles.rs +++ b/concurrency/tests/execution_profiles.rs @@ -22,14 +22,19 @@ impl Message for Audit { type Result = (String, Option); } impl Handler for Probe { - async fn handle(&mut self, _: Audit, reply: spawned_concurrency::reply::PendingReply<'_, ::Result>, _: &Context) { + async fn handle( + &mut self, + _: Audit, + reply: spawned_concurrency::reply::PendingReply<'_, ::Result>, + _: &Context, + ) { let __reply_value: ::Result = { - spawned_rt::tasks::sleep(Duration::from_millis(1)).await; - ( - std::thread::current().name().unwrap_or_default().into(), - spawned_rt::tasks::affinity::current_cpu().ok(), - ) - }; + spawned_rt::tasks::sleep(Duration::from_millis(1)).await; + ( + std::thread::current().name().unwrap_or_default().into(), + spawned_rt::tasks::affinity::current_cpu().ok(), + ) + }; let _ = reply.reply(__reply_value); } } @@ -38,10 +43,14 @@ impl Message for StartPeer { type Result = Result, SystemError>; } impl Handler for Probe { - async fn handle(&mut self, _: StartPeer, reply: spawned_concurrency::reply::PendingReply<'_, ::Result>, ctx: &Context) { - let __reply_value: ::Result = { - ctx.execution_group().unwrap().start(Probe) - }; + async fn handle( + &mut self, + _: StartPeer, + reply: spawned_concurrency::reply::PendingReply<'_, ::Result>, + ctx: &Context, + ) { + let __reply_value: ::Result = + { ctx.execution_group().unwrap().start(Probe) }; let _ = reply.reply(__reply_value); } } @@ -85,13 +94,18 @@ impl Message for Block { type Result = (); } impl Handler for Probe { - async fn handle(&mut self, msg: Block, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: Block, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - tokio::task::block_in_place(|| { - let _ = msg.entered.send(()); - msg.release.recv_timeout(LIMIT).unwrap(); - }); - } + tokio::task::block_in_place(|| { + let _ = msg.entered.send(()); + msg.release.recv_timeout(LIMIT).unwrap(); + }); + } let _ = reply.reply(()); } } @@ -100,10 +114,15 @@ impl Message for Crash { type Result = (); } impl Handler for Probe { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("profile recovery injection"); - } + panic!("profile recovery injection"); + } let _ = reply.reply(()); } } @@ -352,10 +371,15 @@ mod tls_join { type Result = (); } impl Handler for Probe { - async fn handle(&mut self, msg: Install, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: Install, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - PROBE.with(|p| *p.borrow_mut() = Some(msg.0)); - } + PROBE.with(|p| *p.borrow_mut() = Some(msg.0)); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/failpoints.rs b/concurrency/tests/failpoints.rs index f86cda9c..38e47f59 100644 --- a/concurrency/tests/failpoints.rs +++ b/concurrency/tests/failpoints.rs @@ -93,15 +93,25 @@ impl Actor for Worker { } } impl Handler for Worker { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("injected child failure"); - } + panic!("injected child failure"); + } let _ = reply.reply(()); } } impl Handler for Worker { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } @@ -653,31 +663,51 @@ struct Linker { } impl Actor for Linker {} impl Handler for Linker { - async fn handle(&mut self, msg: LinkTo, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + msg: LinkTo, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.link(&msg.0); - } + ctx.link(&msg.0); + } let _ = reply.reply(()); } } impl Handler for Linker { - async fn handle(&mut self, msg: Watch, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + msg: Watch, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.monitor(&msg.0); - } + ctx.monitor(&msg.0); + } let _ = reply.reply(()); } } impl Handler for Linker { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } impl Handler for Linker { - async fn handle(&mut self, msg: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - let _ = self.downs.send(msg.reason); - } + let _ = self.downs.send(msg.reason); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/foreign_panics.rs b/concurrency/tests/foreign_panics.rs index 74e26dd5..8e26c08a 100644 --- a/concurrency/tests/foreign_panics.rs +++ b/concurrency/tests/foreign_panics.rs @@ -182,35 +182,53 @@ mod task_actors { } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0.on_crash(); - } + self.0.on_crash(); + } let _ = reply.reply(()); } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - 7 - }; + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { 7 }; let _ = reply.reply(__reply_value); } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Gift, reply: spawned_concurrency::reply::PendingReply<'_, Bomb>, _: &Context) { + async fn handle( + &mut self, + _: Gift, + reply: spawned_concurrency::reply::PendingReply<'_, Bomb>, + _: &Context, + ) { let __reply_value: Bomb = { - self.0.gate.wait(); - Bomb(Some(REPLY_PANIC)) - }; + self.0.gate.wait(); + Bomb(Some(REPLY_PANIC)) + }; let _ = reply.reply(__reply_value); } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Stash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Stash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0.stash.push(Bomb(Some(REPLY_PANIC))); - } + self.0.stash.push(Bomb(Some(REPLY_PANIC))); + } let _ = reply.reply(()); } } @@ -229,35 +247,53 @@ mod thread_actors { } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0.on_crash(); - } + self.0.on_crash(); + } let _ = reply.reply(()); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - 7 - }; + fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { 7 }; let _ = reply.reply(__reply_value); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Gift, reply: spawned_concurrency::reply::PendingReply<'_, Bomb>, _: &Context) { + fn handle( + &mut self, + _: Gift, + reply: spawned_concurrency::reply::PendingReply<'_, Bomb>, + _: &Context, + ) { let __reply_value: Bomb = { - self.0.gate.wait(); - Bomb(Some(REPLY_PANIC)) - }; + self.0.gate.wait(); + Bomb(Some(REPLY_PANIC)) + }; let _ = reply.reply(__reply_value); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Stash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + _: Stash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0.stash.push(Bomb(Some(REPLY_PANIC))); - } + self.0.stash.push(Bomb(Some(REPLY_PANIC))); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/full_actor_shards.rs b/concurrency/tests/full_actor_shards.rs index 09bbe042..e059d556 100644 --- a/concurrency/tests/full_actor_shards.rs +++ b/concurrency/tests/full_actor_shards.rs @@ -33,13 +33,18 @@ impl Message for Thread { type Result = String; } impl Handler for Worker { - async fn handle(&mut self, _: Thread, reply: spawned_concurrency::reply::PendingReply<'_, String>, _: &Context) { + async fn handle( + &mut self, + _: Thread, + reply: spawned_concurrency::reply::PendingReply<'_, String>, + _: &Context, + ) { let __reply_value: String = { - std::thread::current() - .name() - .unwrap_or("unnamed") - .to_owned() - }; + std::thread::current() + .name() + .unwrap_or("unnamed") + .to_owned() + }; let _ = reply.reply(__reply_value); } } @@ -51,12 +56,17 @@ impl Message for Hold { type Result = u64; } impl Handler for Worker { - async fn handle(&mut self, msg: Hold, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + async fn handle( + &mut self, + msg: Hold, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - let _ = msg.started.send(()); - msg.release.await.unwrap(); - 42 - }; + let _ = msg.started.send(()); + msg.release.await.unwrap(); + 42 + }; let _ = reply.reply(__reply_value); } } @@ -65,26 +75,31 @@ impl Message for Nested { type Result = u64; } impl Handler for Worker { - async fn handle(&mut self, msg: Nested, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + async fn handle( + &mut self, + msg: Nested, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - let child = Worker(Default::default()).start(); - let _ = child.request(Thread).await.unwrap(); - child.child_handle().stop(); - child.join().await; - let helper = tokio::spawn(async { - tokio::time::sleep(Duration::from_millis(1)).await; - 42u64 - }); - let mut socket = tokio::net::TcpStream::connect(msg.0).await.unwrap(); - socket - .write_all(&helper.await.unwrap().to_le_bytes()) - .await - .unwrap(); - let mut reply = [0; 8]; - socket.read_exact(&mut reply).await.unwrap(); - u64::from_le_bytes(reply) - }; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let child = Worker(Default::default()).start(); + let _ = child.request(Thread).await.unwrap(); + child.child_handle().stop(); + child.join().await; + let helper = tokio::spawn(async { + tokio::time::sleep(Duration::from_millis(1)).await; + 42u64 + }); + let mut socket = tokio::net::TcpStream::connect(msg.0).await.unwrap(); + socket + .write_all(&helper.await.unwrap().to_le_bytes()) + .await + .unwrap(); + let mut reply = [0; 8]; + socket.read_exact(&mut reply).await.unwrap(); + u64::from_le_bytes(reply) + }; let _ = reply.reply(__reply_value); } } @@ -248,11 +263,16 @@ impl Message for Freeze { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, msg: Freeze, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: Freeze, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - let _ = msg.started.send(()); - msg.release.recv_timeout(LIMIT).unwrap(); - } + let _ = msg.started.send(()); + msg.release.recv_timeout(LIMIT).unwrap(); + } let _ = reply.reply(()); } } @@ -327,11 +347,16 @@ impl Message for Tick { type Result = u64; } impl Handler for Worker { - async fn handle(&mut self, _: Tick, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + async fn handle( + &mut self, + _: Tick, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - tokio::time::sleep(Duration::from_millis(1)).await; - 42 - }; + tokio::time::sleep(Duration::from_millis(1)).await; + 42 + }; let _ = reply.reply(__reply_value); } } @@ -340,10 +365,19 @@ impl Message for SpawnBlockingChild { type Result = spawned_concurrency::tasks::ActorRef; } impl Handler for Worker { - async fn handle(&mut self, _: SpawnBlockingChild, reply: spawned_concurrency::reply::PendingReply<'_, spawned_concurrency::tasks::ActorRef>, _: &Context) { + async fn handle( + &mut self, + _: SpawnBlockingChild, + reply: spawned_concurrency::reply::PendingReply< + '_, + spawned_concurrency::tasks::ActorRef, + >, + _: &Context, + ) { let __reply_value: spawned_concurrency::tasks::ActorRef = { - Worker(Default::default()).start_with_backend(spawned_concurrency::tasks::Backend::Blocking) - }; + Worker(Default::default()) + .start_with_backend(spawned_concurrency::tasks::Backend::Blocking) + }; let _ = reply.reply(__reply_value); } } @@ -352,14 +386,19 @@ impl Message for Helper { type Result = tokio::task::JoinHandle; } impl Handler for Worker { - async fn handle(&mut self, msg: Helper, reply: spawned_concurrency::reply::PendingReply<'_, tokio::task::JoinHandle>, _: &Context) { + async fn handle( + &mut self, + msg: Helper, + reply: spawned_concurrency::reply::PendingReply<'_, tokio::task::JoinHandle>, + _: &Context, + ) { let __reply_value: tokio::task::JoinHandle = { - tokio::spawn(async move { - msg.0.await.unwrap(); - tokio::time::sleep(Duration::from_millis(1)).await; - 42 - }) - }; + tokio::spawn(async move { + msg.0.await.unwrap(); + tokio::time::sleep(Duration::from_millis(1)).await; + 42 + }) + }; let _ = reply.reply(__reply_value); } } @@ -399,18 +438,28 @@ impl Message for InstallMonitor { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, msg: InstallMonitor, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: InstallMonitor, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - msg.observer.monitor(&msg.target); - } + msg.observer.monitor(&msg.target); + } let _ = reply.reply(()); } } impl Handler for Worker { - async fn handle(&mut self, _: spawned_concurrency::Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: spawned_concurrency::Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0.lock().unwrap().push("down"); - } + self.0.lock().unwrap().push("down"); + } let _ = reply.reply(()); } } @@ -419,10 +468,13 @@ impl Message for Events { type Result = Vec<&'static str>; } impl Handler for Worker { - async fn handle(&mut self, _: Events, reply: spawned_concurrency::reply::PendingReply<'_, Vec<&'static str>>, _: &Context) { - let __reply_value: Vec<&'static str> = { - self.0.lock().unwrap().clone() - }; + async fn handle( + &mut self, + _: Events, + reply: spawned_concurrency::reply::PendingReply<'_, Vec<&'static str>>, + _: &Context, + ) { + let __reply_value: Vec<&'static str> = { self.0.lock().unwrap().clone() }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/future_layout.rs b/concurrency/tests/future_layout.rs index fc92932a..eb5aa363 100644 --- a/concurrency/tests/future_layout.rs +++ b/concurrency/tests/future_layout.rs @@ -23,15 +23,23 @@ trait ProbeProtocol: Send + Sync { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: probe_protocol::Notify, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: probe_protocol::Notify, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } impl Handler for Worker { - async fn handle(&mut self, query: probe_protocol::Query, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - query.value - }; + async fn handle( + &mut self, + query: probe_protocol::Query, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { query.value }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/kill_links.rs b/concurrency/tests/kill_links.rs index ebcc4cc1..b29394df 100644 --- a/concurrency/tests/kill_links.rs +++ b/concurrency/tests/kill_links.rs @@ -22,10 +22,13 @@ fn native_propagated_kill_is_trappable_but_direct_kill_is_not() { } } impl Handler for Node { - fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, u8>, _: &Context) { - let __reply_value: u8 = { - 7 - }; + fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, u8>, + _: &Context, + ) { + let __reply_value: u8 = { 7 }; let _ = reply.reply(__reply_value); } } @@ -81,10 +84,13 @@ async fn async_propagated_kill_is_trappable_but_direct_kill_is_not() { } } impl Handler for Node { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, u8>, _: &Context) { - let __reply_value: u8 = { - 7 - }; + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, u8>, + _: &Context, + ) { + let __reply_value: u8 = { 7 }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/leaks.rs b/concurrency/tests/leaks.rs index b44f6396..736d2b42 100644 --- a/concurrency/tests/leaks.rs +++ b/concurrency/tests/leaks.rs @@ -64,7 +64,12 @@ impl threads::Actor for NativeProbe { } } impl threads::Handler for NativeProbe { - fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context) { + fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &threads::Context, + ) { let _ = reply.reply(()); } } @@ -104,18 +109,28 @@ struct NativeB { impl threads::Actor for NativeA {} impl threads::Actor for NativeB {} impl threads::Handler>> for NativeA { - fn handle(&mut self, msg: SetPeer>, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context) { + fn handle( + &mut self, + msg: SetPeer>, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &threads::Context, + ) { { - self.peer = Some(msg.0); - } + self.peer = Some(msg.0); + } let _ = reply.reply(()); } } impl threads::Handler>> for NativeB { - fn handle(&mut self, msg: SetPeer>, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context) { + fn handle( + &mut self, + msg: SetPeer>, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &threads::Context, + ) { { - self.peer = Some(msg.0); - } + self.peer = Some(msg.0); + } let _ = reply.reply(()); } } @@ -207,7 +222,12 @@ mod async_leaks { } } impl Handler for Probe { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } @@ -216,15 +236,25 @@ mod async_leaks { type Result = (); } impl Handler for Probe { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("injected failure for leak test"); - } + panic!("injected failure for leak test"); + } let _ = reply.reply(()); } } impl Handler for Probe { - async fn handle(&mut self, _: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } @@ -314,30 +344,50 @@ mod async_leaks { impl Actor for AsyncA {} impl Actor for AsyncB {} impl Handler for AsyncA { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } impl Handler for AsyncB { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } impl Handler, tasks::Recipient)>> for AsyncA { - async fn handle(&mut self, msg: SetPeer<(ActorRef, tasks::Recipient)>, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: SetPeer<(ActorRef, tasks::Recipient)>, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.peer = Some(msg.0 .0); - self.recipient = Some(msg.0 .1); - } + self.peer = Some(msg.0 .0); + self.recipient = Some(msg.0 .1); + } let _ = reply.reply(()); } } impl Handler, tasks::Recipient)>> for AsyncB { - async fn handle(&mut self, msg: SetPeer<(ActorRef, tasks::Recipient)>, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: SetPeer<(ActorRef, tasks::Recipient)>, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.peer = Some(msg.0 .0); - self.recipient = Some(msg.0 .1); - } + self.peer = Some(msg.0 .0); + self.recipient = Some(msg.0 .1); + } let _ = reply.reply(()); } } @@ -586,12 +636,22 @@ mod sim_leaks { } impl Actor for Probe {} impl Handler for Probe { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } impl Handler for Probe { - async fn handle(&mut self, _: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } @@ -609,28 +669,48 @@ mod sim_leaks { impl Actor for A {} impl Actor for B {} impl Handler for A { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } impl Handler for B { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } impl Handler, tasks::Recipient)>> for A { - async fn handle(&mut self, msg: SetPeer<(tasks::ActorRef, tasks::Recipient)>, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: SetPeer<(tasks::ActorRef, tasks::Recipient)>, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.peer = Some(msg.0); - } + self.peer = Some(msg.0); + } let _ = reply.reply(()); } } impl Handler, tasks::Recipient)>> for B { - async fn handle(&mut self, msg: SetPeer<(tasks::ActorRef, tasks::Recipient)>, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: SetPeer<(tasks::ActorRef, tasks::Recipient)>, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.peer = Some(msg.0); - } + self.peer = Some(msg.0); + } let _ = reply.reply(()); } } @@ -762,10 +842,15 @@ mod sim_leaks { type Result = (); } impl Handler for Probe { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("injected failure for leak test"); - } + panic!("injected failure for leak test"); + } let _ = reply.reply(()); } } @@ -873,10 +958,15 @@ mod sim_leaks { } } impl Handler for Named { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("injected failure for leak test"); - } + panic!("injected failure for leak test"); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/mailbox_drop_panic.rs b/concurrency/tests/mailbox_drop_panic.rs index 939c5f79..017723e9 100644 --- a/concurrency/tests/mailbox_drop_panic.rs +++ b/concurrency/tests/mailbox_drop_panic.rs @@ -42,20 +42,30 @@ impl Drop for Victim { } impl Actor for Victim {} impl Handler for Victim { - async fn handle(&mut self, msg: Hold, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: Hold, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - msg.0.send(()).unwrap(); - let _ = msg.1.await; - assert!(!msg.2, "original handler failure"); - } + msg.0.send(()).unwrap(); + let _ = msg.1.await; + assert!(!msg.2, "original handler failure"); + } let _ = reply.reply(()); } } impl Handler for Victim { - async fn handle(&mut self, _: Bomb, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Bomb, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0.fetch_add(1, Ordering::SeqCst); - } + self.0.fetch_add(1, Ordering::SeqCst); + } let _ = reply.reply(()); } } @@ -70,15 +80,25 @@ impl Actor for Observer { } } impl Handler for Observer { - async fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0.send((false, down.reason)).unwrap(); - } + self.0.send((false, down.reason)).unwrap(); + } let _ = reply.reply(()); } } impl Handler for Observer { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } diff --git a/concurrency/tests/metrics.rs b/concurrency/tests/metrics.rs index b3383304..67c71b9b 100644 --- a/concurrency/tests/metrics.rs +++ b/concurrency/tests/metrics.rs @@ -39,32 +39,50 @@ impl Actor for Probe { } } impl Handler for Probe { - async fn handle(&mut self, _: metrics_protocol::Query, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - 9 - }; + async fn handle( + &mut self, + _: metrics_protocol::Query, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { 9 }; let _ = reply.reply(__reply_value); } } impl Handler for Probe { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } impl Handler for Probe { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("metric panic regression"); - } + panic!("metric panic regression"); + } let _ = reply.reply(()); } } impl Handler for Probe { - async fn handle(&mut self, hold: Hold, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + hold: Hold, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - let _ = hold.entered.send(()); - let _ = hold.release.await; - } + let _ = hold.entered.send(()); + let _ = hold.release.await; + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/metrics_composable.rs b/concurrency/tests/metrics_composable.rs index fcf98653..daf222ea 100644 --- a/concurrency/tests/metrics_composable.rs +++ b/concurrency/tests/metrics_composable.rs @@ -13,7 +13,12 @@ impl Message for Ping { } impl Actor for Probe {} impl Handler for Probe { - fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } diff --git a/concurrency/tests/metrics_counts_only.rs b/concurrency/tests/metrics_counts_only.rs index 00575813..4be78f6d 100644 --- a/concurrency/tests/metrics_counts_only.rs +++ b/concurrency/tests/metrics_counts_only.rs @@ -11,7 +11,12 @@ impl Message for Ping { } impl Actor for Probe {} impl Handler for Probe { - fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } diff --git a/concurrency/tests/metrics_sampling.rs b/concurrency/tests/metrics_sampling.rs index ba74acf8..5294ff2e 100644 --- a/concurrency/tests/metrics_sampling.rs +++ b/concurrency/tests/metrics_sampling.rs @@ -12,7 +12,12 @@ impl Message for Ping { } impl Actor for Probe {} impl Handler for Probe { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } diff --git a/concurrency/tests/native_dispatcher.rs b/concurrency/tests/native_dispatcher.rs index 8fbc4057..2e25b384 100644 --- a/concurrency/tests/native_dispatcher.rs +++ b/concurrency/tests/native_dispatcher.rs @@ -16,11 +16,16 @@ impl Message for Ping { struct Healthy; impl Actor for Healthy {} impl Handler for Healthy { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - tokio::task::yield_now().await; - 42 - }; + tokio::task::yield_now().await; + 42 + }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/native_mailbox_drop.rs b/concurrency/tests/native_mailbox_drop.rs index 25c7e2f2..8bec67b6 100644 --- a/concurrency/tests/native_mailbox_drop.rs +++ b/concurrency/tests/native_mailbox_drop.rs @@ -40,10 +40,15 @@ impl Drop for Victim { } } impl Handler for Victim { - fn handle(&mut self, _: Bomb, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + _: Bomb, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("queued handler ran"); - } + panic!("queued handler ran"); + } let _ = reply.reply(()); } } @@ -53,10 +58,15 @@ impl Message for Crash { type Result = (); } impl Handler for Victim { - fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("native handler failure"); - } + panic!("native handler failure"); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/nested_supervision.rs b/concurrency/tests/nested_supervision.rs index b56b1256..ffeb2c74 100644 --- a/concurrency/tests/nested_supervision.rs +++ b/concurrency/tests/nested_supervision.rs @@ -44,19 +44,29 @@ impl Message for Crash { struct Counter(usize); impl Actor for Counter {} impl Handler for Counter { - async fn handle(&mut self, _: Next, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { + async fn handle( + &mut self, + _: Next, + reply: spawned_concurrency::reply::PendingReply<'_, usize>, + _: &Context, + ) { let __reply_value: usize = { - self.0 += 1; - self.0 - }; + self.0 += 1; + self.0 + }; let _ = reply.reply(__reply_value); } } impl Handler for Counter { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("nested leaf failed"); - } + panic!("nested leaf failed"); + } let _ = reply.reply(()); } } @@ -523,10 +533,15 @@ impl Actor for MaybeStop { } } impl Handler for MaybeStop { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("nested leaf failed"); - } + panic!("nested leaf failed"); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/panic_safety.rs b/concurrency/tests/panic_safety.rs index bcbafdbf..4c951509 100644 --- a/concurrency/tests/panic_safety.rs +++ b/concurrency/tests/panic_safety.rs @@ -195,61 +195,92 @@ mod tasks_actors { } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Poke, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Poke, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0.fault(Fault::SendHandler, "injected handler panic"); - } + self.0.fault(Fault::SendHandler, "injected handler panic"); + } let _ = reply.reply(()); } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Ask, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - self.0.ask() - }; + async fn handle( + &mut self, + _: Ask, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { self.0.ask() }; let _ = reply.reply(__reply_value); } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Payload, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Payload, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Gift, reply: spawned_concurrency::reply::PendingReply<'_, Bomb>, _: &Context) { - let __reply_value: Bomb = { - self.0.gift() - }; + async fn handle( + &mut self, + _: Gift, + reply: spawned_concurrency::reply::PendingReply<'_, Bomb>, + _: &Context, + ) { + let __reply_value: Bomb = { self.0.gift() }; let _ = reply.reply(__reply_value); } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Hold, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Hold, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0.gate.wait(); - } + self.0.gate.wait(); + } let _ = reply.reply(()); } } impl Handler for TaskVictim { - async fn handle(&mut self, setup: Setup, reply: spawned_concurrency::reply::PendingReply<'_, Option>, ctx: &Context) { + async fn handle( + &mut self, + setup: Setup, + reply: spawned_concurrency::reply::PendingReply<'_, Option>, + ctx: &Context, + ) { let __reply_value: Option = async { - match setup { - Setup::Trap => ctx.trap_exit(true), - Setup::Link(target) => ctx.link(&target), - Setup::Watch(target) => return Some(ctx.monitor(&target)), + match setup { + Setup::Trap => ctx.trap_exit(true), + Setup::Link(target) => ctx.link(&target), + Setup::Watch(target) => return Some(ctx.monitor(&target)), + } + None } - None - } .await; let _ = reply.reply(__reply_value); } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0.fault(Fault::DownHandler, "injected Down panic"); - } + self.0.fault(Fault::DownHandler, "injected Down panic"); + } let _ = reply.reply(()); } } @@ -264,34 +295,47 @@ mod tasks_actors { } } impl Handler for TaskObserver { - async fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0 - .events - .push(Event::Down(down.monitor_ref, down.reason)); - } + self.0 + .events + .push(Event::Down(down.monitor_ref, down.reason)); + } let _ = reply.reply(()); } } impl Handler for TaskObserver { - async fn handle(&mut self, setup: Setup, reply: spawned_concurrency::reply::PendingReply<'_, Option>, ctx: &Context) { + async fn handle( + &mut self, + setup: Setup, + reply: spawned_concurrency::reply::PendingReply<'_, Option>, + ctx: &Context, + ) { let __reply_value: Option = async { - match setup { - Setup::Trap => ctx.trap_exit(true), - Setup::Link(target) => ctx.link(&target), - Setup::Watch(target) => return Some(ctx.monitor(&target)), + match setup { + Setup::Trap => ctx.trap_exit(true), + Setup::Link(target) => ctx.link(&target), + Setup::Watch(target) => return Some(ctx.monitor(&target)), + } + None } - None - } .await; let _ = reply.reply(__reply_value); } } impl Handler for TaskObserver { - async fn handle(&mut self, _: Ask, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - 7 - }; + async fn handle( + &mut self, + _: Ask, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { 7 }; let _ = reply.reply(__reply_value); } } @@ -317,60 +361,91 @@ mod thread_actors { } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Poke, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + _: Poke, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0.fault(Fault::SendHandler, "injected handler panic"); - } + self.0.fault(Fault::SendHandler, "injected handler panic"); + } let _ = reply.reply(()); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Ask, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - self.0.ask() - }; + fn handle( + &mut self, + _: Ask, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { self.0.ask() }; let _ = reply.reply(__reply_value); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Payload, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + _: Payload, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Gift, reply: spawned_concurrency::reply::PendingReply<'_, Bomb>, _: &Context) { - let __reply_value: Bomb = { - self.0.gift() - }; + fn handle( + &mut self, + _: Gift, + reply: spawned_concurrency::reply::PendingReply<'_, Bomb>, + _: &Context, + ) { + let __reply_value: Bomb = { self.0.gift() }; let _ = reply.reply(__reply_value); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Hold, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + _: Hold, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0.gate.wait(); - } + self.0.gate.wait(); + } let _ = reply.reply(()); } } impl Handler for ThreadVictim { - fn handle(&mut self, setup: Setup, reply: spawned_concurrency::reply::PendingReply<'_, Option>, ctx: &Context) { + fn handle( + &mut self, + setup: Setup, + reply: spawned_concurrency::reply::PendingReply<'_, Option>, + ctx: &Context, + ) { let __reply_value: Option = (|| { - match setup { - Setup::Trap => ctx.trap_exit(true), - Setup::Link(target) => ctx.link(&target), - Setup::Watch(target) => return Some(ctx.monitor(&target)), - } - None - })(); + match setup { + Setup::Trap => ctx.trap_exit(true), + Setup::Link(target) => ctx.link(&target), + Setup::Watch(target) => return Some(ctx.monitor(&target)), + } + None + })(); let _ = reply.reply(__reply_value); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + _: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0.fault(Fault::DownHandler, "injected Down panic"); - } + self.0.fault(Fault::DownHandler, "injected Down panic"); + } let _ = reply.reply(()); } } @@ -385,33 +460,46 @@ mod thread_actors { } } impl Handler for ThreadObserver { - fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0 - .events - .push(Event::Down(down.monitor_ref, down.reason)); - } + self.0 + .events + .push(Event::Down(down.monitor_ref, down.reason)); + } let _ = reply.reply(()); } } impl Handler for ThreadObserver { - fn handle(&mut self, setup: Setup, reply: spawned_concurrency::reply::PendingReply<'_, Option>, ctx: &Context) { + fn handle( + &mut self, + setup: Setup, + reply: spawned_concurrency::reply::PendingReply<'_, Option>, + ctx: &Context, + ) { let __reply_value: Option = (|| { - match setup { - Setup::Trap => ctx.trap_exit(true), - Setup::Link(target) => ctx.link(&target), - Setup::Watch(target) => return Some(ctx.monitor(&target)), - } - None - })(); + match setup { + Setup::Trap => ctx.trap_exit(true), + Setup::Link(target) => ctx.link(&target), + Setup::Watch(target) => return Some(ctx.monitor(&target)), + } + None + })(); let _ = reply.reply(__reply_value); } } impl Handler for ThreadObserver { - fn handle(&mut self, _: Ask, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - 7 - }; + fn handle( + &mut self, + _: Ask, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { 7 }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/process_failures.rs b/concurrency/tests/process_failures.rs index fcd68314..169ff7ed 100644 --- a/concurrency/tests/process_failures.rs +++ b/concurrency/tests/process_failures.rs @@ -30,20 +30,25 @@ impl Drop for Fixture { } } impl Handler for Fixture { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - std::fs::write( - self.path.join("effect"), - b"effect happened before process failure", - ) - .unwrap(); - if self.abort { - std::process::abort(); - } - loop { - std::thread::park(); + std::fs::write( + self.path.join("effect"), + b"effect happened before process failure", + ) + .unwrap(); + if self.abort { + std::process::abort(); + } + loop { + std::thread::park(); + } } - } let _ = reply.reply(()); } } diff --git a/concurrency/tests/protocol_timeouts.rs b/concurrency/tests/protocol_timeouts.rs index 68ab15b0..f8105ad3 100644 --- a/concurrency/tests/protocol_timeouts.rs +++ b/concurrency/tests/protocol_timeouts.rs @@ -58,29 +58,44 @@ mod asynchronous { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, msg: timed_protocol::Work, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + async fn handle( + &mut self, + msg: timed_protocol::Work, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; - msg.millis - }; + spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; + msg.millis + }; let _ = reply.reply(__reply_value); } } impl Handler for Worker { - async fn handle(&mut self, msg: timed_protocol::Brief, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + async fn handle( + &mut self, + msg: timed_protocol::Brief, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; - msg.millis - }; + spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; + msg.millis + }; let _ = reply.reply(__reply_value); } } impl Handler for Worker { - async fn handle(&mut self, msg: legacy_protocol::Legacy, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + async fn handle( + &mut self, + msg: legacy_protocol::Legacy, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; - msg.millis - }; + spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; + msg.millis + }; let _ = reply.reply(__reply_value); } } @@ -139,20 +154,30 @@ mod synchronous { struct Worker; impl Actor for Worker {} impl Handler for Worker { - fn handle(&mut self, msg: timed_protocol::Work, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + fn handle( + &mut self, + msg: timed_protocol::Work, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - std::thread::sleep(Duration::from_millis(msg.millis)); - msg.millis - }; + std::thread::sleep(Duration::from_millis(msg.millis)); + msg.millis + }; let _ = reply.reply(__reply_value); } } impl Handler for Worker { - fn handle(&mut self, msg: timed_protocol::Brief, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + fn handle( + &mut self, + msg: timed_protocol::Brief, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - std::thread::sleep(Duration::from_millis(msg.millis)); - msg.millis - }; + std::thread::sleep(Duration::from_millis(msg.millis)); + msg.millis + }; let _ = reply.reply(__reply_value); } } @@ -182,39 +207,56 @@ mod synchronous { } } impl Handler for Held { - fn handle(&mut self, _: zero_protocol::Zero, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + _: zero_protocol::Zero, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } impl Handler for Worker { - fn handle(&mut self, _: naming_protocol::Timeout, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - 31 - }; + fn handle( + &mut self, + _: naming_protocol::Timeout, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { 31 }; let _ = reply.reply(__reply_value); } } impl Handler for Worker { - fn handle(&mut self, _: naming_protocol::ActorRef, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - 32 - }; + fn handle( + &mut self, + _: naming_protocol::ActorRef, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { 32 }; let _ = reply.reply(__reply_value); } } impl Handler for Worker { - fn handle(&mut self, _: naming_protocol::TimeoutOverride, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - 33 - }; + fn handle( + &mut self, + _: naming_protocol::TimeoutOverride, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { 33 }; let _ = reply.reply(__reply_value); } } impl Handler for Worker { - fn handle(&mut self, _: naming_protocol::Dispatch0, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - 34 - }; + fn handle( + &mut self, + _: naming_protocol::Dispatch0, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { 34 }; let _ = reply.reply(__reply_value); } } @@ -234,10 +276,13 @@ mod synchronous { actor.join(); } impl Handler for Worker { - fn handle(&mut self, msg: owned_protocol::Owned, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - msg.__spawned_request_timeout.value - }; + fn handle( + &mut self, + msg: owned_protocol::Owned, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { msg.__spawned_request_timeout.value }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index 39076cf6..6cfb54c3 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -57,7 +57,11 @@ impl Clock { /// once every thread has joined. fn assert_balanced(pool: &CreditPool, slots: &[&Slot]) { let held: u64 = slots.iter().map(|s| credits(s.word())).sum(); - assert_eq!(pool.used() as u64, held, "credit counter diverged from slot words"); + assert_eq!( + pool.used() as u64, + held, + "credit counter diverged from slot words" + ); assert!(pool.used() <= pool.max(), "credit bound exceeded"); } @@ -94,7 +98,10 @@ fn reply_vs_expiry() { match delivery { Delivery::Delivered => { assert_eq!(state(slot.word()), slot::REPLIED); - assert!(replied_at.load(Ordering::SeqCst) < 10, "success at the deadline"); + assert!( + replied_at.load(Ordering::SeqCst) < 10, + "success at the deadline" + ); assert!(back.is_none()); } Delivery::Expired => { @@ -166,7 +173,10 @@ fn reply_vs_owner_close() { loom::thread::spawn(move || slot.reply(1, &|| 0).0) }; let closed = slot.close(); - assert!(slot.is_terminal(), "close returned while a reply was WRITING"); + assert!( + slot.is_terminal(), + "close returned while a reply was WRITING" + ); let delivery = replier.join().unwrap(); if closed { assert_eq!(state(slot.word()), slot::STOPPED); @@ -258,7 +268,8 @@ fn attach_vs_job_end_vs_close() { model(|| { let pool = Arc::new(CreditPool::new(2)); let slot = deferred(&pool, NEVER); - slot.attach(&|| 0).expect("first attach shares the token's credit"); + slot.attach(&|| 0) + .expect("first attach shares the token's credit"); assert_eq!(pool.used(), 1); let second = { let slot = slot.clone(); diff --git a/concurrency/tests/resource_cleanup.rs b/concurrency/tests/resource_cleanup.rs index 5f849034..25a45868 100644 --- a/concurrency/tests/resource_cleanup.rs +++ b/concurrency/tests/resource_cleanup.rs @@ -15,19 +15,29 @@ struct Sink; impl tasks::Actor for Sink {} #[cfg(feature = "async")] impl tasks::Handler for Sink { - async fn handle(&mut self, msg: Probe, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &tasks::Context) { + async fn handle( + &mut self, + msg: Probe, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &tasks::Context, + ) { { - drop(msg.0); - } + drop(msg.0); + } let _ = reply.reply(()); } } impl threads::Actor for Sink {} impl threads::Handler for Sink { - fn handle(&mut self, msg: Probe, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context) { + fn handle( + &mut self, + msg: Probe, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &threads::Context, + ) { { - drop(msg.0); - } + drop(msg.0); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/semantics_gaps.rs b/concurrency/tests/semantics_gaps.rs index 29b835fe..b805ef4b 100644 --- a/concurrency/tests/semantics_gaps.rs +++ b/concurrency/tests/semantics_gaps.rs @@ -101,28 +101,43 @@ mod native { } } impl Handler for Observer { - fn handle(&mut self, watch: Watch, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + fn handle( + &mut self, + watch: Watch, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.trap_exit(true); - ctx.link(&watch.0); - ctx.monitor(&watch.0); - } + ctx.trap_exit(true); + ctx.link(&watch.0); + ctx.monitor(&watch.0); + } let _ = reply.reply(()); } } impl Handler for Observer { - fn handle(&mut self, ping: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + ping: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - let _ = self.0.send(Event::Ping(ping.0)); - } + let _ = self.0.send(Event::Ping(ping.0)); + } let _ = reply.reply(()); } } impl Handler for Observer { - fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - let _ = self.0.send(Event::Down(down.reason)); - } + let _ = self.0.send(Event::Down(down.reason)); + } let _ = reply.reply(()); } } @@ -130,16 +145,21 @@ mod native { struct Source; impl Actor for Source {} impl Handler>> for Source { - fn handle(&mut self, burst: Burst>, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + fn handle( + &mut self, + burst: Burst>, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - for i in 0..burst.count { - burst.to.send(Ping(i)).unwrap(); - } - match burst.reason { - Some(reason) => ctx.stop_with_reason(reason), - None => panic!("source panicked"), + for i in 0..burst.count { + burst.to.send(Ping(i)).unwrap(); + } + match burst.reason { + Some(reason) => ctx.stop_with_reason(reason), + None => panic!("source panicked"), + } } - } let _ = reply.reply(()); } } @@ -175,10 +195,13 @@ mod native { struct Panicker; impl Actor for Panicker {} impl Handler for Panicker { - fn handle(&mut self, _: Boom, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - panic!("request handler panic") - }; + fn handle( + &mut self, + _: Boom, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { panic!("request handler panic") }; let _ = reply.reply(__reply_value); } } @@ -243,28 +266,43 @@ mod tasks { } } impl Handler for Observer { - async fn handle(&mut self, watch: Watch, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + watch: Watch, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.trap_exit(true); - ctx.link(&watch.0); - ctx.monitor(&watch.0); - } + ctx.trap_exit(true); + ctx.link(&watch.0); + ctx.monitor(&watch.0); + } let _ = reply.reply(()); } } impl Handler for Observer { - async fn handle(&mut self, ping: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + ping: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - let _ = self.0.send(Event::Ping(ping.0)); - } + let _ = self.0.send(Event::Ping(ping.0)); + } let _ = reply.reply(()); } } impl Handler for Observer { - async fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - let _ = self.0.send(Event::Down(down.reason)); - } + let _ = self.0.send(Event::Down(down.reason)); + } let _ = reply.reply(()); } } @@ -272,16 +310,21 @@ mod tasks { struct Source; impl Actor for Source {} impl Handler>> for Source { - async fn handle(&mut self, burst: Burst>, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + burst: Burst>, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - for i in 0..burst.count { - burst.to.send(Ping(i)).unwrap(); - } - match burst.reason { - Some(reason) => ctx.stop_with_reason(reason), - None => panic!("source panicked"), + for i in 0..burst.count { + burst.to.send(Ping(i)).unwrap(); + } + match burst.reason { + Some(reason) => ctx.stop_with_reason(reason), + None => panic!("source panicked"), + } } - } let _ = reply.reply(()); } } @@ -327,10 +370,15 @@ mod tasks { struct Ticker(UnboundedSender); impl Actor for Ticker {} impl Handler for Ticker { - async fn handle(&mut self, _: Tick, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Tick, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - let _ = self.0.send(Instant::now()); - } + let _ = self.0.send(Instant::now()); + } let _ = reply.reply(()); } } @@ -414,18 +462,28 @@ mod tasks { } } impl Handler for Clocked { - async fn handle(&mut self, stamp: Stamp, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + stamp: Stamp, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - let _ = self.log.send((self.generation, stamp.0)); - } + let _ = self.log.send((self.generation, stamp.0)); + } let _ = reply.reply(()); } } impl Handler for Clocked { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("injected crash"); - } + panic!("injected crash"); + } let _ = reply.reply(()); } } @@ -501,10 +559,13 @@ mod tasks { struct Panicker; impl Actor for Panicker {} impl Handler for Panicker { - async fn handle(&mut self, _: Boom, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - panic!("request handler panic") - }; + async fn handle( + &mut self, + _: Boom, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { panic!("request handler panic") }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/shutdown_publication.rs b/concurrency/tests/shutdown_publication.rs index 80a5d3c2..0354d9c9 100644 --- a/concurrency/tests/shutdown_publication.rs +++ b/concurrency/tests/shutdown_publication.rs @@ -66,16 +66,21 @@ impl Actor for Worker { } } impl Handler for Worker { - async fn handle(&mut self, msg: Command, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: Command, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - match msg { - Command::Hold { entered, release } => { - let _ = entered.send(()); - let _ = release.await; + match msg { + Command::Hold { entered, release } => { + let _ = entered.send(()); + let _ = release.await; + } + Command::Work => panic!("queued work must be discarded after stop"), } - Command::Work => panic!("queued work must be discarded after stop"), } - } let _ = reply.reply(()); } } diff --git a/concurrency/tests/sim.rs b/concurrency/tests/sim.rs index 1c2b776d..1d116da3 100644 --- a/concurrency/tests/sim.rs +++ b/concurrency/tests/sim.rs @@ -85,10 +85,15 @@ impl Actor for Node { } } impl Handler for Node { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("injected crash after readiness"); - } + panic!("injected crash after readiness"); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/sim_explore.rs b/concurrency/tests/sim_explore.rs index ba9eaf16..6a1ab361 100644 --- a/concurrency/tests/sim_explore.rs +++ b/concurrency/tests/sim_explore.rs @@ -56,28 +56,41 @@ impl Message for Log { type Result = Vec; } impl Handler for Store { - async fn handle(&mut self, _: Get, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + async fn handle( + &mut self, + _: Get, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - self.log.push(Access::Read(self.value)); - self.value - }; + self.log.push(Access::Read(self.value)); + self.value + }; let _ = reply.reply(__reply_value); } } impl Handler for Store { - async fn handle(&mut self, Set(value): Set, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + Set(value): Set, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.log.push(Access::Write(value)); - self.value = value; - } + self.log.push(Access::Write(value)); + self.value = value; + } let _ = reply.reply(()); } } impl Handler for Store { - async fn handle(&mut self, _: Log, reply: spawned_concurrency::reply::PendingReply<'_, Vec>, _: &Context) { - let __reply_value: Vec = { - self.log.clone() - }; + async fn handle( + &mut self, + _: Log, + reply: spawned_concurrency::reply::PendingReply<'_, Vec>, + _: &Context, + ) { + let __reply_value: Vec = { self.log.clone() }; let _ = reply.reply(__reply_value); } } @@ -96,14 +109,19 @@ impl Message for Bump { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _: Bump, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Bump, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - for _ in 0..self.noise { - spawned_rt::tasks::yield_now().await; + for _ in 0..self.noise { + spawned_rt::tasks::yield_now().await; + } + let value = self.store.request(Get).await.unwrap(); + self.store.request(Set(value + 1)).await.unwrap(); } - let value = self.store.request(Get).await.unwrap(); - self.store.request(Set(value + 1)).await.unwrap(); - } let _ = reply.reply(()); } } @@ -212,10 +230,15 @@ impl Actor for Node { } } impl Handler for Node { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("injected crash after readiness"); - } + panic!("injected crash after readiness"); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/stack_budget.rs b/concurrency/tests/stack_budget.rs index 13eaf7f3..e518a776 100644 --- a/concurrency/tests/stack_budget.rs +++ b/concurrency/tests/stack_budget.rs @@ -117,15 +117,20 @@ mod tasks_chain { } impl Actor for Hop {} impl Handler for Hop { - async fn handle(&mut self, msg: relay::Forward, reply: spawned_concurrency::reply::PendingReply<'_, Block>, _: &Context) { + async fn handle( + &mut self, + msg: relay::Forward, + reply: spawned_concurrency::reply::PendingReply<'_, Block>, + _: &Context, + ) { let __reply_value: Block = { - let mut block = msg.block; - stamp(&mut block, self.index); - match &self.next { - Some(next) => next.forward(block).await.expect("downstream hop"), - None => block, - } - }; + let mut block = msg.block; + stamp(&mut block, self.index); + match &self.next { + Some(next) => next.forward(block).await.expect("downstream hop"), + None => block, + } + }; let _ = reply.reply(__reply_value); } } @@ -218,20 +223,28 @@ mod tasks_chain { } impl Actor for Flaky {} impl Handler for Flaky { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - if self.generation == 1 { - panic!("injected stack-budget crash"); + if self.generation == 1 { + panic!("injected stack-budget crash"); + } } - } let _ = reply.reply(()); } } impl Handler for Flaky { - async fn handle(&mut self, msg: relay::Forward, reply: spawned_concurrency::reply::PendingReply<'_, Block>, _: &Context) { - let __reply_value: Block = { - msg.block - }; + async fn handle( + &mut self, + msg: relay::Forward, + reply: spawned_concurrency::reply::PendingReply<'_, Block>, + _: &Context, + ) { + let __reply_value: Block = { msg.block }; let _ = reply.reply(__reply_value); } } @@ -304,15 +317,20 @@ mod threads_chain { } impl Actor for Hop {} impl Handler for Hop { - fn handle(&mut self, msg: relay::Forward, reply: spawned_concurrency::reply::PendingReply<'_, Block>, _: &Context) { + fn handle( + &mut self, + msg: relay::Forward, + reply: spawned_concurrency::reply::PendingReply<'_, Block>, + _: &Context, + ) { let __reply_value: Block = { - let mut block = msg.block; - stamp(&mut block, self.index); - match &self.next { - Some(next) => next.forward(block).unwrap(), - None => block, - } - }; + let mut block = msg.block; + stamp(&mut block, self.index); + match &self.next { + Some(next) => next.forward(block).unwrap(), + None => block, + } + }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/startup_admission.rs b/concurrency/tests/startup_admission.rs index bb767e77..d01bde7e 100644 --- a/concurrency/tests/startup_admission.rs +++ b/concurrency/tests/startup_admission.rs @@ -33,10 +33,15 @@ impl Drop for Queued { } } impl spawned_concurrency::tasks::Handler for Probe { - async fn handle(&mut self, _: Queued, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Queued, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("unstarted handler ran"); - } + panic!("unstarted handler ran"); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/supervision.rs b/concurrency/tests/supervision.rs index 3caf4b7d..e2009ebc 100644 --- a/concurrency/tests/supervision.rs +++ b/concurrency/tests/supervision.rs @@ -40,19 +40,29 @@ impl Message for Crash { struct Counter(usize); impl Actor for Counter {} impl Handler for Counter { - async fn handle(&mut self, _: Next, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { + async fn handle( + &mut self, + _: Next, + reply: spawned_concurrency::reply::PendingReply<'_, usize>, + _: &Context, + ) { let __reply_value: usize = { - self.0 += 1; - self.0 - }; + self.0 += 1; + self.0 + }; let _ = reply.reply(__reply_value); } } impl Handler for Counter { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("injected child failure"); - } + panic!("injected child failure"); + } let _ = reply.reply(()); } } @@ -281,10 +291,15 @@ impl Actor for SlowHelper { } } impl Handler for SlowHelper { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("injected failure with active helper"); - } + panic!("injected failure with active helper"); + } let _ = reply.reply(()); } } @@ -351,10 +366,15 @@ impl Actor for SlowStop { } } impl Handler for SlowStop { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("failure before stuck stopped hook"); - } + panic!("failure before stuck stopped hook"); + } let _ = reply.reply(()); } } @@ -539,19 +559,29 @@ impl Drop for RaceActor { } impl Actor for RaceActor {} impl Handler for RaceActor { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("racing child failure"); - } + panic!("racing child failure"); + } let _ = reply.reply(()); } } impl Handler for RaceActor { - async fn handle(&mut self, _: Next, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { + async fn handle( + &mut self, + _: Next, + reply: spawned_concurrency::reply::PendingReply<'_, usize>, + _: &Context, + ) { let __reply_value: usize = { - self.count += 1; - self.count - }; + self.count += 1; + self.count + }; let _ = reply.reply(__reply_value); } } @@ -835,10 +865,15 @@ impl Actor for ExitOnceDuringRestart { } } impl Handler for ExitOnceDuringRestart { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("injected child failure"); - } + panic!("injected child failure"); + } let _ = reply.reply(()); } } @@ -1307,10 +1342,15 @@ impl Actor for Named { } } impl Handler for Named { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("crash a registered child"); - } + panic!("crash a registered child"); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/supervision_strategies.rs b/concurrency/tests/supervision_strategies.rs index 7c79985c..e25a0077 100644 --- a/concurrency/tests/supervision_strategies.rs +++ b/concurrency/tests/supervision_strategies.rs @@ -70,28 +70,43 @@ impl Drop for Probe { } } impl Handler for Probe { - async fn handle(&mut self, _: Next, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { + async fn handle( + &mut self, + _: Next, + reply: spawned_concurrency::reply::PendingReply<'_, usize>, + _: &Context, + ) { let __reply_value: usize = { - self.value += 1; - self.value - }; + self.value += 1; + self.value + }; let _ = reply.reply(__reply_value); } } impl Handler for Probe { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("injected cohort failure"); - } + panic!("injected cohort failure"); + } let _ = reply.reply(()); } } impl Handler for Probe { - async fn handle(&mut self, msg: CrashTogether, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: CrashTogether, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - msg.0.wait().await; - panic!("simultaneous cohort failure"); - } + msg.0.wait().await; + panic!("simultaneous cohort failure"); + } let _ = reply.reply(()); } } @@ -714,10 +729,15 @@ impl Actor for MaybeStop { } } impl Handler for MaybeStop { - async fn handle(&mut self, _: CrashPlain, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: CrashPlain, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("injected cohort failure"); - } + panic!("injected cohort failure"); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/supervision_trace.rs b/concurrency/tests/supervision_trace.rs index a9bef362..b63312bf 100644 --- a/concurrency/tests/supervision_trace.rs +++ b/concurrency/tests/supervision_trace.rs @@ -98,11 +98,16 @@ impl Message for HoldForDrain { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, msg: HoldForDrain, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: HoldForDrain, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - msg.0.send(()).unwrap(); - let _ = msg.1.await; - } + msg.0.send(()).unwrap(); + let _ = msg.1.await; + } let _ = reply.reply(()); } } @@ -116,10 +121,15 @@ impl Drop for DropBomb { } } impl Handler for Worker { - async fn handle(&mut self, _: DropBomb, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: DropBomb, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("discarded message must not be handled"); - } + panic!("discarded message must not be handled"); + } let _ = reply.reply(()); } } @@ -203,26 +213,41 @@ async fn trace_queued_destructor_failure_preserves_shutdown() { assert!(system.shutdown(LIMIT).await.complete); } impl Handler for Worker { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("injected child failure"); - } + panic!("injected child failure"); + } let _ = reply.reply(()); } } impl Handler for Worker { - async fn handle(&mut self, _: Quit, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + _: Quit, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.stop(); - } + ctx.stop(); + } let _ = reply.reply(()); } } impl Handler for Worker { - async fn handle(&mut self, msg: ExitWith, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + msg: ExitWith, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.stop_with_reason(msg.0); - } + ctx.stop_with_reason(msg.0); + } let _ = reply.reply(()); } } @@ -513,10 +538,15 @@ impl Actor for ExitOnceDuringRestart { } } impl Handler for ExitOnceDuringRestart { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("injected child failure"); - } + panic!("injected child failure"); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/supervisor_policies.rs b/concurrency/tests/supervisor_policies.rs index 3c0a0b2b..b31f5a3e 100644 --- a/concurrency/tests/supervisor_policies.rs +++ b/concurrency/tests/supervisor_policies.rs @@ -25,18 +25,26 @@ impl Message for Crash { struct Probe(u64); impl Actor for Probe {} impl Handler for Probe { - async fn handle(&mut self, _: Generation, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - self.0 - }; + async fn handle( + &mut self, + _: Generation, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { self.0 }; let _ = reply.reply(__reply_value); } } impl Handler for Probe { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("generated supervisor failure"); - } + panic!("generated supervisor failure"); + } let _ = reply.reply(()); } } diff --git a/concurrency/tests/support/failure_actor.rs b/concurrency/tests/support/failure_actor.rs index 0dda354d..58b343d7 100644 --- a/concurrency/tests/support/failure_actor.rs +++ b/concurrency/tests/support/failure_actor.rs @@ -156,44 +156,54 @@ impl spawned_concurrency::tasks::Actor for AsyncWorker { } #[cfg(feature = "async")] impl spawned_concurrency::tasks::Handler for AsyncWorker { - async fn handle(&mut self, command: Command, reply: spawned_concurrency::reply::PendingReply<'_, Reply>, _: &spawned_concurrency::tasks::Context) { + async fn handle( + &mut self, + command: Command, + reply: spawned_concurrency::reply::PendingReply<'_, Reply>, + _: &spawned_concurrency::tasks::Context, + ) { let __reply_value: Reply = { - match command { - Command::Payload { id, payload } => { - assert_eq!(payload.bytes[0], (id % 251) as u8); - self.0.execute(id) - } - Command::Read => Reply::Values(self.0.values.clone()), - Command::Panic => panic!("controlled handler panic"), - Command::Append { - id, - before, - after, - panic_after, - } => { - if let Some(gate) = before { - gate.wait().await; + match command { + Command::Payload { id, payload } => { + assert_eq!(payload.bytes[0], (id % 251) as u8); + self.0.execute(id) } - let result = self.0.execute(id); - if let Some(gate) = after { - gate.wait().await; + Command::Read => Reply::Values(self.0.values.clone()), + Command::Panic => panic!("controlled handler panic"), + Command::Append { + id, + before, + after, + panic_after, + } => { + if let Some(gate) = before { + gate.wait().await; + } + let result = self.0.execute(id); + if let Some(gate) = after { + gate.wait().await; + } + assert!(!panic_after, "controlled panic after effect"); + result } - assert!(!panic_after, "controlled panic after effect"); - result } - } - }; + }; let _ = reply.reply(__reply_value); } } #[cfg(feature = "async")] impl spawned_concurrency::tasks::Handler for AsyncWorker { - async fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &spawned_concurrency::tasks::Context) { + async fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &spawned_concurrency::tasks::Context, + ) { { - if let Some(sender) = &self.0.down { - let _ = sender.send(down); + if let Some(sender) = &self.0.down { + let _ = sender.send(down); + } } - } let _ = reply.reply(()); } } @@ -212,43 +222,53 @@ impl spawned_concurrency::threads::Actor for NativeWorker { } } impl spawned_concurrency::threads::Handler for NativeWorker { - fn handle(&mut self, command: Command, reply: spawned_concurrency::reply::PendingReply<'_, Reply>, _: &spawned_concurrency::threads::Context) { + fn handle( + &mut self, + command: Command, + reply: spawned_concurrency::reply::PendingReply<'_, Reply>, + _: &spawned_concurrency::threads::Context, + ) { let __reply_value: Reply = { - match command { - Command::Payload { id, payload } => { - assert_eq!(payload.bytes[0], (id % 251) as u8); - self.0.execute(id) - } - Command::Read => Reply::Values(self.0.values.clone()), - Command::Panic => panic!("controlled handler panic"), - Command::Append { - id, - before, - after, - panic_after, - } => { - if let Some(gate) = before { - futures::executor::block_on(gate.wait()); + match command { + Command::Payload { id, payload } => { + assert_eq!(payload.bytes[0], (id % 251) as u8); + self.0.execute(id) } - let result = self.0.execute(id); - if let Some(gate) = after { - futures::executor::block_on(gate.wait()); + Command::Read => Reply::Values(self.0.values.clone()), + Command::Panic => panic!("controlled handler panic"), + Command::Append { + id, + before, + after, + panic_after, + } => { + if let Some(gate) = before { + futures::executor::block_on(gate.wait()); + } + let result = self.0.execute(id); + if let Some(gate) = after { + futures::executor::block_on(gate.wait()); + } + assert!(!panic_after, "controlled panic after effect"); + result } - assert!(!panic_after, "controlled panic after effect"); - result } - } - }; + }; let _ = reply.reply(__reply_value); } } impl spawned_concurrency::threads::Handler for NativeWorker { - fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &spawned_concurrency::threads::Context) { + fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &spawned_concurrency::threads::Context, + ) { { - if let Some(sender) = &self.0.down { - let _ = sender.send(down); + if let Some(sender) = &self.0.down { + let _ = sender.send(down); + } } - } let _ = reply.reply(()); } } diff --git a/concurrency/tests/support/store_actor.rs b/concurrency/tests/support/store_actor.rs index 6ca90d2e..344a13c3 100644 --- a/concurrency/tests/support/store_actor.rs +++ b/concurrency/tests/support/store_actor.rs @@ -112,14 +112,19 @@ impl spawned_concurrency::tasks::Actor for AsyncStore {} #[cfg(feature = "async")] impl spawned_concurrency::tasks::Handler for AsyncStore { - async fn handle(&mut self, op: StoreOp, reply: spawned_concurrency::reply::PendingReply<'_, Option>, _: &spawned_concurrency::tasks::Context) { + async fn handle( + &mut self, + op: StoreOp, + reply: spawned_concurrency::reply::PendingReply<'_, Option>, + _: &spawned_concurrency::tasks::Context, + ) { let __reply_value: Option = { - // Yield inside the turn so concurrent requests really queue up. - if op.key().1.is_multiple_of(2) { - tokio::task::yield_now().await; - } - self.0.apply(op) - }; + // Yield inside the turn so concurrent requests really queue up. + if op.key().1.is_multiple_of(2) { + tokio::task::yield_now().await; + } + self.0.apply(op) + }; let _ = reply.reply(__reply_value); } } @@ -129,10 +134,13 @@ pub struct NativeStore(pub Store); impl spawned_concurrency::threads::Actor for NativeStore {} impl spawned_concurrency::threads::Handler for NativeStore { - fn handle(&mut self, op: StoreOp, reply: spawned_concurrency::reply::PendingReply<'_, Option>, _: &spawned_concurrency::threads::Context) { - let __reply_value: Option = { - self.0.apply(op) - }; + fn handle( + &mut self, + op: StoreOp, + reply: spawned_concurrency::reply::PendingReply<'_, Option>, + _: &spawned_concurrency::threads::Context, + ) { + let __reply_value: Option = { self.0.apply(op) }; let _ = reply.reply(__reply_value); } } diff --git a/concurrency/tests/tree_trace.rs b/concurrency/tests/tree_trace.rs index 4c17a5a3..32318c0e 100644 --- a/concurrency/tests/tree_trace.rs +++ b/concurrency/tests/tree_trace.rs @@ -20,10 +20,15 @@ impl Message for Crash { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("nested trace failure"); - } + panic!("nested trace failure"); + } let _ = reply.reply(()); } } diff --git a/examples/capabilities/src/bin/byte_budget_pipeline.rs b/examples/capabilities/src/bin/byte_budget_pipeline.rs index dab3eced..a7e556d5 100644 --- a/examples/capabilities/src/bin/byte_budget_pipeline.rs +++ b/examples/capabilities/src/bin/byte_budget_pipeline.rs @@ -18,15 +18,20 @@ impl Message for Packet { struct Sink(usize); impl Actor for Sink {} impl Handler for Sink { - async fn handle(&mut self, packet: Packet, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { + async fn handle( + &mut self, + packet: Packet, + reply: spawned_concurrency::reply::PendingReply<'_, usize>, + _: &Context, + ) { let __reply_value: usize = { - let Packet { bytes, credit } = packet; - let size = bytes.len(); - self.0 += size; - drop(bytes); - drop(credit); // Capacity returns after the payload is released. - size - }; + let Packet { bytes, credit } = packet; + let size = bytes.len(); + self.0 += size; + drop(bytes); + drop(credit); // Capacity returns after the payload is released. + size + }; let _ = reply.reply(__reply_value); } } @@ -35,20 +40,26 @@ impl Message for Total { type Result = usize; } impl Handler for Sink { - async fn handle(&mut self, _: Total, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { - let __reply_value: usize = { - self.0 - }; + async fn handle( + &mut self, + _: Total, + reply: spawned_concurrency::reply::PendingReply<'_, usize>, + _: &Context, + ) { + let __reply_value: usize = { self.0 }; let _ = reply.reply(__reply_value); } } struct Forwarder(ActorRef); impl Actor for Forwarder {} impl Handler for Forwarder { - async fn handle(&mut self, packet: Packet, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { - let __reply_value: usize = { - self.0.request(packet).await.unwrap() - }; + async fn handle( + &mut self, + packet: Packet, + reply: spawned_concurrency::reply::PendingReply<'_, usize>, + _: &Context, + ) { + let __reply_value: usize = { self.0.request(packet).await.unwrap() }; let _ = reply.reply(__reply_value); } } diff --git a/examples/capabilities/src/bin/coordinated_shutdown.rs b/examples/capabilities/src/bin/coordinated_shutdown.rs index 95abdd59..6cb12fd6 100644 --- a/examples/capabilities/src/bin/coordinated_shutdown.rs +++ b/examples/capabilities/src/bin/coordinated_shutdown.rs @@ -36,18 +36,26 @@ impl Actor for Peers { } } impl Handler for Peers { - async fn handle(&mut self, msg: Peer, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: Peer, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.peers.push(msg.0); - } + self.peers.push(msg.0); + } let _ = reply.reply(()); } } impl Handler for Peers { - async fn handle(&mut self, _: Snapshot, reply: spawned_concurrency::reply::PendingReply<'_, Vec>, _: &Context) { - let __reply_value: Vec = { - self.peers.clone() - }; + async fn handle( + &mut self, + _: Snapshot, + reply: spawned_concurrency::reply::PendingReply<'_, Vec>, + _: &Context, + ) { + let __reply_value: Vec = { self.peers.clone() }; let _ = reply.reply(__reply_value); } } @@ -61,15 +69,25 @@ impl Actor for Importer { } } impl Handler for Importer { - async fn handle(&mut self, msg: Import, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: Import, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.peers.send(Peer(msg.0)).unwrap(); - } + self.peers.send(Peer(msg.0)).unwrap(); + } let _ = reply.reply(()); } } impl Handler for Importer { - async fn handle(&mut self, _: Fence, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Fence, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } diff --git a/examples/capabilities/src/bin/dedicated_thread.rs b/examples/capabilities/src/bin/dedicated_thread.rs index d2cc05b5..86c837d3 100644 --- a/examples/capabilities/src/bin/dedicated_thread.rs +++ b/examples/capabilities/src/bin/dedicated_thread.rs @@ -12,23 +12,28 @@ impl Message for Check { type Result = (usize, u64); } impl Handler for Worker { - async fn handle(&mut self, _: Check, reply: spawned_concurrency::reply::PendingReply<'_, (usize, u64)>, _: &Context) { + async fn handle( + &mut self, + _: Check, + reply: spawned_concurrency::reply::PendingReply<'_, (usize, u64)>, + _: &Context, + ) { let __reply_value: (usize, u64) = { - let root = thread::current().id(); - let workers = tokio::runtime::Handle::current().metrics().num_workers(); - let (tx, rx) = mpsc::channel(); - let helper = tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(1)).await; - assert_ne!(root, thread::current().id()); - tx.send(42).unwrap(); - }); - // This would stall a current-thread executor. The isolated runtime's - // one async worker can drive the helper independently of this root. - let answer = rx.recv_timeout(Duration::from_secs(5)).unwrap(); - helper.await.unwrap(); - assert_eq!(root, thread::current().id()); - (workers, answer) - }; + let root = thread::current().id(); + let workers = tokio::runtime::Handle::current().metrics().num_workers(); + let (tx, rx) = mpsc::channel(); + let helper = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(1)).await; + assert_ne!(root, thread::current().id()); + tx.send(42).unwrap(); + }); + // This would stall a current-thread executor. The isolated runtime's + // one async worker can drive the helper independently of this root. + let answer = rx.recv_timeout(Duration::from_secs(5)).unwrap(); + helper.await.unwrap(); + assert_eq!(root, thread::current().id()); + (workers, answer) + }; let _ = reply.reply(__reply_value); } } diff --git a/examples/capabilities/src/bin/execution_groups.rs b/examples/capabilities/src/bin/execution_groups.rs index 5ce92750..0d02234d 100644 --- a/examples/capabilities/src/bin/execution_groups.rs +++ b/examples/capabilities/src/bin/execution_groups.rs @@ -12,20 +12,26 @@ impl Message for Double { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, msg: Double, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - msg.0 * 2 - }; + async fn handle( + &mut self, + msg: Double, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { msg.0 * 2 }; let _ = reply.reply(__reply_value); } } struct Router(ActorRef); impl Actor for Router {} impl Handler for Router { - async fn handle(&mut self, msg: Double, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - self.0.request(msg).await.unwrap() - }; + async fn handle( + &mut self, + msg: Double, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { self.0.request(msg).await.unwrap() }; let _ = reply.reply(__reply_value); } } diff --git a/examples/capabilities/src/bin/execution_profiles.rs b/examples/capabilities/src/bin/execution_profiles.rs index 03e06fe7..c8c61f7b 100644 --- a/examples/capabilities/src/bin/execution_profiles.rs +++ b/examples/capabilities/src/bin/execution_profiles.rs @@ -12,13 +12,18 @@ impl Message for Location { type Result = (String, Option); } impl Handler for Worker { - async fn handle(&mut self, _: Location, reply: spawned_concurrency::reply::PendingReply<'_, ::Result>, _: &Context) { + async fn handle( + &mut self, + _: Location, + reply: spawned_concurrency::reply::PendingReply<'_, ::Result>, + _: &Context, + ) { let __reply_value: ::Result = { - ( - std::thread::current().name().unwrap_or_default().to_owned(), - spawned_rt::tasks::affinity::current_cpu().ok(), - ) - }; + ( + std::thread::current().name().unwrap_or_default().to_owned(), + spawned_rt::tasks::affinity::current_cpu().ok(), + ) + }; let _ = reply.reply(__reply_value); } } diff --git a/examples/capabilities/src/bin/framework_metrics.rs b/examples/capabilities/src/bin/framework_metrics.rs index e679276f..a1c8dbf4 100644 --- a/examples/capabilities/src/bin/framework_metrics.rs +++ b/examples/capabilities/src/bin/framework_metrics.rs @@ -11,7 +11,12 @@ impl Message for Ping { } impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } diff --git a/examples/capabilities/src/bin/graceful_drain.rs b/examples/capabilities/src/bin/graceful_drain.rs index 7e2e3acd..f3c28391 100644 --- a/examples/capabilities/src/bin/graceful_drain.rs +++ b/examples/capabilities/src/bin/graceful_drain.rs @@ -10,10 +10,15 @@ impl Message for Add { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _: Add, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Add, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0 += 1; - } + self.0 += 1; + } let _ = reply.reply(()); } } @@ -22,10 +27,13 @@ impl Message for Fence { type Result = usize; } impl Handler for Worker { - async fn handle(&mut self, _: Fence, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { - let __reply_value: usize = { - self.0 - }; + async fn handle( + &mut self, + _: Fence, + reply: spawned_concurrency::reply::PendingReply<'_, usize>, + _: &Context, + ) { + let __reply_value: usize = { self.0 }; let _ = reply.reply(__reply_value); } } diff --git a/examples/capabilities/src/bin/native_lifecycle.rs b/examples/capabilities/src/bin/native_lifecycle.rs index b3d037c0..db9b1cd9 100644 --- a/examples/capabilities/src/bin/native_lifecycle.rs +++ b/examples/capabilities/src/bin/native_lifecycle.rs @@ -19,11 +19,16 @@ impl Actor for Observer { } } impl Handler for Observer { - fn handle(&mut self, event: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + event: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.counts.1 += 1; - self.downs.send(event).unwrap(); - } + self.counts.1 += 1; + self.downs.send(event).unwrap(); + } let _ = reply.reply(()); } } @@ -32,10 +37,13 @@ impl Message for Counts { type Result = (usize, usize); } impl Handler for Observer { - fn handle(&mut self, _: Counts, reply: spawned_concurrency::reply::PendingReply<'_, (usize, usize)>, _: &Context) { - let __reply_value: (usize, usize) = { - self.counts - }; + fn handle( + &mut self, + _: Counts, + reply: spawned_concurrency::reply::PendingReply<'_, (usize, usize)>, + _: &Context, + ) { + let __reply_value: (usize, usize) = { self.counts }; let _ = reply.reply(__reply_value); } } diff --git a/examples/capabilities/src/bin/protocol_timeouts.rs b/examples/capabilities/src/bin/protocol_timeouts.rs index c6c2960a..ecb0f6f6 100644 --- a/examples/capabilities/src/bin/protocol_timeouts.rs +++ b/examples/capabilities/src/bin/protocol_timeouts.rs @@ -23,20 +23,30 @@ mod asynchronous { } } impl Handler for Worker { - async fn handle(&mut self, _: lookup_protocol::Value, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + async fn handle( + &mut self, + _: lookup_protocol::Value, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - self.1.fetch_add(1, Ordering::SeqCst); - 42 - }; + self.1.fetch_add(1, Ordering::SeqCst); + 42 + }; let _ = reply.reply(__reply_value); } } impl Handler for Worker { - async fn handle(&mut self, _: lookup_protocol::Quick, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + async fn handle( + &mut self, + _: lookup_protocol::Quick, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - self.1.fetch_add(1, Ordering::SeqCst); - 42 - }; + self.1.fetch_add(1, Ordering::SeqCst); + 42 + }; let _ = reply.reply(__reply_value); } } @@ -69,20 +79,30 @@ mod synchronous { } } impl Handler for Worker { - fn handle(&mut self, _: lookup_protocol::Value, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + fn handle( + &mut self, + _: lookup_protocol::Value, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - self.1.fetch_add(1, Ordering::SeqCst); - 42 - }; + self.1.fetch_add(1, Ordering::SeqCst); + 42 + }; let _ = reply.reply(__reply_value); } } impl Handler for Worker { - fn handle(&mut self, _: lookup_protocol::Quick, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + fn handle( + &mut self, + _: lookup_protocol::Quick, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - self.1.fetch_add(1, Ordering::SeqCst); - 42 - }; + self.1.fetch_add(1, Ordering::SeqCst); + 42 + }; let _ = reply.reply(__reply_value); } } diff --git a/examples/capabilities/src/bin/registrations_and_fakes.rs b/examples/capabilities/src/bin/registrations_and_fakes.rs index 6675f95a..764d1ba5 100644 --- a/examples/capabilities/src/bin/registrations_and_fakes.rs +++ b/examples/capabilities/src/bin/registrations_and_fakes.rs @@ -37,10 +37,15 @@ impl spawned_concurrency::message::Message for Crash { type Result = (); } impl Handler for Service { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("intentional named-service replacement"); - } + panic!("intentional named-service replacement"); + } let _ = reply.reply(()); } } diff --git a/examples/capabilities/src/bin/runtime_isolation.rs b/examples/capabilities/src/bin/runtime_isolation.rs index b9bc67db..a852427c 100644 --- a/examples/capabilities/src/bin/runtime_isolation.rs +++ b/examples/capabilities/src/bin/runtime_isolation.rs @@ -11,21 +11,29 @@ impl Message for Ping { } impl Actor for Critical {} impl Handler for Critical { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, String>, _: &Context) { - let __reply_value: String = { - std::thread::current().name().unwrap().to_owned() - }; + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, String>, + _: &Context, + ) { + let __reply_value: String = { std::thread::current().name().unwrap().to_owned() }; let _ = reply.reply(__reply_value); } } struct Router(ActorRef); impl Actor for Router {} impl Handler for Router { - async fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, String>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, String>, + _: &Context, + ) { let __reply_value: String = { - assert_eq!(std::thread::current().name(), Some("background-worker")); - self.0.request(Ping).await.unwrap() - }; + assert_eq!(std::thread::current().name(), Some("background-worker")); + self.0.request(Ping).await.unwrap() + }; let _ = reply.reply(__reply_value); } } diff --git a/examples/capabilities/src/bin/shutdown_diagnostics.rs b/examples/capabilities/src/bin/shutdown_diagnostics.rs index e017b424..3db8eb3b 100644 --- a/examples/capabilities/src/bin/shutdown_diagnostics.rs +++ b/examples/capabilities/src/bin/shutdown_diagnostics.rs @@ -16,11 +16,16 @@ impl Message for Hold { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, msg: Hold, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: Hold, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - let _ = msg.0.send(()); - let _ = msg.1.await; - } + let _ = msg.0.send(()); + let _ = msg.1.await; + } let _ = reply.reply(()); } } @@ -29,11 +34,16 @@ impl Message for Unhealthy { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _: Unhealthy, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + _: Unhealthy, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - // Cooperative abnormal exit, without a panic or forced interruption. - ctx.stop_with_reason(ExitReason::Panic("reported unhealthy worker".into())); - } + // Cooperative abnormal exit, without a panic or forced interruption. + ctx.stop_with_reason(ExitReason::Panic("reported unhealthy worker".into())); + } let _ = reply.reply(()); } } diff --git a/examples/capabilities/src/bin/streams_and_recipients.rs b/examples/capabilities/src/bin/streams_and_recipients.rs index 177ebb19..2dd4d44c 100644 --- a/examples/capabilities/src/bin/streams_and_recipients.rs +++ b/examples/capabilities/src/bin/streams_and_recipients.rs @@ -19,18 +19,26 @@ mod asynchronous { use spawned_concurrency::tasks::{self, Actor, ActorStart, Context, Handler, Recipient}; impl Actor for Counter {} impl Handler for Counter { - async fn handle(&mut self, msg: Item, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: Item, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0 += N * msg.0; - } + self.0 += N * msg.0; + } let _ = reply.reply(()); } } impl Handler for Counter { - async fn handle(&mut self, _: Total, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - self.0 - }; + async fn handle( + &mut self, + _: Total, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { self.0 }; let _ = reply.reply(__reply_value); } } @@ -79,18 +87,26 @@ mod synchronous { use spawned_concurrency::threads::{self, Actor, ActorStart, Context, Handler, Recipient}; impl Actor for Counter {} impl Handler for Counter { - fn handle(&mut self, msg: Item, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + msg: Item, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0 += N * msg.0; - } + self.0 += N * msg.0; + } let _ = reply.reply(()); } } impl Handler for Counter { - fn handle(&mut self, _: Total, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { - let __reply_value: u64 = { - self.0 - }; + fn handle( + &mut self, + _: Total, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { self.0 }; let _ = reply.reply(__reply_value); } } diff --git a/examples/capabilities/src/bin/supervision_recovery.rs b/examples/capabilities/src/bin/supervision_recovery.rs index 516ac62d..3efc6fb7 100644 --- a/examples/capabilities/src/bin/supervision_recovery.rs +++ b/examples/capabilities/src/bin/supervision_recovery.rs @@ -17,19 +17,29 @@ impl Message for Crash { struct Counter(u64); impl Actor for Counter {} impl Handler for Counter { - async fn handle(&mut self, _: Next, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context) { + async fn handle( + &mut self, + _: Next, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &Context, + ) { let __reply_value: u64 = { - self.0 += 1; - self.0 - }; + self.0 += 1; + self.0 + }; let _ = reply.reply(__reply_value); } } impl Handler for Counter { - async fn handle(&mut self, _: Crash, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("intentional recovery example failure"); - } + panic!("intentional recovery example failure"); + } let _ = reply.reply(()); } } diff --git a/examples/capabilities/src/bin/supervision_tree.rs b/examples/capabilities/src/bin/supervision_tree.rs index 846ce89c..758a1cfe 100644 --- a/examples/capabilities/src/bin/supervision_tree.rs +++ b/examples/capabilities/src/bin/supervision_tree.rs @@ -19,10 +19,15 @@ impl Message for Fail { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _: Fail, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Fail, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - panic!("example failure"); - } + panic!("example failure"); + } let _ = reply.reply(()); } } diff --git a/examples/capabilities/src/bin/timeout_outcomes.rs b/examples/capabilities/src/bin/timeout_outcomes.rs index 32423d6a..4e85ee2e 100644 --- a/examples/capabilities/src/bin/timeout_outcomes.rs +++ b/examples/capabilities/src/bin/timeout_outcomes.rs @@ -17,12 +17,17 @@ impl Message for Apply { type Result = (); } impl Handler for Counter { - async fn handle(&mut self, msg: Apply, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + msg: Apply, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0 += 1; // Effect happens before the reply. - msg.entered.send(()).unwrap(); - let _ = msg.release.await; - } + self.0 += 1; // Effect happens before the reply. + msg.entered.send(()).unwrap(); + let _ = msg.release.await; + } let _ = reply.reply(()); } } @@ -31,10 +36,13 @@ impl Message for Read { type Result = usize; } impl Handler for Counter { - async fn handle(&mut self, _: Read, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { - let __reply_value: usize = { - self.0 - }; + async fn handle( + &mut self, + _: Read, + reply: spawned_concurrency::reply::PendingReply<'_, usize>, + _: &Context, + ) { + let __reply_value: usize = { self.0 }; let _ = reply.reply(__reply_value); } } diff --git a/examples/exit_reason/src/main.rs b/examples/exit_reason/src/main.rs index 93214c5d..7418ac46 100644 --- a/examples/exit_reason/src/main.rs +++ b/examples/exit_reason/src/main.rs @@ -94,7 +94,12 @@ fn main() { } } impl Handler for PanicOnStart { - async fn handle(&mut self, _msg: Noop, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _ctx: &Context) { + async fn handle( + &mut self, + _msg: Noop, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { let _ = reply.reply(()); } } @@ -172,31 +177,44 @@ fn main() { } impl Actor for Observer {} impl Handler for Observer { - async fn handle(&mut self, msg: StartMonitor, reply: spawned_concurrency::reply::PendingReply<'_, MonitorRef>, ctx: &Context) { - let __reply_value: MonitorRef = { - ctx.monitor(&msg.0) - }; + async fn handle( + &mut self, + msg: StartMonitor, + reply: spawned_concurrency::reply::PendingReply<'_, MonitorRef>, + ctx: &Context, + ) { + let __reply_value: MonitorRef = { ctx.monitor(&msg.0) }; let _ = reply.reply(__reply_value); } } impl Handler for Observer { - async fn handle(&mut self, msg: StopMonitor, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + msg: StopMonitor, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.demonitor(msg.0); - } + ctx.demonitor(msg.0); + } let _ = reply.reply(()); } } impl Handler for Observer { - async fn handle(&mut self, msg: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _ctx: &Context) { + async fn handle( + &mut self, + msg: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { { - tracing::info!( - "[observer] received Down for {} ({})", - msg.monitor_ref, - msg.reason - ); - self.log.lock().unwrap().push(msg); - } + tracing::info!( + "[observer] received Down for {} ({})", + msg.monitor_ref, + msg.reason + ); + self.log.lock().unwrap().push(msg); + } let _ = reply.reply(()); } } @@ -270,10 +288,15 @@ fn main() { } } impl Handler for Supervisor { - async fn handle(&mut self, msg: LinkChild, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + msg: LinkChild, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { { - ctx.link(&msg.0); - } + ctx.link(&msg.0); + } let _ = reply.reply(()); } } diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 92aa6bb6..928bac70 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1296,10 +1296,7 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { .to_compile_error() .into(); } - if is_started - || is_stopped - || method.attrs.iter().any(is_handler_attribute) - { + if is_started || is_stopped || method.attrs.iter().any(is_handler_attribute) { return syn::Error::new_spanned(&method.sig, "#[exit_received] cannot be combined with other lifecycle or handler attributes").to_compile_error().into(); } if method.sig.inputs.len() != 3 @@ -1574,11 +1571,13 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { (true, true) => quote! { self.#method_name(msg, reply, ctx).await }, (true, false) => quote! { self.#method_name(msg, reply, ctx) }, (false, true) => { - let value = quote::quote_spanned! { ret_span => self.#method_name(msg, ctx).await }; + let value = + quote::quote_spanned! { ret_span => self.#method_name(msg, ctx).await }; quote! { let _ = reply.reply(#value); } } (false, false) => { - let value = quote::quote_spanned! { ret_span => self.#method_name(msg, ctx) }; + let value = + quote::quote_spanned! { ret_span => self.#method_name(msg, ctx) }; quote! { let _ = reply.reply(#value); } } }; diff --git a/rt/src/tasks/mod.rs b/rt/src/tasks/mod.rs index f302cf23..ca6e44e3 100644 --- a/rt/src/tasks/mod.rs +++ b/rt/src/tasks/mod.rs @@ -44,8 +44,6 @@ pub use crate::tasks::sim::{ pub use crate::tasks::tokio::mpsc; pub use crate::tasks::tokio::oneshot; #[cfg(not(spawned_sim))] -pub use crate::tasks::tokio::{sleep, Sleep}; -#[cfg(not(spawned_sim))] pub use crate::tasks::tokio::timeout; pub use crate::tasks::tokio::watch; pub use crate::tasks::tokio::CancellationToken; @@ -54,6 +52,8 @@ pub use crate::tasks::tokio::{ block_in_place, spawn, spawn_blocking, task_id, unconstrained, yield_now, Handle, JoinHandle, Runtime, RuntimeFlavor, }; +#[cfg(not(spawned_sim))] +pub use crate::tasks::tokio::{sleep, Sleep}; pub use crate::tasks::tokio::{BroadcastStream, ReceiverStream}; use std::future::Future; diff --git a/testing/src/faults.rs b/testing/src/faults.rs index a09ddd14..29cacda5 100644 --- a/testing/src/faults.rs +++ b/testing/src/faults.rs @@ -61,10 +61,15 @@ impl Message for Detonate { type Result = (); } impl threads::Handler for Fuse { - fn handle(&mut self, msg: Detonate, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context) { + fn handle( + &mut self, + msg: Detonate, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &threads::Context, + ) { { - panic!("{}", msg.0); - } + panic!("{}", msg.0); + } let _ = reply.reply(()); } } diff --git a/testing/src/tasks.rs b/testing/src/tasks.rs index 43a3f090..cfd3cd16 100644 --- a/testing/src/tasks.rs +++ b/testing/src/tasks.rs @@ -15,13 +15,13 @@ use crate::rules::{Fence, Plan, Rules}; use crate::shared::SETTLE_TIMEOUT; use spawned_concurrency::error::ActorError; use spawned_concurrency::message::Message; +use spawned_concurrency::reply::{Deadline, PendingReply, ReplyReceiver}; use spawned_concurrency::supervision::{ChildState, SupervisedChild, Supervisor, SupervisorReport}; use spawned_concurrency::system::{ActorSystem, ExecutionGroup, ShutdownReport, SystemError}; use spawned_concurrency::tasks::{ Actor, ActorRef, ActorStart, Context, Handler, Receiver, Recipient, }; use spawned_concurrency::{ChildHandle, ExitReason}; -use spawned_concurrency::reply::{Deadline, PendingReply, ReplyReceiver}; use spawned_rt::tasks::{sleep, timeout}; use std::future::Future; use std::ops::Deref; diff --git a/testing/src/threads.rs b/testing/src/threads.rs index aed2e565..70f77f11 100644 --- a/testing/src/threads.rs +++ b/testing/src/threads.rs @@ -6,11 +6,11 @@ use crate::rules::{Fence, Plan, Rules}; use crate::shared::SETTLE_TIMEOUT; use spawned_concurrency::error::ActorError; use spawned_concurrency::message::Message; +use spawned_concurrency::reply::{Deadline, PendingReply, ReplyReceiver}; use spawned_concurrency::threads::{ Actor, ActorRef, ActorStart, Context, Handler, Receiver, Recipient, }; use spawned_concurrency::{ChildHandle, ExitReason}; -use spawned_concurrency::reply::{Deadline, PendingReply, ReplyReceiver}; use std::ops::Deref; use std::panic::Location; use std::sync::Arc; diff --git a/testing/tests/tasks_kit.rs b/testing/tests/tasks_kit.rs index 569ebb75..bd49ea8c 100644 --- a/testing/tests/tasks_kit.rs +++ b/testing/tests/tasks_kit.rs @@ -225,18 +225,26 @@ impl Message for Get { } impl Actor for Counter {} impl Handler for Counter { - async fn handle(&mut self, _: Bump, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Bump, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0.fetch_add(1, Ordering::SeqCst); - } + self.0.fetch_add(1, Ordering::SeqCst); + } let _ = reply.reply(()); } } impl Handler for Counter { - async fn handle(&mut self, _: Get, reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context) { - let __reply_value: usize = { - self.0.load(Ordering::SeqCst) - }; + async fn handle( + &mut self, + _: Get, + reply: spawned_concurrency::reply::PendingReply<'_, usize>, + _: &Context, + ) { + let __reply_value: usize = { self.0.load(Ordering::SeqCst) }; let _ = reply.reply(__reply_value); } } @@ -336,12 +344,17 @@ impl Message for Hang { } impl Actor for Stuck {} impl Handler for Stuck { - async fn handle(&mut self, _: Hang, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + async fn handle( + &mut self, + _: Hang, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { { - self.0.store(true, Ordering::SeqCst); - // Blocking code is never preempted, so shutdown cannot finish in time. - std::thread::sleep(Duration::from_millis(500)); - } + self.0.store(true, Ordering::SeqCst); + // Blocking code is never preempted, so shutdown cannot finish in time. + std::thread::sleep(Duration::from_millis(500)); + } let _ = reply.reply(()); } } diff --git a/testing/tests/threads_kit.rs b/testing/tests/threads_kit.rs index 4a90ffc9..9e2c51e5 100644 --- a/testing/tests/threads_kit.rs +++ b/testing/tests/threads_kit.rs @@ -150,7 +150,12 @@ impl Message for Ping { type Result = (); } impl Handler for Idle { - fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { let _ = reply.reply(()); } } From 57aeaffb149e988dc9389f13a56d7c46621f435e Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:24:35 -0300 Subject: [PATCH 011/208] fix: adapt tests, benches and examples to ReplyReceiver --- concurrency/benches/baseline.rs | 28 +++++++---- concurrency/examples/panic_abort.rs | 10 ++-- concurrency/src/child_handle.rs | 6 +-- concurrency/src/link_regression_tests.rs | 3 +- concurrency/src/tasks/actor.rs | 47 ++++++++----------- concurrency/src/threads/actor.rs | 38 ++++++--------- concurrency/tests/cross_backend.rs | 9 +++- concurrency/tests/execution_profiles.rs | 3 +- concurrency/tests/leaks.rs | 3 +- concurrency/tests/native_mailbox_drop.rs | 8 ++-- concurrency/tests/nested_supervision.rs | 6 +-- concurrency/tests/process_failures.rs | 3 +- concurrency/tests/semantics_gaps.rs | 21 ++++----- concurrency/tests/startup_admission.rs | 3 +- concurrency/tests/supervision.rs | 18 +++---- concurrency/tests/supervision_strategies.rs | 9 ++-- concurrency/tests/supervisor_policies.rs | 3 +- concurrency/tests/support/failure_actor.rs | 36 ++++++++++---- .../src/bin/registrations_and_fakes.rs | 3 +- .../src/bin/supervision_recovery.rs | 3 +- .../capabilities/src/bin/supervision_tree.rs | 3 +- .../capabilities/src/bin/timeout_outcomes.rs | 17 ++++--- 22 files changed, 134 insertions(+), 146 deletions(-) diff --git a/concurrency/benches/baseline.rs b/concurrency/benches/baseline.rs index 05e70b76..3eaf94d4 100644 --- a/concurrency/benches/baseline.rs +++ b/concurrency/benches/baseline.rs @@ -1,7 +1,7 @@ //! Process-isolated baseline driver; methodology in benchmarks/README.md. #[path = "support/args.rs"] mod cli; -use spawned_concurrency::{message::Message, tasks, threads, ChildHandle}; +use spawned_concurrency::{message::Message, tasks, threads, ChildHandle, Deadline, ReplyReceiver}; use std::{ alloc::{GlobalAlloc, Layout, System}, hint::black_box, @@ -127,19 +127,28 @@ enum Target { Tokio(tokio::sync::mpsc::UnboundedSender<(Work, tokio::sync::oneshot::Sender)>), } enum Reply { - Tasks(tokio::sync::oneshot::Receiver), - Threads(mpsc::Receiver), + Tasks(ReplyReceiver), + Tokio(tokio::sync::oneshot::Receiver), + Threads(ReplyReceiver), } impl Target { fn request(&self, work: Work) -> Reply { match self { - Self::Tasks(actor) => Reply::Tasks(actor.request_raw(work).expect("actor stopped")), - Self::Threads(actor) => Reply::Threads(actor.request_raw(work).expect("actor stopped")), + Self::Tasks(actor) => Reply::Tasks( + actor + .request_raw_until(work, Deadline::after(Duration::from_secs(10))) + .expect("actor stopped"), + ), + Self::Threads(actor) => Reply::Threads( + actor + .request_raw_until(work, Deadline::after(Duration::from_secs(10))) + .expect("actor stopped"), + ), Self::Tokio(tx) => { let (reply, rx) = tokio::sync::oneshot::channel(); tx.send((work, reply)) .unwrap_or_else(|_| panic!("baseline stopped")); - Reply::Tasks(rx) + Reply::Tokio(rx) } } } @@ -147,15 +156,14 @@ impl Target { impl Reply { fn wait(self, rt: &tokio::runtime::Handle) -> Ack { match self { - Self::Tasks(rx) => rt.block_on(async { + Self::Tasks(rx) => rt.block_on(rx).expect("reply timeout or dropped"), + Self::Tokio(rx) => rt.block_on(async { tokio::time::timeout(Duration::from_secs(10), rx) .await .expect("reply timeout") .expect("reply dropped") }), - Self::Threads(rx) => rx - .recv_timeout(Duration::from_secs(10)) - .expect("reply timeout/disconnected"), + Self::Threads(rx) => rx.recv().expect("reply timeout/disconnected"), } } } diff --git a/concurrency/examples/panic_abort.rs b/concurrency/examples/panic_abort.rs index ac92f789..aacb7f55 100644 --- a/concurrency/examples/panic_abort.rs +++ b/concurrency/examples/panic_abort.rs @@ -46,11 +46,10 @@ mod native { fn handle( &mut self, _: Boom, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { panic!("{PANIC_TEXT}") }; - let _ = reply.reply(__reply_value); + let _reply_value: u64 = { panic!("{PANIC_TEXT}") }; } } pub fn handler_panic() { @@ -75,11 +74,10 @@ mod tasks { async fn handle( &mut self, _: Boom, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { panic!("{PANIC_TEXT}") }; - let _ = reply.reply(__reply_value); + let _reply_value: u64 = { panic!("{PANIC_TEXT}") }; } } pub fn handler_panic() { diff --git a/concurrency/src/child_handle.rs b/concurrency/src/child_handle.rs index fd4f99f7..addb110e 100644 --- a/concurrency/src/child_handle.rs +++ b/concurrency/src/child_handle.rs @@ -639,13 +639,12 @@ mod tests { fn handle( &mut self, _msg: Boom, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { { panic!("intentional panic in handler"); } - let _ = reply.reply(()); } } @@ -679,13 +678,12 @@ mod tests { async fn handle( &mut self, _msg: Boom, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { { panic!("intentional panic in handler"); } - let _ = reply.reply(()); } } diff --git a/concurrency/src/link_regression_tests.rs b/concurrency/src/link_regression_tests.rs index 6a203d7c..354f2ed5 100644 --- a/concurrency/src/link_regression_tests.rs +++ b/concurrency/src/link_regression_tests.rs @@ -223,13 +223,12 @@ impl crate::tasks::Handler for Idler { async fn handle( &mut self, _: Crash, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("linked failure during another actor's stopped hook"); } - let _ = reply.reply(()); } } struct Stopping { diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 3bc7a68b..8da166cc 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -9,6 +9,7 @@ use crate::message::Message; use crate::monitor::{Down, MonitorCleanup, MonitorRef, MonitorTable}; use crate::reply::{registry::Registry, reply_slot, SlotHandle}; #[doc(no_inline)] +#[allow(unused_imports)] // re-exported for `use spawned_concurrency::tasks::*` users pub use crate::reply::{ Deadline, DeferError, Delivery, PendingReply, ReplyReceiver, ReplyTo, DEFAULT_MAX_OUTSTANDING, }; @@ -2172,13 +2173,12 @@ mod tests { async fn handle( &mut self, _msg: Explode, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { { panic!("boom in handler"); } - let _ = reply.reply(()); } } impl Handler for PanicOnMsg { @@ -2324,13 +2324,12 @@ mod tests { async fn handle( &mut self, _msg: Explode, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { { panic!("boom in handler"); } - let _ = reply.reply(()); } } @@ -2616,13 +2615,12 @@ mod tests { async fn handle( &mut self, _msg: PanicMsg, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { { panic!("intentional panic"); } - let _ = reply.reply(()); } } @@ -2761,13 +2759,12 @@ mod tests { async fn handle( &mut self, _msg: Boom, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { { panic!("boom"); } - let _ = reply.reply(()); } } @@ -2797,13 +2794,12 @@ mod tests { async fn handle( &mut self, _msg: Boom2, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { { panic!("boom2"); } - let _ = reply.reply(()); } } @@ -2864,13 +2860,12 @@ mod tests { async fn handle( &mut self, _msg: Boom3, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { { panic!("boom3"); } - let _ = reply.reply(()); } } let target = make_trapper(false); @@ -2936,13 +2931,12 @@ mod tests { async fn handle( &mut self, _msg: Boom4, - reply: crate::reply::PendingReply<'_, u64>, + _reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context, ) { - let __reply_value: u64 = { + let _reply_value: u64 = { panic!("boom4"); }; - let _ = reply.reply(__reply_value); } } // Counter's Boom4 returns u64; just use send with no expectation of reply @@ -2977,13 +2971,12 @@ mod tests { async fn handle( &mut self, _msg: Boom5, - reply: crate::reply::PendingReply<'_, u64>, + _reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context, ) { - let __reply_value: u64 = { + let _reply_value: u64 = { panic!("boom5"); }; - let _ = reply.reply(__reply_value); } } let _ = a.send(Boom5); @@ -3028,13 +3021,12 @@ mod tests { async fn handle( &mut self, _msg: Boom6, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { { panic!("boom6"); } - let _ = reply.reply(()); } } let _ = a.send(Boom6); @@ -3069,13 +3061,12 @@ mod tests { async fn handle( &mut self, _msg: Boom7, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { { panic!("boom7"); } - let _ = reply.reply(()); } } @@ -3125,13 +3116,12 @@ mod tests { async fn handle( &mut self, _msg: PanicParent, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { { panic!("parent boom"); } - let _ = reply.reply(()); } } @@ -3336,6 +3326,7 @@ mod turn_regressions { linked_reason: new_linked_exit_reason(), mailbox: crate::instrumentation::Mailbox::default(), managed: None, + replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), }), }; let count = Arc::new(AtomicUsize::new(0)); @@ -3412,6 +3403,7 @@ mod idle_stop_regression { linked_reason: new_linked_exit_reason(), mailbox: crate::instrumentation::Mailbox::default(), managed: None, + replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), }), }; let count = Arc::new(AtomicUsize::new(0)); @@ -3426,10 +3418,10 @@ mod idle_stop_regression { // Start and park the actual actor loop, without allowing another // executor task to run it between admission and stop below. assert!(futures::poll!(actor.as_mut()).is_pending()); - let (reply, response) = oneshot::channel(); + let (slot, response) = reply_slot(crate::reply::Deadline::Never); tx.send(Box::new(MessageEnvelope { msg: Increment, - tx: Some(reply), + slot, #[cfg(feature = "metrics")] observation: None, }) as Box + Send>) @@ -3512,13 +3504,12 @@ mod api_coverage { async fn handle( &mut self, _: Boom, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("boom"); } - let _ = reply.reply(()); } } diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index f4ffda20..d20a3c4f 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -18,6 +18,7 @@ use crate::message::Message; use crate::monitor::{Down, MonitorCleanup, MonitorRef, MonitorTable}; use crate::reply::{registry::Registry, reply_slot, SlotHandle}; #[doc(no_inline)] +#[allow(unused_imports)] // re-exported for `use spawned_concurrency::threads::*` users pub use crate::reply::{ Deadline, DeferError, Delivery, PendingReply, ReplyReceiver, ReplyTo, DEFAULT_MAX_OUTSTANDING, }; @@ -1265,13 +1266,12 @@ mod tests { fn handle( &mut self, _msg: ExplodeThread, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { { panic!("boom in handler"); } - let _ = reply.reply(()); } } impl Handler for PanicOnMsg { @@ -1392,13 +1392,12 @@ mod tests { fn handle( &mut self, _msg: ExplodeThread2, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { { panic!("boom in handler"); } - let _ = reply.reply(()); } } @@ -1589,13 +1588,12 @@ mod tests { fn handle( &mut self, _msg: PanicMsg, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { { panic!("intentional panic"); } - let _ = reply.reply(()); } } @@ -1684,13 +1682,12 @@ mod tests { fn handle( &mut self, _msg: Boom, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { { panic!("boom"); } - let _ = reply.reply(()); } } @@ -1715,13 +1712,12 @@ mod tests { fn handle( &mut self, _msg: Boom, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { { panic!("boom"); } - let _ = reply.reply(()); } } @@ -1758,13 +1754,12 @@ mod tests { fn handle( &mut self, _msg: PanicParent, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { { panic!("parent boom"); } - let _ = reply.reply(()); } } @@ -1933,6 +1928,7 @@ mod idle_stop_regression { links: new_link_table(), linked_reason: new_linked_exit_reason(), mailbox: crate::instrumentation::Mailbox::default(), + replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), }; let count = Arc::new(AtomicUsize::new(0)); let seen = count.clone(); @@ -1960,10 +1956,10 @@ mod idle_stop_regression { ) }); entered.wait(); - let (reply, response) = oneshot::channel(); + let (slot, response) = reply_slot(crate::reply::Deadline::Never); tx.send(MailboxItem::Message(Box::new(MessageEnvelope { msg: Increment(events.clone(), monitors), - tx: Some(reply), + slot, #[cfg(feature = "metrics")] observation: None, }))) @@ -2054,6 +2050,7 @@ mod stop_wakeup_regression { links: new_link_table(), linked_reason: new_linked_exit_reason(), mailbox: crate::instrumentation::Mailbox::default(), + replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), }; let (entered_tx, entered) = mpsc::channel(); let (stopped_tx, stopped) = mpsc::channel(); @@ -2088,13 +2085,12 @@ mod stop_wakeup_regression { fn handle( &mut self, _: Crash, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("linked wakeup regression"); } - let _ = reply.reply(()); } } @@ -2189,12 +2185,9 @@ mod stop_wakeup_regression { } }); release.send(()).unwrap(); - first.recv_timeout(Duration::from_secs(2)).unwrap(); + first.recv().unwrap(); for reply in replies { - assert!(matches!( - reply.recv_timeout(Duration::from_secs(2)), - Err(RecvTimeoutError::Disconnected) - )); + assert!(matches!(reply.recv(), Err(ActorError::ActorStopped))); } assert_eq!(actor.wait_exit(), ExitReason::Normal); assert_eq!(handled.load(Ordering::SeqCst), 1); @@ -2272,13 +2265,12 @@ mod api_coverage { fn handle( &mut self, _: Boom, - reply: crate::reply::PendingReply<'_, ()>, + _reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("boom"); } - let _ = reply.reply(()); } } diff --git a/concurrency/tests/cross_backend.rs b/concurrency/tests/cross_backend.rs index 2b2e13ae..70448e8f 100644 --- a/concurrency/tests/cross_backend.rs +++ b/concurrency/tests/cross_backend.rs @@ -335,11 +335,16 @@ mod threads_node { .peers .iter() .enumerate() - .map(|(i, peer)| peer.request_raw(Cmd::Echo(v + i as u64))) + .map(|(i, peer)| { + peer.request_raw_until( + Cmd::Echo(v + i as u64), + spawned_concurrency::Deadline::after(LIMIT), + ) + }) .collect(); let mut values = Vec::new(); for rx in pending { - match rx.map(|rx| rx.recv_timeout(LIMIT)) { + match rx.map(|rx| rx.recv()) { Ok(Ok(Reply::Value(v))) => values.push(v), _ => return Reply::Failed, } diff --git a/concurrency/tests/execution_profiles.rs b/concurrency/tests/execution_profiles.rs index 93db6c79..6931ab12 100644 --- a/concurrency/tests/execution_profiles.rs +++ b/concurrency/tests/execution_profiles.rs @@ -117,13 +117,12 @@ impl Handler for Probe { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("profile recovery injection"); } - let _ = reply.reply(()); } } // Not simulated: native execution groups need OS threads. diff --git a/concurrency/tests/leaks.rs b/concurrency/tests/leaks.rs index 736d2b42..fc1bca91 100644 --- a/concurrency/tests/leaks.rs +++ b/concurrency/tests/leaks.rs @@ -239,13 +239,12 @@ mod async_leaks { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("injected failure for leak test"); } - let _ = reply.reply(()); } } impl Handler for Probe { diff --git a/concurrency/tests/native_mailbox_drop.rs b/concurrency/tests/native_mailbox_drop.rs index 8bec67b6..26fba587 100644 --- a/concurrency/tests/native_mailbox_drop.rs +++ b/concurrency/tests/native_mailbox_drop.rs @@ -43,13 +43,12 @@ impl Handler for Victim { fn handle( &mut self, _: Bomb, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("queued handler ran"); } - let _ = reply.reply(()); } } @@ -61,13 +60,12 @@ impl Handler for Victim { fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("native handler failure"); } - let _ = reply.reply(()); } } @@ -127,7 +125,7 @@ fn native_drop_child() { ); assert_eq!(dropped.load(Ordering::SeqCst), 2); for reply in replies { - assert!(reply.recv_timeout(LIMIT).is_err()); + assert!(reply.recv().is_err()); } // Keep ActorRef alive through all assertions: cleanup cannot depend // on whoever eventually drops its last sender. diff --git a/concurrency/tests/nested_supervision.rs b/concurrency/tests/nested_supervision.rs index ffeb2c74..ef4e75e0 100644 --- a/concurrency/tests/nested_supervision.rs +++ b/concurrency/tests/nested_supervision.rs @@ -61,13 +61,12 @@ impl Handler for Counter { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("nested leaf failed"); } - let _ = reply.reply(()); } } type Published = Arc>>>; @@ -536,13 +535,12 @@ impl Handler for MaybeStop { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("nested leaf failed"); } - let _ = reply.reply(()); } } diff --git a/concurrency/tests/process_failures.rs b/concurrency/tests/process_failures.rs index 169ff7ed..d1ff53c9 100644 --- a/concurrency/tests/process_failures.rs +++ b/concurrency/tests/process_failures.rs @@ -33,7 +33,7 @@ impl Handler for Fixture { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { @@ -49,7 +49,6 @@ impl Handler for Fixture { std::thread::park(); } } - let _ = reply.reply(()); } } #[test] diff --git a/concurrency/tests/semantics_gaps.rs b/concurrency/tests/semantics_gaps.rs index b805ef4b..8c79d330 100644 --- a/concurrency/tests/semantics_gaps.rs +++ b/concurrency/tests/semantics_gaps.rs @@ -198,11 +198,10 @@ mod native { fn handle( &mut self, _: Boom, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { panic!("request handler panic") }; - let _ = reply.reply(__reply_value); + let _reply_value: u64 = { panic!("request handler panic") }; } } @@ -225,10 +224,10 @@ mod native { ); let raw = Panicker.start(); - let receiver = raw.request_raw(Boom).unwrap(); - assert!(receiver - .recv_timeout(LIMIT) - .is_err_and(|e| { e == std::sync::mpsc::RecvTimeoutError::Disconnected })); + let receiver = raw + .request_raw_until(Boom, spawned_concurrency::Deadline::after(LIMIT)) + .unwrap(); + assert!(matches!(receiver.recv(), Err(ActorError::ActorStopped))); } } @@ -478,13 +477,12 @@ mod tasks { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("injected crash"); } - let _ = reply.reply(()); } } @@ -562,11 +560,10 @@ mod tasks { async fn handle( &mut self, _: Boom, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { panic!("request handler panic") }; - let _ = reply.reply(__reply_value); + let _reply_value: u64 = { panic!("request handler panic") }; } } diff --git a/concurrency/tests/startup_admission.rs b/concurrency/tests/startup_admission.rs index d01bde7e..9723e0e9 100644 --- a/concurrency/tests/startup_admission.rs +++ b/concurrency/tests/startup_admission.rs @@ -36,13 +36,12 @@ impl spawned_concurrency::tasks::Handler for Probe { async fn handle( &mut self, _: Queued, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("unstarted handler ran"); } - let _ = reply.reply(()); } } impl Actor for Probe { diff --git a/concurrency/tests/supervision.rs b/concurrency/tests/supervision.rs index e2009ebc..3cf93828 100644 --- a/concurrency/tests/supervision.rs +++ b/concurrency/tests/supervision.rs @@ -57,13 +57,12 @@ impl Handler for Counter { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("injected child failure"); } - let _ = reply.reply(()); } } @@ -294,13 +293,12 @@ impl Handler for SlowHelper { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("injected failure with active helper"); } - let _ = reply.reply(()); } } // Not simulated: a helper blocks an OS worker on purpose. @@ -369,13 +367,12 @@ impl Handler for SlowStop { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("failure before stuck stopped hook"); } - let _ = reply.reply(()); } } // Not simulated: a helper blocks an OS worker on purpose. @@ -562,13 +559,12 @@ impl Handler for RaceActor { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("racing child failure"); } - let _ = reply.reply(()); } } impl Handler for RaceActor { @@ -868,13 +864,12 @@ impl Handler for ExitOnceDuringRestart { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("injected child failure"); } - let _ = reply.reply(()); } } @@ -1345,13 +1340,12 @@ impl Handler for Named { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("crash a registered child"); } - let _ = reply.reply(()); } } diff --git a/concurrency/tests/supervision_strategies.rs b/concurrency/tests/supervision_strategies.rs index e25a0077..570288ce 100644 --- a/concurrency/tests/supervision_strategies.rs +++ b/concurrency/tests/supervision_strategies.rs @@ -87,27 +87,25 @@ impl Handler for Probe { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("injected cohort failure"); } - let _ = reply.reply(()); } } impl Handler for Probe { async fn handle( &mut self, msg: CrashTogether, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { msg.0.wait().await; panic!("simultaneous cohort failure"); } - let _ = reply.reply(()); } } @@ -732,12 +730,11 @@ impl Handler for MaybeStop { async fn handle( &mut self, _: CrashPlain, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("injected cohort failure"); } - let _ = reply.reply(()); } } diff --git a/concurrency/tests/supervisor_policies.rs b/concurrency/tests/supervisor_policies.rs index b31f5a3e..819ddbc4 100644 --- a/concurrency/tests/supervisor_policies.rs +++ b/concurrency/tests/supervisor_policies.rs @@ -39,13 +39,12 @@ impl Handler for Probe { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("generated supervisor failure"); } - let _ = reply.reply(()); } } #[derive(Clone, Debug)] diff --git a/concurrency/tests/support/failure_actor.rs b/concurrency/tests/support/failure_actor.rs index 58b343d7..8e414340 100644 --- a/concurrency/tests/support/failure_actor.rs +++ b/concurrency/tests/support/failure_actor.rs @@ -278,10 +278,12 @@ pub enum Handle { Async(spawned_concurrency::tasks::ActorRef), Native(spawned_concurrency::threads::ActorRef), } +/// A raw request sent without a deadline; [`Response::wait`] is the caller +/// giving up after `duration` (dropping the receiver abandons the request). pub enum Response { #[cfg(feature = "async")] - Async(tokio::sync::oneshot::Receiver), - Native(mpsc::Receiver), + Async(spawned_concurrency::ReplyReceiver), + Native(spawned_concurrency::ReplyReceiver), } impl Response { pub fn wait( @@ -291,13 +293,23 @@ impl Response { ) -> Result { match self { #[cfg(feature = "async")] - Self::Async(receiver) => _rt.block_on( - spawned_concurrency::Response::from_with_timeout(Ok(receiver), duration), - ), - Self::Native(receiver) => receiver.recv_timeout(duration).map_err(|e| match e { - mpsc::RecvTimeoutError::Timeout => ActorError::RequestTimeout, - mpsc::RecvTimeoutError::Disconnected => ActorError::ActorStopped, + Self::Async(receiver) => _rt.block_on(async { + tokio::time::timeout(duration, receiver) + .await + .unwrap_or(Err(ActorError::RequestTimeout)) }), + Self::Native(mut receiver) => { + let give_up = std::time::Instant::now() + duration; + loop { + if let Some(result) = receiver.try_recv() { + return result; + } + if std::time::Instant::now() >= give_up { + return Err(ActorError::RequestTimeout); + } + std::thread::sleep(Duration::from_millis(1)); + } + } } } } @@ -327,8 +339,12 @@ impl Handle { pub fn raw(&self, command: Command) -> Result { match self { #[cfg(feature = "async")] - Self::Async(actor) => actor.request_raw(command).map(Response::Async), - Self::Native(actor) => actor.request_raw(command).map(Response::Native), + Self::Async(actor) => actor + .request_raw_until(command, spawned_concurrency::Deadline::Never) + .map(Response::Async), + Self::Native(actor) => actor + .request_raw_until(command, spawned_concurrency::Deadline::Never) + .map(Response::Native), } } pub fn request( diff --git a/examples/capabilities/src/bin/registrations_and_fakes.rs b/examples/capabilities/src/bin/registrations_and_fakes.rs index 764d1ba5..35466802 100644 --- a/examples/capabilities/src/bin/registrations_and_fakes.rs +++ b/examples/capabilities/src/bin/registrations_and_fakes.rs @@ -40,13 +40,12 @@ impl Handler for Service { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("intentional named-service replacement"); } - let _ = reply.reply(()); } } #[tokio::main(flavor = "current_thread")] diff --git a/examples/capabilities/src/bin/supervision_recovery.rs b/examples/capabilities/src/bin/supervision_recovery.rs index 3efc6fb7..ed73fad3 100644 --- a/examples/capabilities/src/bin/supervision_recovery.rs +++ b/examples/capabilities/src/bin/supervision_recovery.rs @@ -34,13 +34,12 @@ impl Handler for Counter { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("intentional recovery example failure"); } - let _ = reply.reply(()); } } #[tokio::main(flavor = "current_thread")] diff --git a/examples/capabilities/src/bin/supervision_tree.rs b/examples/capabilities/src/bin/supervision_tree.rs index 758a1cfe..d098e3b5 100644 --- a/examples/capabilities/src/bin/supervision_tree.rs +++ b/examples/capabilities/src/bin/supervision_tree.rs @@ -22,13 +22,12 @@ impl Handler for Worker { async fn handle( &mut self, _: Fail, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { { panic!("example failure"); } - let _ = reply.reply(()); } } #[tokio::main(flavor = "multi_thread", worker_threads = 2)] diff --git a/examples/capabilities/src/bin/timeout_outcomes.rs b/examples/capabilities/src/bin/timeout_outcomes.rs index 4e85ee2e..a36932ae 100644 --- a/examples/capabilities/src/bin/timeout_outcomes.rs +++ b/examples/capabilities/src/bin/timeout_outcomes.rs @@ -2,7 +2,7 @@ use spawned_concurrency::{ message::Message, tasks::{Actor, ActorStart, Context, Handler}, - ActorError, Response, + ActorError, Deadline, Response, }; use std::time::Duration; use tokio::sync::oneshot; @@ -52,13 +52,18 @@ async fn main() { let counter = Counter(0).start(); let (entered_tx, entered_rx) = oneshot::channel(); let (release_tx, release_rx) = oneshot::channel(); - let reply = counter.request_raw(Apply { - entered: entered_tx, - release: release_rx, - }); + // The deadline is fixed at send (OTP's call/3): already due here. An + // expired request is still dispatched, as in OTP; its late reply is dropped. + let reply = counter.request_raw_until( + Apply { + entered: entered_tx, + release: release_rx, + }, + Deadline::after(Duration::ZERO), + ); entered_rx.await.unwrap(); // A gate keeps the reply pending; no race-dependent sleep is needed. - let result = Response::from_with_timeout(reply, Duration::ZERO).await; + let result = Response::from(reply).await; assert!(matches!(result, Err(ActorError::RequestTimeout))); release_tx.send(()).unwrap(); assert_eq!(counter.request(Read).await.unwrap(), 1); From f859b670f447166620d4ef0647ead70182fc88b6 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:24:59 -0300 Subject: [PATCH 012/208] fix: unreachable replies after panics in migrated handlers --- concurrency/tests/canary_feedback.rs | 7 ++----- testing/src/faults.rs | 7 ++----- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/concurrency/tests/canary_feedback.rs b/concurrency/tests/canary_feedback.rs index 32dbdfc2..c40d8f9a 100644 --- a/concurrency/tests/canary_feedback.rs +++ b/concurrency/tests/canary_feedback.rs @@ -413,13 +413,10 @@ async fn panicked_turn_is_cleared_before_stopped_hook() { async fn handle( &mut self, _: Quit, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { - panic!("diagnostic turn cleanup regression"); - } - let _ = reply.reply(()); + panic!("diagnostic turn cleanup regression"); } } let system = ActorSystem::new().unwrap(); diff --git a/testing/src/faults.rs b/testing/src/faults.rs index 29cacda5..762389d8 100644 --- a/testing/src/faults.rs +++ b/testing/src/faults.rs @@ -64,13 +64,10 @@ impl threads::Handler for Fuse { fn handle( &mut self, msg: Detonate, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context, ) { - { - panic!("{}", msg.0); - } - let _ = reply.reply(()); + panic!("{}", msg.0); } } From 34d3a26fb1efb937adc30ea544b7e7865c98eaff Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:27:44 -0300 Subject: [PATCH 013/208] perf: do not keep the request Result alive across the reply await --- concurrency/src/tasks/actor.rs | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 8da166cc..ee74eff2 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -412,10 +412,18 @@ impl Context { M: Message, { let observation = crate::instrumentation::RequestObservation::new(); - let result = match self.request_raw_until(msg, Deadline::after(duration)) { - Err(error) => Err(error), - Ok(rx) => rx.await, + // Bind the receiver before awaiting it: awaiting inside a `match` on + // the `Result` keeps the scrutinee's storage alive beside the awaited + // receiver and doubles this future's size. + let rx = match self.request_raw_until(msg, Deadline::after(duration)) { + Ok(rx) => rx, + Err(error) => { + let result = Err(error); + observation.finish(&result); + return result; + } }; + let result = rx.await; observation.finish(&result); result } @@ -861,10 +869,18 @@ impl ActorRef { M: Message, { let observation = crate::instrumentation::RequestObservation::new(); - let result = match self.request_raw_until(msg, Deadline::after(duration)) { - Err(error) => Err(error), - Ok(rx) => rx.await, + // Bind the receiver before awaiting it: awaiting inside a `match` on + // the `Result` keeps the scrutinee's storage alive beside the awaited + // receiver and doubles this future's size. + let rx = match self.request_raw_until(msg, Deadline::after(duration)) { + Ok(rx) => rx, + Err(error) => { + let result = Err(error); + observation.finish(&result); + return result; + } }; + let result = rx.await; observation.finish(&result); result } From 84f3f98fced769cd99ff7558c249a16d6d7e9fb7 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:28:23 -0300 Subject: [PATCH 014/208] test(ui): deferred handler pass case and misuse diagnostics --- concurrency/src/tasks/actor.rs | 12 +++- .../tests/ui-pass/deferred_handlers.rs | 69 +++++++++++++++++++ .../tests/ui/deferred_handler_return_type.rs | 17 +++++ .../ui/deferred_handler_without_reply.rs | 15 ++++ concurrency/tests/ui/pending_reply_stored.rs | 21 ++++++ concurrency/tests/ui/reply_to_not_clone.rs | 6 ++ 6 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 concurrency/tests/ui-pass/deferred_handlers.rs create mode 100644 concurrency/tests/ui/deferred_handler_return_type.rs create mode 100644 concurrency/tests/ui/deferred_handler_without_reply.rs create mode 100644 concurrency/tests/ui/pending_reply_stored.rs create mode 100644 concurrency/tests/ui/reply_to_not_clone.rs diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index ee74eff2..6a288041 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -751,10 +751,16 @@ pub async fn request( ) -> Result { let observation = crate::instrumentation::RequestObservation::new(); // The deadline is computed before enqueueing (send time, as OTP's call). - let result = match recipient.request_raw_until(msg, Deadline::after(timeout_duration)) { - Err(error) => Err(error), - Ok(rx) => rx.await, + // Bind the receiver before awaiting it (see `ActorRef::request_with_timeout`). + let rx = match recipient.request_raw_until(msg, Deadline::after(timeout_duration)) { + Ok(rx) => rx, + Err(error) => { + let result = Err(error); + observation.finish(&result); + return result; + } }; + let result = rx.await; observation.finish(&result); result } diff --git a/concurrency/tests/ui-pass/deferred_handlers.rs b/concurrency/tests/ui-pass/deferred_handlers.rs new file mode 100644 index 00000000..1888164f --- /dev/null +++ b/concurrency/tests/ui-pass/deferred_handlers.rs @@ -0,0 +1,69 @@ +// `#[deferred_handler]` in both modes: answer in place, or defer and answer +// from a later turn; a `ReplyTo` is storable and `Send + 'static`. +use spawned_concurrency::{actor, protocol, DeferError, PendingReply, ReplyTo, Response}; + +#[protocol] +pub trait WaitProtocol: Send + Sync { + fn wait(&self) -> Response; + fn release(&self, value: u64); +} +use wait_protocol::{Release, Wait}; + +mod tasks_mode { + use super::*; + use spawned_concurrency::tasks::{Actor, Context, Handler}; + pub struct Gate { + waiters: Vec>, + ready: Option, + } + #[actor(protocol = WaitProtocol)] + impl Gate { + #[deferred_handler] + async fn wait(&mut self, _msg: Wait, reply: PendingReply<'_, u64>, _ctx: &Context) { + if let Some(value) = self.ready { + reply.reply(value); + return; + } + match reply.defer() { + Ok(token) => self.waiters.push(token), + Err(DeferError::Overloaded | DeferError::Closed | DeferError::NoCaller) => {} + } + } + #[send_handler] + async fn release(&mut self, msg: Release, _ctx: &Context) { + self.ready = Some(msg.value); + for token in self.waiters.drain(..) { + token.reply(msg.value); + } + } + } +} + +mod threads_mode { + use super::*; + use spawned_concurrency::threads::{Actor, Context, Handler}; + pub struct Gate { + waiters: Vec>, + } + #[actor(protocol = WaitProtocol)] + impl Gate { + #[deferred_handler] + fn wait(&mut self, _msg: Wait, reply: PendingReply<'_, u64>, _ctx: &Context) { + if let Ok(token) = reply.defer() { + self.waiters.push(token); + } + } + #[send_handler] + fn release(&mut self, msg: Release, _ctx: &Context) { + for token in self.waiters.drain(..) { + token.reply(msg.value); + } + } + } +} + +fn assert_storable() {} + +fn main() { + assert_storable::>(); +} diff --git a/concurrency/tests/ui/deferred_handler_return_type.rs b/concurrency/tests/ui/deferred_handler_return_type.rs new file mode 100644 index 00000000..6d00947e --- /dev/null +++ b/concurrency/tests/ui/deferred_handler_return_type.rs @@ -0,0 +1,17 @@ +#![allow(unused_imports)] +use spawned_concurrency::{actor, protocol, PendingReply, Response}; +#[protocol] +pub trait CounterProtocol: Send + Sync { + fn read(&self) -> Response; +} +use counter_protocol::Read; +pub struct Counter(u64); +use spawned_concurrency::threads::{Actor, Context, Handler}; +#[actor] +impl Counter { + #[deferred_handler] + fn read(&mut self, _msg: Read, reply: PendingReply<'_, u64>, _ctx: &Context) -> u64 { + self.0 + } +} +fn main() {} diff --git a/concurrency/tests/ui/deferred_handler_without_reply.rs b/concurrency/tests/ui/deferred_handler_without_reply.rs new file mode 100644 index 00000000..b5f2a618 --- /dev/null +++ b/concurrency/tests/ui/deferred_handler_without_reply.rs @@ -0,0 +1,15 @@ +#![allow(unused_imports)] +use spawned_concurrency::{actor, protocol, Response}; +#[protocol] +pub trait CounterProtocol: Send + Sync { + fn read(&self) -> Response; +} +use counter_protocol::Read; +pub struct Counter(u64); +use spawned_concurrency::threads::{Actor, Context, Handler}; +#[actor] +impl Counter { + #[deferred_handler] + fn read(&mut self, _msg: Read, _ctx: &Context) {} +} +fn main() {} diff --git a/concurrency/tests/ui/pending_reply_stored.rs b/concurrency/tests/ui/pending_reply_stored.rs new file mode 100644 index 00000000..253a661e --- /dev/null +++ b/concurrency/tests/ui/pending_reply_stored.rs @@ -0,0 +1,21 @@ +// A `PendingReply` is turn-scoped: storing it without `defer()` (which takes +// a credit) does not compile. +#![allow(unused_imports)] +use spawned_concurrency::{actor, protocol, PendingReply, Response}; +#[protocol] +pub trait CounterProtocol: Send + Sync { + fn read(&self) -> Response; +} +use counter_protocol::Read; +pub struct Counter { + held: Vec>, +} +use spawned_concurrency::threads::{Actor, Context, Handler}; +#[actor] +impl Counter { + #[deferred_handler] + fn read(&mut self, _msg: Read, reply: PendingReply<'_, u64>, _ctx: &Context) { + self.held.push(reply); + } +} +fn main() {} diff --git a/concurrency/tests/ui/reply_to_not_clone.rs b/concurrency/tests/ui/reply_to_not_clone.rs new file mode 100644 index 00000000..3bc994b4 --- /dev/null +++ b/concurrency/tests/ui/reply_to_not_clone.rs @@ -0,0 +1,6 @@ +// A `ReplyTo` answers exactly once: it is not `Clone`. +use spawned_concurrency::ReplyTo; +fn duplicate(token: &ReplyTo) -> ReplyTo { + token.clone() +} +fn main() {} From 89959c80c35a939e7b2d8fa6421184c4c563f642 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:29:32 -0300 Subject: [PATCH 015/208] fix(macros): handlers returning impl Future are awaited before replying --- macros/src/lib.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 928bac70..a187a6e7 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1566,7 +1566,13 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { ReturnType::Default => method.sig.paren_token.span.join(), ReturnType::Type(_, ty) => ty.span(), }; - let asynchronous = method.sig.asyncness.is_some(); + // A tasks handler may also be a plain `fn` returning + // `impl Future`; its result is awaited like an `async fn`'s. + let returns_future = matches!( + &method.sig.output, + ReturnType::Type(_, ty) if matches!(**ty, Type::ImplTrait(_)) + ); + let asynchronous = method.sig.asyncness.is_some() || returns_future; let call = match (deferred, asynchronous) { (true, true) => quote! { self.#method_name(msg, reply, ctx).await }, (true, false) => quote! { self.#method_name(msg, reply, ctx) }, From a41df960f5c95ae3dbb195aafa8a189ffd788ad9 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:30:35 -0300 Subject: [PATCH 016/208] test(ui): regenerate macro diagnostics for the new Handler signature --- .../ui/actor_async_handler_threads.stderr | 6 ++--- .../ui/actor_handler_ctx_wrong_type.stderr | 8 +++---- .../ui/actor_handler_missing_return.stderr | 23 ++++++++++++------- .../tests/ui/actor_handler_not_message.stderr | 21 +++++++++++++++++ .../ui/actor_handler_wrong_return.stderr | 15 ++++++------ .../tests/ui/actor_sync_handler_tasks.stderr | 12 +++++----- .../ui/actor_two_handler_attributes.stderr | 2 +- .../ui/deferred_handler_return_type.stderr | 5 ++++ .../ui/deferred_handler_without_reply.stderr | 5 ++++ .../tests/ui/pending_reply_stored.stderr | 17 ++++++++++++++ .../tests/ui/reply_to_not_clone.stderr | 15 ++++++++++++ 11 files changed, 100 insertions(+), 29 deletions(-) create mode 100644 concurrency/tests/ui/deferred_handler_return_type.stderr create mode 100644 concurrency/tests/ui/deferred_handler_without_reply.stderr create mode 100644 concurrency/tests/ui/pending_reply_stored.stderr create mode 100644 concurrency/tests/ui/reply_to_not_clone.stderr diff --git a/concurrency/tests/ui/actor_async_handler_threads.stderr b/concurrency/tests/ui/actor_async_handler_threads.stderr index e85a6f5d..b4f66c25 100644 --- a/concurrency/tests/ui/actor_async_handler_threads.stderr +++ b/concurrency/tests/ui/actor_async_handler_threads.stderr @@ -2,7 +2,7 @@ error[E0053]: method `handle` has an incompatible type for trait --> tests/ui/actor_async_handler_threads.rs:14:5 | 14 | async fn read(&mut self, _msg: Read, _ctx: &Context) -> u64 { self.0 } - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `u64`, found future + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `()`, found future | - = note: expected signature `fn(&mut Counter, counter_protocol::Read, &spawned_concurrency::threads::Context) -> u64` - found signature `fn(&mut Counter, counter_protocol::Read, &spawned_concurrency::threads::Context) -> impl Future` + = note: expected signature `fn(&mut Counter, counter_protocol::Read, PendingReply<'_, u64>, &spawned_concurrency::threads::Context) -> ()` + found signature `fn(&mut Counter, counter_protocol::Read, PendingReply<'_, u64>, &spawned_concurrency::threads::Context) -> impl Future` diff --git a/concurrency/tests/ui/actor_handler_ctx_wrong_type.stderr b/concurrency/tests/ui/actor_handler_ctx_wrong_type.stderr index 15671edd..fdabe96e 100644 --- a/concurrency/tests/ui/actor_handler_ctx_wrong_type.stderr +++ b/concurrency/tests/ui/actor_handler_ctx_wrong_type.stderr @@ -1,10 +1,10 @@ error[E0308]: mismatched types - --> tests/ui/actor_handler_ctx_wrong_type.rs:14:5 + --> tests/ui/actor_handler_ctx_wrong_type.rs:14:12 | 14 | fn bump(&mut self, msg: Bump, _ctx: &u64) { self.0 += msg.by; } - | ^^ ---- arguments to this method are incorrect - | | - | expected `&u64`, found `&Context` + | ----^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `&u64`, found `&Context` + | | + | arguments to this method are incorrect | = note: expected reference `&u64` found reference `&spawned_concurrency::threads::Context` diff --git a/concurrency/tests/ui/actor_handler_missing_return.stderr b/concurrency/tests/ui/actor_handler_missing_return.stderr index 9a69cdff..c535dadd 100644 --- a/concurrency/tests/ui/actor_handler_missing_return.stderr +++ b/concurrency/tests/ui/actor_handler_missing_return.stderr @@ -1,12 +1,19 @@ -error[E0053]: method `handle` has an incompatible type for trait - --> tests/ui/actor_handler_missing_return.rs:14:57 +error[E0308]: mismatched types + --> tests/ui/actor_handler_missing_return.rs:14:12 | +11 | #[actor] + | -------- arguments to this method are incorrect +... 14 | fn read(&mut self, _msg: Read, _ctx: &Context) {} - | ^ expected `u64`, found `()` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `u64`, found `()` | - = note: expected signature `fn(&mut Counter, counter_protocol::Read, &spawned_concurrency::threads::Context) -> u64` - found signature `fn(&mut Counter, counter_protocol::Read, &spawned_concurrency::threads::Context) -> ()` -help: change the output type to match the trait +note: method `read` modifies its receiver in-place + --> tests/ui/actor_handler_missing_return.rs:14:8 | -14 | fn read(&mut self, _msg: Read, _ctx: &Context) -> u64 {} - | ++++++ +14 | fn read(&mut self, _msg: Read, _ctx: &Context) {} + | ^^^^ this call modifies `self` in-place +note: method defined here + --> src/reply/mod.rs + | + | pub fn reply(mut self, value: R) -> Delivery { + | ^^^^^ diff --git a/concurrency/tests/ui/actor_handler_not_message.stderr b/concurrency/tests/ui/actor_handler_not_message.stderr index 234704c7..51986569 100644 --- a/concurrency/tests/ui/actor_handler_not_message.stderr +++ b/concurrency/tests/ui/actor_handler_not_message.stderr @@ -44,3 +44,24 @@ note: required by a bound in `spawned_concurrency::threads::Handler` | pub trait Handler: Actor { | ^^^^^^^ required by this bound in `Handler` = note: this error originates in the attribute macro `protocol` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: the trait bound `u64: Message` is not satisfied + --> tests/ui/actor_handler_not_message.rs:11:1 + | +11 | #[actor] + | ^^^^^^^^ the trait `Message` is not implemented for `u64` + | +help: the following other types implement trait `Message` + --> tests/ui/actor_handler_not_message.rs:3:1 + | + 3 | #[protocol] + | ^^^^^^^^^^^ + | | + | `Bump` + | `counter_protocol::Read` + | + ::: src/monitor.rs + | + | impl Message for Down { + | ^^^^^^^^^^^^^^^^^^^^^ `Down` + = note: this error originates in the attribute macro `actor` which comes from the expansion of the attribute macro `protocol` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/concurrency/tests/ui/actor_handler_wrong_return.stderr b/concurrency/tests/ui/actor_handler_wrong_return.stderr index 3a7b60ff..51c1a7ac 100644 --- a/concurrency/tests/ui/actor_handler_wrong_return.stderr +++ b/concurrency/tests/ui/actor_handler_wrong_return.stderr @@ -1,13 +1,14 @@ -error[E0053]: method `handle` has an incompatible type for trait +error[E0308]: mismatched types --> tests/ui/actor_handler_wrong_return.rs:14:61 | +11 | #[actor] + | -------- arguments to this method are incorrect +... 14 | fn read(&mut self, _msg: Read, _ctx: &Context) -> String { self.0.to_string() } | ^^^^^^ expected `u64`, found `String` | - = note: expected signature `fn(&mut Counter, counter_protocol::Read, &spawned_concurrency::threads::Context) -> u64` - found signature `fn(&mut Counter, counter_protocol::Read, &spawned_concurrency::threads::Context) -> String` -help: change the output type to match the trait - | -14 - fn read(&mut self, _msg: Read, _ctx: &Context) -> String { self.0.to_string() } -14 + fn read(&mut self, _msg: Read, _ctx: &Context) -> u64 { self.0.to_string() } +note: method defined here + --> src/reply/mod.rs | + | pub fn reply(mut self, value: R) -> Delivery { + | ^^^^^ diff --git a/concurrency/tests/ui/actor_sync_handler_tasks.stderr b/concurrency/tests/ui/actor_sync_handler_tasks.stderr index f77b3e28..08b2ced9 100644 --- a/concurrency/tests/ui/actor_sync_handler_tasks.stderr +++ b/concurrency/tests/ui/actor_sync_handler_tasks.stderr @@ -1,12 +1,12 @@ -error[E0277]: `u64` is not a future - --> tests/ui/actor_sync_handler_tasks.rs:14:61 +error[E0277]: `()` is not a future + --> tests/ui/actor_sync_handler_tasks.rs:14:57 | 14 | fn read(&mut self, _msg: Read, _ctx: &Context) -> u64 { self.0 } - | ^^^ `u64` is not a future + | ^ `()` is not a future | - = help: the trait `Future` is not implemented for `u64` + = help: the trait `Future` is not implemented for `()` note: required by a bound in `spawned_concurrency::tasks::Handler::handle::{anon_assoc#0}` --> src/tasks/actor.rs | - | fn handle(&mut self, msg: M, ctx: &Context) -> impl Future + Send; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `Handler::handle::{anon_assoc#0}` + | ) -> impl Future + Send; + | ^^^^^^^^^^^^^^^^^^^ required by this bound in `Handler::handle::{anon_assoc#0}` diff --git a/concurrency/tests/ui/actor_two_handler_attributes.stderr b/concurrency/tests/ui/actor_two_handler_attributes.stderr index 972dc80c..3bc34613 100644 --- a/concurrency/tests/ui/actor_two_handler_attributes.stderr +++ b/concurrency/tests/ui/actor_two_handler_attributes.stderr @@ -1,4 +1,4 @@ -error: a method can carry only one of #[handler], #[send_handler] and #[request_handler] +error: a method can carry only one of #[handler], #[send_handler], #[request_handler] and #[deferred_handler] --> tests/ui/actor_two_handler_attributes.rs:14:5 | 14 | #[request_handler] diff --git a/concurrency/tests/ui/deferred_handler_return_type.stderr b/concurrency/tests/ui/deferred_handler_return_type.stderr new file mode 100644 index 00000000..8fa1ddc0 --- /dev/null +++ b/concurrency/tests/ui/deferred_handler_return_type.stderr @@ -0,0 +1,5 @@ +error: #[deferred_handler] answers through its PendingReply; remove the return type + --> tests/ui/deferred_handler_return_type.rs:13:91 + | +13 | fn read(&mut self, _msg: Read, reply: PendingReply<'_, u64>, _ctx: &Context) -> u64 { + | ^^^ diff --git a/concurrency/tests/ui/deferred_handler_without_reply.stderr b/concurrency/tests/ui/deferred_handler_without_reply.stderr new file mode 100644 index 00000000..10661bc2 --- /dev/null +++ b/concurrency/tests/ui/deferred_handler_without_reply.stderr @@ -0,0 +1,5 @@ +error: #[deferred_handler] method must have 4 parameters (&mut self, msg: M, reply: PendingReply<'_, R>, ctx: &Context), found 3 + --> tests/ui/deferred_handler_without_reply.rs:13:5 + | +13 | fn read(&mut self, _msg: Read, _ctx: &Context) {} + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/concurrency/tests/ui/pending_reply_stored.stderr b/concurrency/tests/ui/pending_reply_stored.stderr new file mode 100644 index 00000000..680ecee0 --- /dev/null +++ b/concurrency/tests/ui/pending_reply_stored.stderr @@ -0,0 +1,17 @@ +error[E0521]: borrowed data escapes outside of method + --> tests/ui/pending_reply_stored.rs:18:9 + | +17 | fn read(&mut self, _msg: Read, reply: PendingReply<'_, u64>, _ctx: &Context) { + | ----- + | | + | `reply` is a reference that is only valid in the method body + | has type `PendingReply<'1, u64>` +18 | self.held.push(reply); + | ^^^^^^^^^^^^^^^^^^^^^ + | | + | `reply` escapes the method body here + | argument requires that `'1` must outlive `'static` + | + = note: requirement occurs because of a mutable reference to `Vec>` + = note: mutable references are invariant over their type parameter + = help: see for more information about variance diff --git a/concurrency/tests/ui/reply_to_not_clone.stderr b/concurrency/tests/ui/reply_to_not_clone.stderr new file mode 100644 index 00000000..d361802f --- /dev/null +++ b/concurrency/tests/ui/reply_to_not_clone.stderr @@ -0,0 +1,15 @@ +error[E0308]: mismatched types + --> tests/ui/reply_to_not_clone.rs:4:5 + | +3 | fn duplicate(token: &ReplyTo) -> ReplyTo { + | ------------ expected `ReplyTo` because of return type +4 | token.clone() + | ^^^^^^^^^^^^^ expected `ReplyTo`, found `&ReplyTo` + | + = note: expected struct `ReplyTo` + found reference `&ReplyTo` +note: `ReplyTo` does not implement `Clone`, so `&ReplyTo` was cloned instead + --> tests/ui/reply_to_not_clone.rs:4:5 + | +4 | token.clone() + | ^^^^^ From ef49870072132d28b39e0c8297e1d6aeccb65efb Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:32:13 -0300 Subject: [PATCH 017/208] test: deferred reply discharge paths in both modes; ctx.deferred_outstanding() --- concurrency/src/tasks/actor.rs | 6 + concurrency/src/threads/actor.rs | 6 + concurrency/tests/deferred_replies.rs | 507 ++++++++++++++++++++++++++ 3 files changed, 519 insertions(+) create mode 100644 concurrency/tests/deferred_replies.rs diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 6a288041..cfca57fa 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -327,6 +327,12 @@ impl Context { self.shared.mailbox.depth() } + /// Deferred-reply credits in use by this instance: live deferred + /// tokens plus running linked jobs (bounded by `Actor::MAX_OUTSTANDING`). + pub fn deferred_outstanding(&self) -> usize { + self.shared.replies.outstanding() + } + /// The actor's unique identity. pub fn id(&self) -> ActorId { self.shared.id diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index d20a3c4f..bfe45631 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -241,6 +241,12 @@ impl Context { self.mailbox.depth() } + /// Deferred-reply credits in use by this instance: live deferred + /// tokens plus running linked jobs (bounded by `Actor::MAX_OUTSTANDING`). + pub fn deferred_outstanding(&self) -> usize { + self.replies.outstanding() + } + /// The actor's unique identity. pub fn id(&self) -> ActorId { self.id diff --git a/concurrency/tests/deferred_replies.rs b/concurrency/tests/deferred_replies.rs new file mode 100644 index 00000000..65fa58f1 --- /dev/null +++ b/concurrency/tests/deferred_replies.rs @@ -0,0 +1,507 @@ +//! Deferred replies (docs/design/deferred-replies.md): every discharge path of +//! a request's reply slot, in `tasks` mode (also under `spawned_sim`) and in +//! `threads` mode. +use spawned_concurrency::{message::Message, ActorError, Deadline, DeferError, Delivery, ReplyTo}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +/// Park the request: the handler defers it and stores the token. +pub struct Park(pub u64); +impl Message for Park { + type Result = u64; +} +/// Answer every parked request with its value plus `0`. +pub struct Release(pub u64); +impl Message for Release { + type Result = usize; +} +/// Drop every parked token between turns. +pub struct DropAll; +impl Message for DropAll { + type Result = (); +} +/// Return without answering (the `PendingReply` is dropped in the turn). +pub struct Forget; +impl Message for Forget { + type Result = u64; +} +/// Panic in the turn, after optionally deferring into a local. +pub struct Explode { + pub defer_first: bool, +} +impl Message for Explode { + type Result = u64; +} +/// Report what `defer()` returned (for sends, expired requests). +pub struct TryDefer; +impl Message for TryDefer { + type Result = (); +} +/// Stop, answering parked tokens in `stopped()` when `answer` is set. +pub struct Stop { + pub answer: bool, +} +impl Message for Stop { + type Result = (); +} +pub struct Outstanding; +impl Message for Outstanding { + type Result = usize; +} + +/// Shared observations of the deferred-reply actor. +#[derive(Default)] +pub struct Seen { + pub defer_errors: Mutex>, + pub late: Mutex>, +} + +#[cfg(feature = "async")] +mod tasks_mode { + use super::*; + use spawned_concurrency::tasks::{Actor, ActorStart, Context, Handler}; + use spawned_concurrency::PendingReply; + + pub struct Parker { + pub parked: Vec<(u64, ReplyTo)>, + pub answer_in_stopped: bool, + pub seen: Arc, + } + impl Parker { + pub fn new(seen: Arc) -> Self { + Self { + parked: Vec::new(), + answer_in_stopped: false, + seen, + } + } + } + impl Actor for Parker { + const MAX_OUTSTANDING: usize = 2; + async fn stopped(&mut self, _ctx: &Context) { + if self.answer_in_stopped { + for (v, token) in self.parked.drain(..) { + token.reply(v + 1000); + } + } + } + } + impl Handler for Parker { + async fn handle(&mut self, msg: Park, reply: PendingReply<'_, u64>, _: &Context) { + match reply.defer() { + Ok(token) => self.parked.push((msg.0, token)), + Err(e) => self.seen.defer_errors.lock().unwrap().push(e), + } + } + } + impl Handler for Parker { + async fn handle(&mut self, msg: Release, reply: PendingReply<'_, usize>, _: &Context) { + let n = self.parked.len(); + for (v, token) in self.parked.drain(..) { + let delivery = token.reply(v + msg.0); + if delivery != Delivery::Delivered { + self.seen.late.lock().unwrap().push(delivery); + } + } + reply.reply(n); + } + } + impl Handler for Parker { + async fn handle(&mut self, _: DropAll, reply: PendingReply<'_, ()>, _: &Context) { + self.parked.clear(); + reply.reply(()); + } + } + impl Handler for Parker { + async fn handle(&mut self, _: Forget, _reply: PendingReply<'_, u64>, _: &Context) {} + } + impl Handler for Parker { + async fn handle(&mut self, msg: Explode, reply: PendingReply<'_, u64>, _: &Context) { + let _held = if msg.defer_first { + Some(reply.defer().expect("defer")) + } else { + None + }; + panic!("explode"); + } + } + impl Handler for Parker { + async fn handle(&mut self, _: TryDefer, reply: PendingReply<'_, ()>, _: &Context) { + match reply.defer() { + Ok(token) => drop(token), + Err(e) => self.seen.defer_errors.lock().unwrap().push(e), + } + } + } + impl Handler for Parker { + async fn handle(&mut self, msg: Stop, reply: PendingReply<'_, ()>, ctx: &Context) { + self.answer_in_stopped = msg.answer; + ctx.stop(); + reply.reply(()); + } + } + impl Handler for Parker { + async fn handle(&mut self, _: Outstanding, reply: PendingReply<'_, usize>, ctx: &Context) { + reply.reply(ctx.deferred_outstanding()); + } + } + + fn start() -> (spawned_concurrency::tasks::ActorRef, Arc) { + let seen = Arc::new(Seen::default()); + (Parker::new(seen.clone()).start(), seen) + } + + async fn settle(actor: &spawned_concurrency::tasks::ActorRef) -> usize { + actor.request(Outstanding).await.unwrap() + } + + #[spawned_macros::sim_test] + async fn deferred_reply_answers_from_a_later_turn() { + let (actor, _) = start(); + let a = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); + let b = actor.request_raw_until(Park(2), Deadline::Never).unwrap(); + assert_eq!(settle(&actor).await, 2); + // The mailbox kept serving while both requests were parked. + assert_eq!(actor.request(Release(10)).await.unwrap(), 2); + assert_eq!(a.await.unwrap(), 11); + assert_eq!(b.await.unwrap(), 12); + assert_eq!(settle(&actor).await, 0); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn dropped_token_between_turns_answers_reply_dropped() { + let (actor, _) = start(); + let rx = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); + actor.request(DropAll).await.unwrap(); + assert!(matches!(rx.await, Err(ActorError::ReplyDropped))); + assert_eq!(settle(&actor).await, 0); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn unanswered_turn_answers_reply_dropped() { + let (actor, _) = start(); + assert!(matches!(actor.request(Forget).await, Err(ActorError::ReplyDropped))); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn panicking_turn_answers_actor_stopped() { + for defer_first in [false, true] { + let (actor, _) = start(); + let result = actor.request(Explode { defer_first }).await; + assert!(matches!(result, Err(ActorError::ActorStopped)), "{result:?}"); + actor.join().await; + } + } + + #[spawned_macros::sim_test] + async fn defer_at_the_bound_answers_overloaded() { + let (actor, seen) = start(); + let a = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); + let b = actor.request_raw_until(Park(2), Deadline::Never).unwrap(); + // MAX_OUTSTANDING is 2: the third deferral is refused in the same step. + assert!(matches!(actor.request(Park(3)).await, Err(ActorError::Overloaded))); + assert_eq!(*seen.defer_errors.lock().unwrap(), [DeferError::Overloaded]); + assert_eq!(actor.request(Release(0)).await.unwrap(), 2); + assert_eq!((a.await.unwrap(), b.await.unwrap()), (1, 2)); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn abandoned_caller_returns_the_credit_and_drops_the_late_reply() { + let (actor, seen) = start(); + let rx = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); + assert_eq!(settle(&actor).await, 1); + drop(rx); + assert_eq!(settle(&actor).await, 0, "abandonment returns the credit at once"); + actor.request(Release(0)).await.unwrap(); + assert_eq!(*seen.late.lock().unwrap(), [Delivery::CallerGone]); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn sends_and_expired_requests_cannot_defer() { + let (actor, seen) = start(); + actor.send(TryDefer).unwrap(); + let expired = actor + .request_raw_until(TryDefer, Deadline::after(Duration::ZERO)) + .unwrap(); + assert!(matches!(expired.await, Err(ActorError::RequestTimeout))); + settle(&actor).await; + assert_eq!( + *seen.defer_errors.lock().unwrap(), + [DeferError::NoCaller, DeferError::Closed] + ); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn late_reply_after_the_deadline_is_expired() { + let (actor, seen) = start(); + let rx = actor + .request_raw_until(Park(1), Deadline::after(Duration::from_millis(50))) + .unwrap(); + assert!(matches!(rx.await, Err(ActorError::RequestTimeout))); + actor.request(Release(0)).await.unwrap(); + assert_eq!(*seen.late.lock().unwrap(), [Delivery::Expired]); + assert_eq!(settle(&actor).await, 0); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn stopped_may_answer_stored_tokens() { + let (actor, _) = start(); + let rx = actor.request_raw_until(Park(7), Deadline::Never).unwrap(); + actor.request(Stop { answer: true }).await.unwrap(); + assert_eq!(rx.await.unwrap(), 1007); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn death_answers_stored_tokens_actor_stopped() { + let (actor, _) = start(); + let rx = actor.request_raw_until(Park(7), Deadline::Never).unwrap(); + let queued_behind_stop = { + actor.request(Stop { answer: false }).await.unwrap(); + actor.request_raw_until(Park(8), Deadline::Never) + }; + assert!(matches!(rx.await, Err(ActorError::ActorStopped))); + if let Ok(queued) = queued_behind_stop { + assert!(matches!(queued.await, Err(ActorError::ActorStopped))); + } + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn token_closed_resolves_when_the_caller_leaves() { + use spawned_concurrency::tasks::ActorRef; + struct Watcher { + closed: Option>, + } + impl Actor for Watcher {} + struct Watch; + impl Message for Watch { + type Result = (); + } + struct Ping; + impl Message for Ping { + type Result = (); + } + struct Done; + impl Message for Done { + type Result = bool; + } + impl Handler for Watcher { + async fn handle(&mut self, _: Watch, reply: PendingReply<'_, ()>, _: &Context) { + let token = reply.defer().unwrap(); + self.closed = Some(spawned_rt::tasks::spawn(async move { + token.closed().await; + token.is_closed() + })); + } + } + impl Handler for Watcher { + async fn handle(&mut self, _: Ping, reply: PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } + } + impl Handler for Watcher { + async fn handle(&mut self, _: Done, reply: PendingReply<'_, bool>, _: &Context) { + let closed = self.closed.take().unwrap().await.unwrap(); + reply.reply(closed); + } + } + let actor: ActorRef = Watcher { closed: None }.start(); + let rx = actor.request_raw_until(Watch, Deadline::Never).unwrap(); + actor.request(Ping).await.unwrap(); // `Watch` was handled (FIFO) + drop(rx); + assert!(actor.request(Done).await.unwrap()); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn request_timeout_starts_at_send() { + let (actor, _) = start(); + // Sent now with a short deadline, polled only after it passed. + let response = actor + .request_raw_until(Park(1), Deadline::after(Duration::from_millis(10))) + .unwrap(); + spawned_rt::tasks::sleep(Duration::from_millis(50)).await; + assert!(matches!(response.await, Err(ActorError::RequestTimeout))); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn reply_pair_drives_a_manual_handler_directly() { + let seen = Arc::new(Seen::default()); + let mut parker = Parker::new(seen); + let actor = Parker::new(Arc::new(Seen::default())).start(); + let ctx = actor.context(); + let (reply, rx) = spawned_concurrency::reply::reply_pair::(Deadline::Never); + parker.handle(Park(5), reply, &ctx).await; + let (release, released) = spawned_concurrency::reply::reply_pair::(Deadline::Never); + parker.handle(Release(1), release, &ctx).await; + assert_eq!(released.await.unwrap(), 1); + assert_eq!(rx.await.unwrap(), 6); + actor.context().stop(); + actor.join().await; + } +} + +#[cfg(not(spawned_sim))] +mod threads_mode { + use super::*; + use spawned_concurrency::threads::{Actor, ActorStart, Context, Handler}; + use spawned_concurrency::PendingReply; + + pub struct Parker { + parked: Vec<(u64, ReplyTo)>, + answer_in_stopped: bool, + seen: Arc, + } + impl Actor for Parker { + const MAX_OUTSTANDING: usize = 2; + fn stopped(&mut self, _ctx: &Context) { + if self.answer_in_stopped { + for (v, token) in self.parked.drain(..) { + token.reply(v + 1000); + } + } + } + } + impl Handler for Parker { + fn handle(&mut self, msg: Park, reply: PendingReply<'_, u64>, _: &Context) { + match reply.defer() { + Ok(token) => self.parked.push((msg.0, token)), + Err(e) => self.seen.defer_errors.lock().unwrap().push(e), + } + } + } + impl Handler for Parker { + fn handle(&mut self, msg: Release, reply: PendingReply<'_, usize>, _: &Context) { + let n = self.parked.len(); + for (v, token) in self.parked.drain(..) { + let delivery = token.reply(v + msg.0); + if delivery != Delivery::Delivered { + self.seen.late.lock().unwrap().push(delivery); + } + } + reply.reply(n); + } + } + impl Handler for Parker { + fn handle(&mut self, _: DropAll, reply: PendingReply<'_, ()>, _: &Context) { + self.parked.clear(); + reply.reply(()); + } + } + impl Handler for Parker { + fn handle(&mut self, _: Forget, _reply: PendingReply<'_, u64>, _: &Context) {} + } + impl Handler for Parker { + fn handle(&mut self, msg: Explode, reply: PendingReply<'_, u64>, _: &Context) { + let _held = if msg.defer_first { + Some(reply.defer().expect("defer")) + } else { + None + }; + panic!("explode"); + } + } + impl Handler for Parker { + fn handle(&mut self, msg: Stop, reply: PendingReply<'_, ()>, ctx: &Context) { + self.answer_in_stopped = msg.answer; + ctx.stop(); + reply.reply(()); + } + } + impl Handler for Parker { + fn handle(&mut self, _: Outstanding, reply: PendingReply<'_, usize>, ctx: &Context) { + reply.reply(ctx.deferred_outstanding()); + } + } + + fn start() -> (spawned_concurrency::threads::ActorRef, Arc) { + let seen = Arc::new(Seen::default()); + let actor = Parker { + parked: Vec::new(), + answer_in_stopped: false, + seen: seen.clone(), + } + .start(); + (actor, seen) + } + + #[test] + fn every_discharge_path() { + // Later turn. + let (actor, seen) = start(); + let a = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); + assert_eq!(actor.request(Outstanding).unwrap(), 1); + assert_eq!(actor.request(Release(10)).unwrap(), 1); + assert_eq!(a.recv().unwrap(), 11); + // Dropped between turns. + let b = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); + actor.request(DropAll).unwrap(); + assert!(matches!(b.recv(), Err(ActorError::ReplyDropped))); + // Unanswered turn. + assert!(matches!(actor.request(Forget), Err(ActorError::ReplyDropped))); + // Bound. + let c = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); + let d = actor.request_raw_until(Park(2), Deadline::Never).unwrap(); + assert!(matches!(actor.request(Park(3)), Err(ActorError::Overloaded))); + assert_eq!(*seen.defer_errors.lock().unwrap(), [DeferError::Overloaded]); + // Abandonment returns the credit. + drop(c); + assert_eq!(actor.request(Outstanding).unwrap(), 1); + // Expired deadline: the receiver expires the slot; the late reply is dropped. + let e = actor + .request_raw_until(Park(4), Deadline::after(Duration::from_millis(20))) + .unwrap(); + assert!(matches!(e.recv(), Err(ActorError::RequestTimeout))); + actor.request(Release(0)).unwrap(); + assert_eq!(d.recv().unwrap(), 2); + assert_eq!( + *seen.late.lock().unwrap(), + [Delivery::CallerGone, Delivery::Expired] + ); + assert_eq!(actor.request(Outstanding).unwrap(), 0); + // stopped() may answer; then death closes the rest. + let f = actor.request_raw_until(Park(7), Deadline::Never).unwrap(); + actor.request(Stop { answer: true }).unwrap(); + assert_eq!(f.recv().unwrap(), 1007); + actor.join(); + assert!(matches!( + actor.request(Outstanding), + Err(ActorError::ActorStopped) + )); + } + + #[test] + fn panic_and_death_answer_actor_stopped() { + for defer_first in [false, true] { + let (actor, _) = start(); + assert!(matches!( + actor.request(Explode { defer_first }), + Err(ActorError::ActorStopped) + )); + actor.join(); + } + let (actor, _) = start(); + let parked = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); + actor.request(Stop { answer: false }).unwrap(); + assert!(matches!(parked.recv(), Err(ActorError::ActorStopped))); + actor.join(); + } +} From ba978cf1a968c98a067ae9ac03796ab6d17efb93 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:32:32 -0300 Subject: [PATCH 018/208] style: rustfmt --- concurrency/tests/deferred_replies.rs | 45 ++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/concurrency/tests/deferred_replies.rs b/concurrency/tests/deferred_replies.rs index 65fa58f1..10e5d941 100644 --- a/concurrency/tests/deferred_replies.rs +++ b/concurrency/tests/deferred_replies.rs @@ -95,7 +95,12 @@ mod tasks_mode { } } impl Handler for Parker { - async fn handle(&mut self, msg: Release, reply: PendingReply<'_, usize>, _: &Context) { + async fn handle( + &mut self, + msg: Release, + reply: PendingReply<'_, usize>, + _: &Context, + ) { let n = self.parked.len(); for (v, token) in self.parked.drain(..) { let delivery = token.reply(v + msg.0); @@ -141,7 +146,12 @@ mod tasks_mode { } } impl Handler for Parker { - async fn handle(&mut self, _: Outstanding, reply: PendingReply<'_, usize>, ctx: &Context) { + async fn handle( + &mut self, + _: Outstanding, + reply: PendingReply<'_, usize>, + ctx: &Context, + ) { reply.reply(ctx.deferred_outstanding()); } } @@ -184,7 +194,10 @@ mod tasks_mode { #[spawned_macros::sim_test] async fn unanswered_turn_answers_reply_dropped() { let (actor, _) = start(); - assert!(matches!(actor.request(Forget).await, Err(ActorError::ReplyDropped))); + assert!(matches!( + actor.request(Forget).await, + Err(ActorError::ReplyDropped) + )); actor.context().stop(); actor.join().await; } @@ -194,7 +207,10 @@ mod tasks_mode { for defer_first in [false, true] { let (actor, _) = start(); let result = actor.request(Explode { defer_first }).await; - assert!(matches!(result, Err(ActorError::ActorStopped)), "{result:?}"); + assert!( + matches!(result, Err(ActorError::ActorStopped)), + "{result:?}" + ); actor.join().await; } } @@ -205,7 +221,10 @@ mod tasks_mode { let a = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); let b = actor.request_raw_until(Park(2), Deadline::Never).unwrap(); // MAX_OUTSTANDING is 2: the third deferral is refused in the same step. - assert!(matches!(actor.request(Park(3)).await, Err(ActorError::Overloaded))); + assert!(matches!( + actor.request(Park(3)).await, + Err(ActorError::Overloaded) + )); assert_eq!(*seen.defer_errors.lock().unwrap(), [DeferError::Overloaded]); assert_eq!(actor.request(Release(0)).await.unwrap(), 2); assert_eq!((a.await.unwrap(), b.await.unwrap()), (1, 2)); @@ -219,7 +238,11 @@ mod tasks_mode { let rx = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); assert_eq!(settle(&actor).await, 1); drop(rx); - assert_eq!(settle(&actor).await, 0, "abandonment returns the credit at once"); + assert_eq!( + settle(&actor).await, + 0, + "abandonment returns the credit at once" + ); actor.request(Release(0)).await.unwrap(); assert_eq!(*seen.late.lock().unwrap(), [Delivery::CallerGone]); actor.context().stop(); @@ -456,11 +479,17 @@ mod threads_mode { actor.request(DropAll).unwrap(); assert!(matches!(b.recv(), Err(ActorError::ReplyDropped))); // Unanswered turn. - assert!(matches!(actor.request(Forget), Err(ActorError::ReplyDropped))); + assert!(matches!( + actor.request(Forget), + Err(ActorError::ReplyDropped) + )); // Bound. let c = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); let d = actor.request_raw_until(Park(2), Deadline::Never).unwrap(); - assert!(matches!(actor.request(Park(3)), Err(ActorError::Overloaded))); + assert!(matches!( + actor.request(Park(3)), + Err(ActorError::Overloaded) + )); assert_eq!(*seen.defer_errors.lock().unwrap(), [DeferError::Overloaded]); // Abandonment returns the credit. drop(c); From d70bad24552d5294c9a606cfbb5683b3c36a511f Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:35:08 -0300 Subject: [PATCH 019/208] feat(reply): per-actor deadline heap and deadline service (thread; per-heap sim task) --- concurrency/src/reply/deadline.rs | 321 ++++++++++++++++++++++++++ concurrency/src/reply/mod.rs | 2 + concurrency/src/reply/registry.rs | 91 +++++++- concurrency/tests/deferred_replies.rs | 40 ++++ 4 files changed, 447 insertions(+), 7 deletions(-) create mode 100644 concurrency/src/reply/deadline.rs diff --git a/concurrency/src/reply/deadline.rs b/concurrency/src/reply/deadline.rs new file mode 100644 index 00000000..f393368f --- /dev/null +++ b/concurrency/src/reply/deadline.rs @@ -0,0 +1,321 @@ +//! Per-actor deadline heaps and the deadline service +//! (docs/design/deferred-replies.md §2, "Deadline storage"). +//! +//! Each actor's registry owns a min-heap of `(deadline, entry)`, where an entry +//! is a deferred token's slot (and, with jobs, a job state). Entries hold +//! `Weak` references, so the heap owns nothing. An entry is stale when its +//! item is gone or terminal, or when its deadline is later than the item's +//! current one (superseded by `with_deadline`); the heap is rebuilt when stale +//! entries exceed live ones. +//! +//! The service holds one entry per actor, its earliest deadline, and expires +//! due entries even when no caller polls, so credits return on time: +//! +//! - real time: one process-wide service thread with its own timer, so +//! per-test Tokio runtimes cannot break it; +//! - under `spawned_sim`: a task of the simulated runtime per actor heap, so +//! clock domains never mix and virtual time drives it. + +use super::registry::Registry; +use std::cmp::Reverse; +use std::collections::BinaryHeap; +use std::sync::{Arc, Weak}; + +/// Something with a deadline the owner's heap services. +pub(crate) trait Expirable: Send + Sync { + /// The current deadline while the item is pending, else `None`. + fn pending_deadline(&self) -> Option; + /// The deadline passed (`now >= deadline`): expire the item. + fn expire(&self, now: u64); +} + +struct Entry { + at: u64, + seq: u64, + item: Weak, +} +impl PartialEq for Entry { + fn eq(&self, other: &Self) -> bool { + (self.at, self.seq) == (other.at, other.seq) + } +} +impl Eq for Entry {} +impl PartialOrd for Entry { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl Ord for Entry { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + (self.at, self.seq).cmp(&(other.at, other.seq)) + } +} + +impl Entry { + fn is_live(&self) -> bool { + self.item + .upgrade() + .is_some_and(|item| item.pending_deadline() == Some(self.at)) + } +} + +/// One actor's deadline heap. +#[derive(Default)] +pub(crate) struct DeadlineHeap { + heap: BinaryHeap>, + seq: u64, + /// Live entries at the last rebuild; the heap is rebuilt when it holds + /// more than twice that many (plus slack). + live_at_rebuild: usize, +} + +impl DeadlineHeap { + /// Add an entry. Returns whether it is the new earliest deadline. + pub(crate) fn push(&mut self, at: u64, item: Weak) -> bool { + if self.heap.len() >= 2 * self.live_at_rebuild + 16 { + self.compact(); + } + self.seq += 1; + let earliest = self.earliest().is_none_or(|e| at < e); + self.heap.push(Reverse(Entry { + at, + seq: self.seq, + item, + })); + earliest + } + + pub(crate) fn earliest(&self) -> Option { + self.heap.peek().map(|Reverse(e)| e.at) + } + + pub(crate) fn len(&self) -> usize { + self.heap.len() + } + + /// Rebuild without stale entries. + fn compact(&mut self) { + let entries = std::mem::take(&mut self.heap).into_vec(); + self.heap = entries.into_iter().filter(|Reverse(e)| e.is_live()).collect(); + self.live_at_rebuild = self.heap.len(); + } + + /// Pop every entry due at `now`; the caller expires them outside the lock. + pub(crate) fn due(&mut self, now: u64) -> Vec> { + let mut due = Vec::new(); + while let Some(Reverse(top)) = self.heap.peek() { + if top.at > now { + break; + } + let Reverse(entry) = self.heap.pop().expect("peeked"); + if let Some(item) = entry.item.upgrade() { + due.push(item); + } + } + due + } + + /// Drop every entry (the owner died). + pub(crate) fn clear(&mut self) { + self.heap.clear(); + } +} + +// --------------------------------------------------------------------------- +// Service +// --------------------------------------------------------------------------- + +/// Ask the service to visit `registry` at `at`. +pub(crate) fn schedule(registry: &Arc, at: u64) { + #[cfg(all(spawned_sim, feature = "async"))] + sim::schedule(registry, at); + #[cfg(not(all(spawned_sim, feature = "async")))] + thread_service::schedule(registry, at); +} + +#[cfg(not(all(spawned_sim, feature = "async")))] +mod thread_service { + use super::*; + use std::collections::HashMap; + use std::sync::{Condvar, Mutex, OnceLock}; + use std::time::Duration; + + #[derive(Default)] + struct State { + /// `(at, key)`; an entry is current only if `actors[key].0 == at`. + queue: BinaryHeap>, + actors: HashMap)>, + } + + struct Service { + state: Mutex, + wake: Condvar, + } + + fn lock(service: &Service) -> std::sync::MutexGuard<'_, State> { + service.state.lock().unwrap_or_else(|p| p.into_inner()) + } + + fn service() -> &'static Service { + static SERVICE: OnceLock<&'static Service> = OnceLock::new(); + SERVICE.get_or_init(|| { + let service: &'static Service = Box::leak(Box::new(Service { + state: Mutex::new(State::default()), + wake: Condvar::new(), + })); + std::thread::Builder::new() + .name("spawned-deadlines".into()) + .spawn(move || run(service)) + .expect("spawn the deadline service thread"); + service + }) + } + + pub(super) fn schedule(registry: &Arc, at: u64) { + let service = service(); + let key = registry.key(); + let mut state = lock(service); + match state.actors.get(&key) { + Some((current, _)) if *current <= at => return, + _ => {} + } + state.actors.insert(key, (at, Arc::downgrade(registry))); + state.queue.push(Reverse((at, key))); + if state.queue.len() > 2 * state.actors.len() + 64 { + let actors = &state.actors; + let live: BinaryHeap<_> = actors + .iter() + .map(|(key, (at, _))| Reverse((*at, *key))) + .collect(); + state.queue = live; + } + drop(state); + service.wake.notify_one(); + } + + fn run(service: &'static Service) { + let mut state = lock(service); + loop { + let now = super::super::now_ticks(); + match state.queue.peek().copied() { + None => { + state = service.wake.wait(state).unwrap_or_else(|p| p.into_inner()); + } + Some(Reverse((at, key))) if at <= now => { + state.queue.pop(); + let current = state.actors.get(&key).map(|(a, w)| (*a, w.clone())); + let Some((current_at, weak)) = current else { + continue; + }; + if current_at != at { + continue; // superseded by an earlier schedule + } + state.actors.remove(&key); + drop(state); + if let Some(registry) = weak.upgrade() { + if let Some(next) = registry.service_deadlines(now) { + schedule(®istry, next); + } + } + state = lock(service); + } + Some(Reverse((at, _))) => { + let wait = Duration::from_nanos(at - now); + state = service + .wake + .wait_timeout(state, wait) + .unwrap_or_else(|p| p.into_inner()) + .0; + } + } + } + } +} + +#[cfg(all(spawned_sim, feature = "async"))] +mod sim { + use super::*; + use futures::task::AtomicWaker; + use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + use std::task::Poll; + use std::time::Duration; + + /// Per-heap service task state, kept by the registry. + #[derive(Default)] + pub(crate) struct SimService { + started: AtomicBool, + earliest: AtomicU64, + waker: AtomicWaker, + } + + pub(super) fn schedule(registry: &Arc, at: u64) { + let service = registry.sim_service(); + service.earliest.fetch_min(at, Ordering::AcqRel); + if !service.started.swap(true, Ordering::AcqRel) { + service.earliest.store(at, Ordering::Release); + let weak = Arc::downgrade(registry); + spawned_rt::tasks::spawn(run(weak)); + } + service.waker.wake(); + } + + async fn run(weak: Weak) { + loop { + let Some(registry) = weak.upgrade() else { + return; + }; + if registry.is_dead() { + return; + } + let at = registry.sim_service().earliest.load(Ordering::Acquire); + drop(registry); + let now = super::super::now_ticks(); + if at <= now { + let Some(registry) = weak.upgrade() else { + return; + }; + registry + .sim_service() + .earliest + .store(u64::MAX, Ordering::Release); + if let Some(next) = registry.service_deadlines(now) { + registry + .sim_service() + .earliest + .fetch_min(next, Ordering::AcqRel); + } + continue; + } + // Sleep until `at`, or until an earlier deadline or the death + // wakes us. + let mut sleep = (at != u64::MAX) + .then(|| Box::pin(spawned_rt::tasks::sleep(Duration::from_nanos(at - now)))); + let weak_poll = weak.clone(); + std::future::poll_fn(|cx| { + let Some(registry) = weak_poll.upgrade() else { + return Poll::Ready(()); + }; + let service = registry.sim_service(); + service.waker.register(cx.waker()); + if registry.is_dead() || service.earliest.load(Ordering::Acquire) < at { + return Poll::Ready(()); + } + match &mut sleep { + Some(sleep) => sleep.as_mut().poll(cx), + None => Poll::Pending, + } + }) + .await; + } + } + + impl SimService { + pub(crate) fn wake(&self) { + self.waker.wake(); + } + } + use std::future::Future; +} + +#[cfg(all(spawned_sim, feature = "async"))] +pub(crate) use sim::SimService; diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index d8e545e0..f3c8f70e 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -6,6 +6,7 @@ //! a storable [`ReplyTo`]), and by the caller's [`ReplyReceiver`]. Each slot //! gets exactly one terminal discharge. +pub(crate) mod deadline; pub(crate) mod registry; pub(crate) mod slot; @@ -230,6 +231,7 @@ impl<'t, R: Send + 'static> PendingReply<'t, R> { match slot.defer(registry.pool(), &now_ticks) { Ok(()) => { registry.register(&slot); + registry.track_deadline(&slot); Ok(ReplyTo { slot: Some(slot), registry: registry.clone(), diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index 34d8d2da..6609285b 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -14,9 +14,10 @@ //! two uncontended atomic stores; the turn list is locked only when a drop //! lands on it (`IN_TURN_PENDING`). -use super::slot::{CreditPool, Slot, Terminal}; +use super::deadline::{self, DeadlineHeap, Expirable}; +use super::slot::{CreditPool, Slot, Terminal, NEVER}; use crate::ActorId; -use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; use std::sync::{Arc, Mutex, OnceLock, Weak}; /// Default `Actor::MAX_OUTSTANDING`: deferred credits per actor instance. @@ -35,6 +36,16 @@ pub(crate) trait Closeable: Send + Sync { fn is_terminal(&self) -> bool; } +impl Expirable for Slot { + fn pending_deadline(&self) -> Option { + let deadline = self.deadline(); + (!self.is_terminal() && deadline != NEVER).then_some(deadline) + } + fn expire(&self, now: u64) { + self.check_deadline(&|| now); + } +} + impl Closeable for Slot { fn close(&self) -> bool { Slot::close(self) @@ -52,13 +63,18 @@ struct Lists { tokens: Vec>, prune_at: usize, turn_drops: Vec>, + deadlines: DeadlineHeap, } pub(crate) struct Registry { owner: ActorId, + /// Service key, unique per registry. + key: u64, pool: Arc, phase: AtomicU8, lists: Mutex, + #[cfg(all(spawned_sim, feature = "async"))] + sim: deadline::SimService, } fn lock(lists: &Mutex) -> std::sync::MutexGuard<'_, Lists> { @@ -68,14 +84,18 @@ fn lock(lists: &Mutex) -> std::sync::MutexGuard<'_, Lists> { impl Registry { pub(crate) fn new(owner: ActorId, max_outstanding: usize) -> Arc { let max = max_outstanding.min(super::slot::MAX_JOBS as usize); + static NEXT_KEY: AtomicU64 = AtomicU64::new(1); Arc::new(Self { owner, + key: NEXT_KEY.fetch_add(1, Ordering::Relaxed), pool: Arc::new(CreditPool::new(max)), phase: AtomicU8::new(RUNNING), lists: Mutex::new(Lists { prune_at: 16, ..Lists::default() }), + #[cfg(all(spawned_sim, feature = "async"))] + sim: deadline::SimService::default(), }) } @@ -113,9 +133,61 @@ impl Registry { lists.tokens.push(weak); } - /// Placeholder for the deadline heap (step 5); deferred tokens are - /// otherwise expired by their polling receiver or a reply. - pub(crate) fn track_deadline(&self, _slot: &Arc>) {} + pub(crate) fn key(&self) -> u64 { + self.key + } + + #[cfg(all(spawned_sim, feature = "async"))] + pub(crate) fn sim_service(&self) -> &deadline::SimService { + &self.sim + } + + pub(crate) fn is_dead(&self) -> bool { + self.phase.load(Ordering::Acquire) == DEAD + } + + /// Put a deferred token's deadline in the owner's heap, so it expires + /// (and returns its credit) even when no caller polls. + pub(crate) fn track_deadline(self: &Arc, slot: &Arc>) { + let at = slot.deadline(); + if at == NEVER || slot.is_terminal() { + return; + } + let weak: Weak = Arc::downgrade(slot) as Weak; + self.track(at, weak); + } + + /// Add a heap entry; reschedule the service on a new earliest deadline. + pub(crate) fn track(self: &Arc, at: u64, item: Weak) { + let earliest = { + let mut lists = lock(&self.lists); + if self.is_dead() { + return; + } + lists.deadlines.push(at, item) + }; + if earliest { + deadline::schedule(self, at); + } + } + + /// Entries in the deadline heap, live and stale (bounded by compaction). + #[cfg(test)] + pub(crate) fn heap_entries(&self) -> usize { + lock(&self.lists).deadlines.len() + } + + /// The service visits: expire every due entry outside the lock, and + /// return the next earliest deadline. + pub(crate) fn service_deadlines(&self, now: u64) -> Option { + let due = lock(&self.lists).deadlines.due(now); + for item in due { + if item.pending_deadline().is_some_and(|d| d <= now) { + item.expire(now); + } + } + lock(&self.lists).deadlines.earliest() + } /// A reply obligation (`PendingReply` or `ReplyTo`) was dropped unanswered. pub(crate) fn dropped(&self, slot: Arc>) { @@ -224,11 +296,16 @@ impl Registry { self.commit(Terminal::Stopped); } let tokens = std::mem::take(&mut lock(&self.lists).tokens); - tokens + let closed = tokens .into_iter() .filter_map(|t| t.upgrade()) .filter(|s| s.close()) - .count() + .count(); + // The heap is dropped after the close (§5 step 3.4). + lock(&self.lists).deadlines.clear(); + #[cfg(all(spawned_sim, feature = "async"))] + self.sim.wake(); + closed } } diff --git a/concurrency/tests/deferred_replies.rs b/concurrency/tests/deferred_replies.rs index 10e5d941..f6891d19 100644 --- a/concurrency/tests/deferred_replies.rs +++ b/concurrency/tests/deferred_replies.rs @@ -280,6 +280,28 @@ mod tasks_mode { actor.join().await; } + #[spawned_macros::sim_test] + async fn unpolled_deferred_token_expires_by_the_owner_heap() { + let (actor, seen) = start(); + // Nobody polls this receiver: only the owner's deadline heap can + // expire the token and return its credit. + let rx = actor + .request_raw_until(Park(1), Deadline::after(Duration::from_millis(20))) + .unwrap(); + assert_eq!(settle(&actor).await, 1); + let mut waited = Duration::ZERO; + while settle(&actor).await != 0 { + assert!(waited < Duration::from_secs(5), "token never expired"); + spawned_rt::tasks::sleep(Duration::from_millis(10)).await; + waited += Duration::from_millis(10); + } + actor.request(Release(0)).await.unwrap(); + assert_eq!(*seen.late.lock().unwrap(), [Delivery::Expired]); + assert!(matches!(rx.await, Err(ActorError::RequestTimeout))); + actor.context().stop(); + actor.join().await; + } + #[spawned_macros::sim_test] async fn stopped_may_answer_stored_tokens() { let (actor, _) = start(); @@ -517,6 +539,24 @@ mod threads_mode { )); } + #[test] + fn unpolled_deferred_token_expires_by_the_owner_heap() { + let (actor, seen) = start(); + let rx = actor + .request_raw_until(Park(1), Deadline::after(Duration::from_millis(20))) + .unwrap(); + let began = std::time::Instant::now(); + while actor.request(Outstanding).unwrap() != 0 { + assert!(began.elapsed() < Duration::from_secs(5), "token never expired"); + std::thread::sleep(Duration::from_millis(5)); + } + actor.request(Release(0)).unwrap(); + assert_eq!(*seen.late.lock().unwrap(), [Delivery::Expired]); + assert!(matches!(rx.recv(), Err(ActorError::RequestTimeout))); + actor.context().stop(); + actor.join(); + } + #[test] fn panic_and_death_answer_actor_stopped() { for defer_first in [false, true] { From f678aa54a56369201543601d00d7601357ef71c1 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:36:07 -0300 Subject: [PATCH 020/208] perf(reply): cheaper turn phase and no clock read in the async receiver poll --- concurrency/src/reply/mod.rs | 2 +- concurrency/src/reply/registry.rs | 21 ++++++--------------- concurrency/src/reply/slot.rs | 11 +++++------ concurrency/tests/reply_slot_loom.rs | 4 ++-- 4 files changed, 14 insertions(+), 24 deletions(-) diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index f3c8f70e..c619aeea 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -471,7 +471,7 @@ impl std::future::Future for ReplyReceiver { let mut this = self.project(); assert!(!*this.done, "ReplyReceiver polled after completion"); loop { - if let Poll::Ready(result) = this.slot.poll(cx.waker(), &now_ticks) { + if let Poll::Ready(result) = this.slot.poll(cx.waker()) { *this.done = true; return Poll::Ready(result.map_err(terminal_error)); } diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index 6609285b..22c9a1dc 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -245,26 +245,17 @@ impl Registry { } /// The run loop is about to dispatch a handler. + /// + /// Only the run loop moves the phase out of `RUNNING`/`IN_TURN` (droppers + /// only mark `IN_TURN_PENDING`), so a plain store suffices here. pub(crate) fn begin_turn(&self) { - let _ = self - .phase - .compare_exchange(RUNNING, IN_TURN, Ordering::AcqRel, Ordering::Acquire); + debug_assert_eq!(self.phase.load(Ordering::Relaxed), RUNNING); + self.phase.store(IN_TURN, Ordering::Release); } /// The handler returned (`panicked == false`) or unwound. pub(crate) fn end_turn(&self, panicked: bool) { - let previous = self - .phase - .compare_exchange(IN_TURN, RUNNING, Ordering::AcqRel, Ordering::Acquire) - .or_else(|_| { - self.phase.compare_exchange( - IN_TURN_PENDING, - RUNNING, - Ordering::AcqRel, - Ordering::Acquire, - ) - }); - if previous == Ok(IN_TURN_PENDING) { + if self.phase.swap(RUNNING, Ordering::AcqRel) == IN_TURN_PENDING { self.commit(if panicked { Terminal::Stopped } else { diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 1200eda3..52344c16 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -499,16 +499,15 @@ impl Slot { }) } - /// Poll from the caller's receiver. Expires the slot when its deadline has - /// passed; otherwise registers `waker` and returns `Pending`. - pub(crate) fn poll(&self, waker: &Waker, now: &impl Fn() -> u64) -> Poll> { + /// Poll from the caller's receiver: registers `waker` and returns + /// `Pending` while the slot is open. The caller's timer expires the slot + /// at its deadline (`ReplyReceiver` sleeps until it), so the poll itself + /// reads no clock. + pub(crate) fn poll(&self, waker: &Waker) -> Poll> { loop { if let Some(result) = self.take(state(self.word())) { return Poll::Ready(result); } - if self.check_deadline(now) { - continue; - } let mut inner = lock(&self.inner); let w = self.word(); if !live(state(w)) { diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index 6cfb54c3..50454dfa 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -383,7 +383,7 @@ fn polled_receiver_vs_reply() { let slot = slot.clone(); loom::thread::spawn(move || slot.reply(4, &|| 0).0) }; - let got = loom::future::block_on(std::future::poll_fn(|cx| slot.poll(cx.waker(), &|| 0))); + let got = loom::future::block_on(std::future::poll_fn(|cx| slot.poll(cx.waker()))); assert_eq!(got, Ok(4)); assert_eq!(replier.join().unwrap(), Delivery::Delivered); }); @@ -399,7 +399,7 @@ fn polled_receiver_vs_token_drop() { let slot = slot.clone(); loom::thread::spawn(move || slot.terminate(Terminal::Dropped)) }; - let got = loom::future::block_on(std::future::poll_fn(|cx| slot.poll(cx.waker(), &|| 0))); + let got = loom::future::block_on(std::future::poll_fn(|cx| slot.poll(cx.waker()))); assert_eq!(got, Err(Terminal::Dropped)); assert!(dropper.join().unwrap()); assert_balanced(&pool, &[&slot]); From aa8f0192fadab08173987c7d3fa629af7dcfe3d4 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:40:49 -0300 Subject: [PATCH 021/208] feat(jobs): tasks-mode pipe, pipe_keyed, pipe_with, pipe_blocking_with, pipe_reply* and JobCtx emits (WIP) --- concurrency/src/reply/jobs.rs | 573 +++++++++++++++++++++++++++++++ concurrency/src/reply/mod.rs | 1 + concurrency/src/tasks/actor.rs | 35 ++ concurrency/src/tasks/jobs.rs | 603 +++++++++++++++++++++++++++++++++ 4 files changed, 1212 insertions(+) create mode 100644 concurrency/src/reply/jobs.rs create mode 100644 concurrency/src/tasks/jobs.rs diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs new file mode 100644 index 00000000..7e4b0efa --- /dev/null +++ b/concurrency/src/reply/jobs.rs @@ -0,0 +1,573 @@ +//! Pipe-to-self jobs: the mode-independent core +//! (docs/design/deferred-replies.md §8, §9). +//! +//! Each job has one shared [`JobState`] with three parts: +//! +//! - **execution** (running or finished), physical: cancelled or timed-out +//! work may keep running, and keeps its credit (or native slot) until it +//! ends; +//! - **cancellation**, a flag set by `JobHandle::cancel`, a newer job with the +//! same key, owner exit, or (opt-in) the linked caller leaving; it wakes +//! blocked emitters and suppresses later results; +//! - **terminal notification** (pending, notified, suppressed): a one-shot +//! under the job's lock. The result, a caught panic, or the job deadline +//! takes it and enqueues the terminal message; emits check the same lock, +//! so no emit follows the terminal message. +//! +//! Messages a job produces carry its `JobState` and the owner's `ActorId` and +//! are fenced at dispatch when the job is cancelled. + +use super::deadline::Expirable; +use super::registry::Registry; +use super::slot::CreditPool; +use crate::ActorId; +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard, Weak}; + +/// Default emit credits per job (`JobOptions::emit_credits`). +pub const DEFAULT_EMIT_CREDITS: u16 = 16; + +/// Default `Actor::MAX_NATIVE_JOBS`: `min(4, available_parallelism)`. +pub fn default_max_native_jobs() -> usize { + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + .min(4) +} + +/// A key for cancel-previous jobs (`pipe_keyed`, `JobOptions::key`). +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct JobKey(std::borrow::Cow<'static, str>); + +impl JobKey { + pub const fn new(name: &'static str) -> Self { + Self(std::borrow::Cow::Borrowed(name)) + } + pub fn owned(name: String) -> Self { + Self(std::borrow::Cow::Owned(name)) + } +} +impl From<&'static str> for JobKey { + fn from(name: &'static str) -> Self { + Self::new(name) + } +} + +/// Options for `pipe_with`, `pipe_blocking_with`, `pipe_reply_with` and +/// `pipe_reply_blocking_with`. Passed first; never chained afterwards. +#[derive(Debug, Clone)] +pub struct JobOptions { + /// Deliver `JobError::DeadlineExceeded` at this instant if the job has + /// not finished. The work keeps running and keeps its credit. + pub deadline: Option, + /// Cancel the previous job with this key (whatever its notification + /// state) and fence its queued messages. + pub key: Option, + /// `pipe_reply` only: cancel the job when its caller leaves or its + /// deadline passes. Off by default, as in OTP. + pub cancel_on_caller_gone: bool, + /// Emits in flight before `emit` waits (blocks in `emit_blocking`). + pub emit_credits: u16, +} + +impl Default for JobOptions { + fn default() -> Self { + Self { + deadline: None, + key: None, + cancel_on_caller_gone: false, + emit_credits: DEFAULT_EMIT_CREDITS, + } + } +} + +/// How a job ended, delivered to `done` in the owner's turn. +#[derive(Debug)] +pub enum JobOutcome { + Ok(T), + Err(JobError), +} + +impl JobOutcome { + pub fn into_result(self) -> Result { + match self { + Self::Ok(value) => Ok(value), + Self::Err(error) => Err(error), + } + } +} + +/// Why a job produced no value. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum JobError { + /// The work panicked; caught at the job boundary. + #[error("job panicked: {0}")] + Panicked(String), + /// `JobOptions::deadline` passed first. The work may still be running. + #[error("job deadline exceeded")] + DeadlineExceeded, +} + +/// Why a job was not spawned. +#[derive(Debug, thiserror::Error)] +pub enum PipeErrorKind { + /// The owner's `MAX_OUTSTANDING` credits are in use. + #[error("deferred credits at the bound")] + Overloaded, + /// The owner's `MAX_NATIVE_JOBS` native jobs are running. + #[error("native job budget at the bound")] + NativeBudget, + /// The native thread could not be spawned. + #[error("spawning the job failed: {0}")] + SpawnFailed(std::io::Error), + /// The owner is exiting. + #[error("the owner is stopping")] + OwnerStopping, +} + +/// `pipe*` failed; no job was spawned and any reserved credit was released. +#[derive(Debug, thiserror::Error)] +#[error("{kind}")] +pub struct PipeError { + pub kind: PipeErrorKind, +} + +/// `pipe_reply*` failed; the token comes back in every case. +#[derive(Debug)] +pub enum PipeReplyError { + Failed { + token: super::ReplyTo, + kind: PipeErrorKind, + }, + /// The request is no longer open (or past its deadline). + Closed(super::ReplyTo), + /// `token.owner()` is not this actor. + WrongOwner(super::ReplyTo), +} + +impl PipeReplyError { + /// The token handed back. + pub fn into_token(self) -> super::ReplyTo { + match self { + Self::Failed { token, .. } | Self::Closed(token) | Self::WrongOwner(token) => token, + } + } +} + +/// The job is cancelled or already notified: an emit was not enqueued. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("the job is gone (cancelled, replaced or past its terminal notification)")] +pub struct JobGone; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Notification { + Pending, + Notified, + Suppressed, +} + +struct Inner { + cancelled: bool, + notification: Notification, + running: bool, + emit_credits: u16, +} + +/// One job's shared state. +pub(crate) struct JobState { + owner: ActorId, + inner: Mutex, + emitters: Condvar, + cancelled_fast: AtomicBool, + /// Async cancellation for `tasks` jobs (drops the work future). + #[cfg(feature = "async")] + cancel: spawned_rt::tasks::CancellationToken, + /// Wakers of async emitters waiting for a credit. + #[cfg(feature = "async")] + emit_wakers: Mutex>, +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(|p| p.into_inner()) +} + +impl JobState { + pub(crate) fn new(owner: ActorId, emit_credits: u16) -> Arc { + Arc::new(Self { + owner, + inner: Mutex::new(Inner { + cancelled: false, + notification: Notification::Pending, + running: true, + emit_credits, + }), + emitters: Condvar::new(), + cancelled_fast: AtomicBool::new(false), + #[cfg(feature = "async")] + cancel: spawned_rt::tasks::CancellationToken::new(), + #[cfg(feature = "async")] + emit_wakers: Mutex::new(Vec::new()), + }) + } + + pub(crate) fn owner(&self) -> ActorId { + self.owner + } + + pub(crate) fn is_cancelled(&self) -> bool { + self.cancelled_fast.load(Ordering::Acquire) + } + + #[cfg(feature = "async")] + pub(crate) fn cancellation(&self) -> &spawned_rt::tasks::CancellationToken { + &self.cancel + } + + /// Set the cancellation flag: suppresses a pending notification and + /// wakes blocked emitters. Never undoes effects. + pub(crate) fn cancel(&self) { + { + let mut inner = lock(&self.inner); + inner.cancelled = true; + if inner.notification == Notification::Pending { + inner.notification = Notification::Suppressed; + } + } + self.cancelled_fast.store(true, Ordering::Release); + self.emitters.notify_all(); + #[cfg(feature = "async")] + { + self.cancel.cancel(); + self.wake_emitters(); + } + } + + #[cfg(feature = "async")] + fn wake_emitters(&self) { + let wakers = std::mem::take(&mut *lock(&self.emit_wakers)); + for waker in wakers { + waker.wake(); + } + } + + /// Take the terminal notification and run `enqueue` under the job's + /// lock. Returns whether it was taken; `enqueue` is not called when the + /// job was cancelled or already notified. Whatever `enqueue` hands back + /// (an envelope the mailbox refused) is returned to be dropped by the + /// caller after the lock is released. + pub(crate) fn notify(&self, enqueue: impl FnOnce() -> Option) -> (bool, Option) { + let unsent = { + let mut inner = lock(&self.inner); + if inner.cancelled || inner.notification != Notification::Pending { + return (false, None); + } + inner.notification = Notification::Notified; + enqueue() + }; + self.emitters.notify_all(); + #[cfg(feature = "async")] + self.wake_emitters(); + (true, unsent) + } + + /// Execution ended (physically). + pub(crate) fn finished(&self) { + lock(&self.inner).running = false; + } + + pub(crate) fn is_running(&self) -> bool { + lock(&self.inner).running + } + + /// Emit step 1 (blocking): take an emit credit, waiting while all are in + /// flight. Fails with `JobGone` once the job is cancelled or notified. + pub(crate) fn acquire_blocking(&self) -> Result<(), JobGone> { + let mut inner = lock(&self.inner); + loop { + if inner.cancelled || inner.notification != Notification::Pending { + return Err(JobGone); + } + if inner.emit_credits > 0 { + inner.emit_credits -= 1; + return Ok(()); + } + inner = self + .emitters + .wait(inner) + .unwrap_or_else(|p| p.into_inner()); + } + } + + /// Emit step 1 (async). + #[cfg(feature = "async")] + pub(crate) fn poll_acquire(&self, cx: &mut std::task::Context<'_>) -> std::task::Poll> { + let mut inner = lock(&self.inner); + if inner.cancelled || inner.notification != Notification::Pending { + return std::task::Poll::Ready(Err(JobGone)); + } + if inner.emit_credits > 0 { + inner.emit_credits -= 1; + return std::task::Poll::Ready(Ok(())); + } + lock(&self.emit_wakers).push(cx.waker().clone()); + std::task::Poll::Pending + } + + /// Emit step 2: under the job's lock, enqueue unless the job is + /// cancelled or notified (then the credit returns here). An envelope the + /// mailbox refused comes back to be dropped after the lock is released; + /// its credit guard returns the credit then. + pub(crate) fn enqueue_emit( + &self, + enqueue: impl FnOnce() -> Result<(), X>, + ) -> Result<(), (JobGone, Option)> { + let mut inner = lock(&self.inner); + if inner.cancelled || inner.notification != Notification::Pending { + inner.emit_credits += 1; + drop(inner); + self.emitters.notify_all(); + return Err((JobGone, None)); + } + enqueue().map_err(|unsent| (JobGone, Some(unsent))) + } + + /// Emit step 3: a credit comes back when its message is dispatched, + /// fenced or dropped with the mailbox. + pub(crate) fn return_emit_credit(&self) { + lock(&self.inner).emit_credits += 1; + self.emitters.notify_all(); + #[cfg(feature = "async")] + self.wake_emitters(); + } +} + +/// A handle to a spawned job. Dropping it leaves the job owned by the +/// actor's job set (it is not detached). +#[derive(Clone)] +pub struct JobHandle { + state: Arc, +} + +impl std::fmt::Debug for JobHandle { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("JobHandle") + .field("owner", &self.state.owner) + .field("cancelled", &self.state.is_cancelled()) + .finish_non_exhaustive() + } +} + +impl JobHandle { + pub(crate) fn new(state: Arc) -> Self { + Self { state } + } + /// Cancel the job: its later results and queued messages are fenced, and + /// blocked emitters return `JobGone`. Work already done is not undone. + pub fn cancel(&self) { + self.state.cancel(); + } + pub fn is_cancelled(&self) -> bool { + self.state.is_cancelled() + } + /// Whether the job's work is still executing (cancelled work may still + /// run until it returns). + pub fn is_running(&self) -> bool { + self.state.is_running() + } +} + +// --------------------------------------------------------------------------- +// Job set +// --------------------------------------------------------------------------- + +/// An owner instance's jobs: key map, running count, native budget. +pub(crate) struct JobSet { + owner: ActorId, + jobs: Mutex>>, + keys: Mutex>>, + running: Mutex, + idle: Condvar, + native: AtomicUsize, + max_native: usize, + #[cfg(feature = "async")] + idle_async: Mutex>, +} + +impl JobSet { + pub(crate) fn new(owner: ActorId, max_native: usize) -> Self { + Self { + owner, + jobs: Mutex::new(Vec::new()), + keys: Mutex::new(HashMap::new()), + running: Mutex::new(0), + idle: Condvar::new(), + native: AtomicUsize::new(0), + max_native, + #[cfg(feature = "async")] + idle_async: Mutex::new(Vec::new()), + } + } + + pub(crate) fn owner(&self) -> ActorId { + self.owner + } + + /// Reserve a native slot (`MAX_NATIVE_JOBS`), rejecting at the bound. + pub(crate) fn reserve_native(&self) -> bool { + self.native + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |n| { + (n < self.max_native).then_some(n + 1) + }) + .is_ok() + } + + pub(crate) fn release_native(&self) { + self.native.fetch_sub(1, Ordering::AcqRel); + } + + /// Admit a job: replace (cancel) the previous one with the same key and + /// count it as running. + pub(crate) fn admit(&self, state: &Arc, key: Option) { + if let Some(key) = key { + let previous = lock(&self.keys).insert(key, state.clone()); + if let Some(previous) = previous { + previous.cancel(); + } + } + { + let mut jobs = lock(&self.jobs); + if jobs.len() >= 32 && jobs.len().is_power_of_two() { + jobs.retain(|j| j.upgrade().is_some_and(|j| j.is_running())); + } + jobs.push(Arc::downgrade(state)); + } + *lock(&self.running) += 1; + } + + /// A job's execution ended. + pub(crate) fn ended(&self, state: &Arc) { + state.finished(); + { + let mut keys = lock(&self.keys); + keys.retain(|_, current| !Arc::ptr_eq(current, state)); + } + let idle = { + let mut running = lock(&self.running); + *running -= 1; + *running == 0 + }; + if idle { + self.idle.notify_all(); + #[cfg(feature = "async")] + for waker in std::mem::take(&mut *lock(&self.idle_async)) { + waker.wake(); + } + } + } + + pub(crate) fn running(&self) -> usize { + *lock(&self.running) + } + + /// Owner exit: cancel every job (flag set, blocked emitters woken). + pub(crate) fn cancel_all(&self) { + let jobs = std::mem::take(&mut *lock(&self.jobs)); + lock(&self.keys).clear(); + for job in jobs.iter().filter_map(Weak::upgrade) { + job.cancel(); + } + } + + /// Join the cancelled jobs, bounded by `deadline`; returns how many are + /// still running (unfinished at stop). + pub(crate) fn join_blocking(&self, deadline: std::time::Instant) -> usize { + let mut running = lock(&self.running); + while *running > 0 { + let now = std::time::Instant::now(); + if now >= deadline { + break; + } + running = self + .idle + .wait_timeout(running, deadline - now) + .unwrap_or_else(|p| p.into_inner()) + .0; + } + *running + } + + /// Async join, bounded by `timeout` on the runtime's clock. + #[cfg(feature = "async")] + pub(crate) async fn join(&self, timeout: std::time::Duration) -> usize { + let idle = std::future::poll_fn(|cx| { + if self.running() == 0 { + return std::task::Poll::Ready(()); + } + lock(&self.idle_async).push(cx.waker().clone()); + if self.running() == 0 { + return std::task::Poll::Ready(()); + } + std::task::Poll::Pending + }); + let _ = spawned_rt::tasks::timeout(timeout, idle).await; + self.running() + } +} + +/// Report jobs still running when the owner's cleanup deadline passed. +pub(crate) fn report_unfinished(owner: ActorId, unfinished: usize) { + if unfinished > 0 { + tracing::warn!( + event = "unfinished_at_stop", + actor = %owner, + unfinished, + "job cleanup incomplete: jobs still running at the cleanup deadline" + ); + } +} + +// --------------------------------------------------------------------------- +// Credits and deadlines +// --------------------------------------------------------------------------- + +/// The credit an unlinked job holds while it executes. +pub(crate) struct JobCredit(Option>); + +impl JobCredit { + pub(crate) fn reserve(registry: &Registry) -> Option { + let pool = registry.pool().clone(); + pool.reserve(1).then_some(Self(Some(pool))) + } +} + +impl Drop for JobCredit { + fn drop(&mut self) { + if let Some(pool) = self.0.take() { + pool.release(1); + } + } +} + +/// The deadline notification of one job, serviced by the owner's heap. +pub(crate) struct JobDeadline { + pub(crate) at: u64, + pub(crate) state: Arc, + pub(crate) notify: Mutex>>, +} + +impl Expirable for JobDeadline { + fn pending_deadline(&self) -> Option { + let pending = { + let inner = lock(&self.state.inner); + !inner.cancelled && inner.notification == Notification::Pending + }; + pending.then_some(self.at) + } + fn expire(&self, _now: u64) { + let notify = lock(&self.notify).take(); + if let Some(notify) = notify { + notify(); + } + } +} diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index c619aeea..e34329b2 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -7,6 +7,7 @@ //! gets exactly one terminal discharge. pub(crate) mod deadline; +pub mod jobs; pub(crate) mod registry; pub(crate) mod slot; diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index cfca57fa..3c4fa7de 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -32,6 +32,19 @@ use std::{ pub use crate::response::DEFAULT_REQUEST_TIMEOUT; +/// Cleanup deadline for actors without a supervisor or system grace: the +/// bound on joining an exiting actor's cancelled jobs. +pub const DEFAULT_STOP_DEADLINE: Duration = Duration::from_secs(5); + +#[path = "jobs.rs"] +mod jobs; +pub use jobs::JobCtx; +#[doc(no_inline)] +pub use crate::reply::jobs::{ + JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, PipeError, PipeErrorKind, + PipeReplyError, +}; + // --------------------------------------------------------------------------- // Backend // --------------------------------------------------------------------------- @@ -72,6 +85,10 @@ pub trait Actor: Send + Sized + 'static { /// Capped at 65535. const MAX_OUTSTANDING: usize = DEFAULT_MAX_OUTSTANDING; + /// Native threads this instance's blocking jobs may run at once + /// (`pipe_blocking*`). `None`: `min(4, available_parallelism)`. + const MAX_NATIVE_JOBS: Option = None; + fn started(&mut self, _ctx: &Context) -> impl Future + Send { async {} } @@ -294,6 +311,8 @@ struct ActorShared { managed: Option>, /// This instance's reply registry: deferred tokens, credits, turn phase. replies: Arc, + /// This instance's pipe-to-self jobs. + jobs: crate::reply::jobs::JobSet, } impl Clone for Context { @@ -1058,6 +1077,10 @@ impl ActorRef { shared: Arc::new(ActorShared { id, replies: Registry::new(id, A::MAX_OUTSTANDING), + jobs: crate::reply::jobs::JobSet::new( + id, + A::MAX_NATIVE_JOBS.unwrap_or_else(crate::reply::jobs::default_max_native_jobs), + ), sender: tx, cancellation_token: cancellation_token.clone(), completion_rx, @@ -1124,6 +1147,7 @@ impl ActorRef { .with_mailbox_failures(actor_ref.shared.sender.failures()); let startup = crate::actor_cleanup::ActorStartup::new((actor, ctx, rx, cancellation_token), cleanup); + let job_owner = actor_ref.shared.clone(); let inner_future = async move { let ((actor, ctx, rx, cancellation_token), mut cleanup) = startup.into_parts(); #[cfg(feature = "backend-experiments")] @@ -1153,6 +1177,13 @@ impl ActorRef { let actual = crate::system::lock(&scope.cause).clone().unwrap_or(reason); scope.exited(actual); } + // Job cleanup (deferred-replies §5 step 4), after death: join the + // cancelled jobs, bounded by the cleanup deadline; a job still + // running then does not undo death and is reported unfinished. + if job_owner.jobs.running() > 0 { + let unfinished = job_owner.jobs.join(DEFAULT_STOP_DEADLINE).await; + crate::reply::jobs::report_unfinished(job_owner.id, unfinished); + } }; if let Some(lease) = lease { @@ -1363,6 +1394,7 @@ async fn run_actor_lifecycle( scope.end_running(); } ctx.shared.replies.exiting(); + ctx.shared.jobs.cancel_all(); if let Err(panic) = AssertUnwindSafe(async { fail_point!("actor.before_stopped", async); actor.stopped(&ctx).await @@ -1460,6 +1492,7 @@ async fn run_actor_lifecycle( } cancellation_token.cancel(); ctx.shared.replies.exiting(); + ctx.shared.jobs.cancel_all(); let stop_result = AssertUnwindSafe(async { fail_point!("actor.before_stopped", async); actor.stopped(&ctx).await @@ -3355,6 +3388,7 @@ mod turn_regressions { mailbox: crate::instrumentation::Mailbox::default(), managed: None, replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), + jobs: crate::reply::jobs::JobSet::new(ActorId::next(), 1), }), }; let count = Arc::new(AtomicUsize::new(0)); @@ -3432,6 +3466,7 @@ mod idle_stop_regression { mailbox: crate::instrumentation::Mailbox::default(), managed: None, replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), + jobs: crate::reply::jobs::JobSet::new(ActorId::next(), 1), }), }; let count = Arc::new(AtomicUsize::new(0)); diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs new file mode 100644 index 00000000..3b575a6d --- /dev/null +++ b/concurrency/src/tasks/jobs.rs @@ -0,0 +1,603 @@ +//! `tasks` mode pipe-to-self jobs (docs/design/deferred-replies.md §9): +//! `ctx.pipe*` and `ctx.pipe_reply*`, their envelopes and `JobCtx`. +//! +//! A child module of `tasks::actor`, so it reaches the private envelope and +//! shared-state types. + +use super::{Actor, Context, Envelope, Handler}; +use crate::message::Message; +use crate::reply::deadline::Expirable; +use crate::reply::jobs::{ + JobCredit, JobDeadline, JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, + JobState, PipeError, PipeErrorKind, PipeReplyError, +}; +use crate::reply::slot::{Refused, Slot}; +use crate::reply::{now_ticks, PendingReply, ReplyTo}; +use futures::future::{self, FutureExt as _}; +use std::future::Future; +use std::panic::AssertUnwindSafe; +use std::pin::{pin, Pin}; +use std::sync::{Arc, Mutex, Weak}; + +// --------------------------------------------------------------------------- +// Envelopes +// --------------------------------------------------------------------------- + +/// Returns one emit credit when the emitted message is dispatched, fenced or +/// dropped with the mailbox. +struct EmitCredit(Arc); +impl Drop for EmitCredit { + fn drop(&mut self) { + self.0.return_emit_credit(); + } +} + +/// A message emitted by a job; fenced at dispatch when the job is cancelled +/// or belongs to another instance. +struct EmitEnvelope { + msg: M, + credit: EmitCredit, +} + +impl Envelope for EmitEnvelope +where + A: Actor + Handler, + M: Message, +{ + #[cfg(feature = "diagnostics")] + fn message_type(&self) -> &'static str { + std::any::type_name::() + } + fn handle<'a>( + self: Box, + actor: &'a mut A, + ctx: &'a Context, + ) -> Pin + Send + 'a>> { + let EmitEnvelope { msg, credit } = *self; + let fenced = credit.0.is_cancelled() || credit.0.owner() != ctx.id(); + // The credit returns at dispatch (or here, when fenced). + drop(credit); + if fenced { + drop(msg); + return Box::pin(async {}); + } + Box::pin(actor.handle(msg, PendingReply::detached(), ctx)) + } +} + +type Done = Box) -> M + Send>; + +/// A job's terminal message: `done(outcome)` runs in the owner's turn and the +/// produced message is dispatched in the same turn. +struct TerminalEnvelope { + job: Arc, + done: Done, + outcome: JobOutcome, +} + +impl Envelope for TerminalEnvelope +where + A: Actor + Handler, + M: Message, + T: Send + 'static, +{ + #[cfg(feature = "diagnostics")] + fn message_type(&self) -> &'static str { + std::any::type_name::() + } + fn handle<'a>( + self: Box, + actor: &'a mut A, + ctx: &'a Context, + ) -> Pin + Send + 'a>> { + let TerminalEnvelope { job, done, outcome } = *self; + if job.is_cancelled() || job.owner() != ctx.id() { + // Fenced: `done` and its captures (a linked token) drop in the + // turn, by the drop rules. + drop((done, outcome)); + return Box::pin(async {}); + } + let msg = done(outcome); + Box::pin(actor.handle(msg, PendingReply::detached(), ctx)) + } +} + +// --------------------------------------------------------------------------- +// Job runs +// --------------------------------------------------------------------------- + +/// The parts shared by a job's completion path and its deadline. +struct Run { + state: Arc, + ctx: Context, + done: Mutex>>, +} + +impl Run +where + A: Actor + Handler, + M: Message, + T: Send + 'static, +{ + /// Take the terminal notification and enqueue `(done, outcome)`. + fn notify(&self, outcome: JobOutcome) { + let (_, unsent) = self.state.notify(|| { + let done = self.done.lock().unwrap_or_else(|p| p.into_inner()).take()?; + let envelope: Box + Send> = Box::new(TerminalEnvelope { + job: self.state.clone(), + done, + outcome, + }); + self.ctx.shared.sender.send(envelope).err() + }); + drop(unsent); + } +} + +/// Holds a `pipe_reply` job's place in its slot's `JOBS` count. +struct LinkedJob(Arc>); +impl Drop for LinkedJob { + fn drop(&mut self) { + self.0.job_end(); + } +} + +/// What a job holds while it executes; released when execution ends. +enum Hold { + Credit(#[allow(dead_code)] JobCredit), + Linked(#[allow(dead_code)] LinkedJob), +} + +/// Native slot (`MAX_NATIVE_JOBS`) held by a blocking job. +struct NativeSlot(Context); +impl Drop for NativeSlot { + fn drop(&mut self) { + self.0.shared.jobs.release_native(); + } +} + +/// Marks the job ended in the owner's job set when execution ends. +struct Ended(Context, Arc); +impl Drop for Ended { + fn drop(&mut self) { + self.0.shared.jobs.ended(&self.1); + } +} + +fn caught(result: std::thread::Result) -> JobOutcome { + match result { + Ok(value) => JobOutcome::Ok(value), + Err(panic) => { + let message = crate::error::panic_message(&*panic); + crate::mailbox::dispose_panic(panic); + JobOutcome::Err(JobError::Panicked(message)) + } + } +} + +/// The job's handle on its owner while it runs. +pub struct JobCtx { + state: Arc, + ctx: Context, +} + +impl std::fmt::Debug for JobCtx { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("JobCtx") + .field("cancelled", &self.state.is_cancelled()) + .finish_non_exhaustive() + } +} + +impl JobCtx { + /// Whether the job was cancelled (handle, key replacement, owner exit, + /// or opt-in caller give-up). Cancellation suppresses later results. + pub fn is_cancelled(&self) -> bool { + self.state.is_cancelled() + } + + /// Resolves when the job is cancelled. + pub async fn cancelled(&self) { + self.state.cancellation().cancelled().await; + } + + /// Send `msg` to the owner as one of this job's messages, fenced like the + /// terminal one and never after it. Waits while `emit_credits` messages + /// are in flight; fails with [`JobGone`] once the job is cancelled or its + /// terminal notification was taken. + pub async fn emit(&self, msg: M) -> Result<(), JobGone> + where + A: Handler, + M: Message, + { + std::future::poll_fn(|cx| self.state.poll_acquire(cx)).await?; + self.enqueue(msg) + } + + /// Blocking [`emit`](Self::emit), for `pipe_blocking*` jobs. + pub fn emit_blocking(&self, msg: M) -> Result<(), JobGone> + where + A: Handler, + M: Message, + { + self.state.acquire_blocking()?; + self.enqueue(msg) + } + + fn enqueue(&self, msg: M) -> Result<(), JobGone> + where + A: Handler, + M: Message, + { + let credit = EmitCredit(self.state.clone()); + self.state + .enqueue_emit(|| { + let envelope: Box + Send> = Box::new(EmitEnvelope { msg, credit }); + self.ctx.shared.sender.send(envelope) + }) + .map_err(|(gone, unsent)| { + drop(unsent); + gone + }) + } +} + +// --------------------------------------------------------------------------- +// Context API +// --------------------------------------------------------------------------- + +impl Context { + fn admit(&self, opts: &JobOptions) -> Result, PipeErrorKind> { + if self.shared.replies.is_exiting() { + return Err(PipeErrorKind::OwnerStopping); + } + Ok(JobState::new(self.id(), opts.emit_credits)) + } + + fn arm_deadline(&self, opts: &JobOptions, run: &Arc>) -> Option> + where + A: Handler, + M: Message, + T: Send + 'static, + { + let at = spawned_rt::time::to_ticks(opts.deadline?); + let weak = Arc::downgrade(run); + let deadline = Arc::new(JobDeadline { + at, + state: run.state.clone(), + notify: Mutex::new(Some(Box::new(move || { + if let Some(run) = weak.upgrade() { + run.notify(JobOutcome::Err(JobError::DeadlineExceeded)); + } + }))), + }); + if at <= now_ticks() { + deadline.expire(at); + } else { + let item: Weak = Arc::downgrade(&deadline) as Weak; + self.shared.replies.track(at, item); + } + Some(deadline) + } + + fn launch_async( + &self, + opts: JobOptions, + state: Arc, + hold: Hold, + work: W, + done: Done, + caller_gone: Option>>, + ) -> JobHandle + where + A: Handler, + M: Message, + T: Send + 'static, + R: Send + 'static, + W: Future + Send + 'static, + { + self.shared.jobs.admit(&state, opts.key.clone()); + let run = Arc::new(Run { + state: state.clone(), + ctx: self.clone(), + done: Mutex::new(Some(done)), + }); + let deadline = self.arm_deadline(&opts, &run); + let ended = Ended(self.clone(), state.clone()); + let handle = JobHandle::new(state.clone()); + self.spawn_framework(async move { + let _ended = ended; + let _hold = hold; + let _deadline = deadline; + let cancelled = pin!(state.cancellation().cancelled()); + let gone = pin!(async { + match &caller_gone { + Some(slot) => { + std::future::poll_fn(|cx| slot.poll_closed(cx.waker(), &now_ticks)).await + } + None => std::future::pending().await, + } + }); + let work = pin!(AssertUnwindSafe(work).catch_unwind()); + let stop = future::select(cancelled, gone); + match future::select(work, stop).await { + future::Either::Left((result, _)) => run.notify(caught(result)), + // Cancelled (or the opt-in caller give-up): the work future is + // dropped here, and with it any downstream receiver. + future::Either::Right(_) => state.cancel(), + } + }); + handle + } + + fn launch_blocking( + &self, + opts: JobOptions, + state: Arc, + hold: Hold, + work: W, + done: Done, + ) -> Result + where + A: Handler, + M: Message, + T: Send + 'static, + R: Send + 'static, + W: FnOnce(&JobCtx) -> T + Send + 'static, + { + if !self.shared.jobs.reserve_native() { + return Err(PipeErrorKind::NativeBudget); + } + let native = NativeSlot(self.clone()); + let run = Arc::new(Run { + state: state.clone(), + ctx: self.clone(), + done: Mutex::new(Some(done)), + }); + let thread_run = run.clone(); + let job = JobCtx { + state: state.clone(), + ctx: self.clone(), + }; + // Admission happens only once the thread exists; the thread waits + // for it, so a spawn failure leaves no trace in the job set. + type Start = (Ended, Option>); + let (start, started) = std::sync::mpsc::channel::>(); + let spawned = std::thread::Builder::new() + .name("spawned-job".into()) + .spawn(move || { + let _native = native; + let _hold = hold; + let Ok((ended, deadline)) = started.recv() else { + return; + }; + let _ended = ended; + let _deadline = deadline; + let result = std::panic::catch_unwind(AssertUnwindSafe(|| work(&job))); + thread_run.notify(caught(result)); + }); + match spawned { + Ok(_thread) => { + self.shared.jobs.admit(&state, opts.key.clone()); + let deadline = self.arm_deadline(&opts, &run); + let ended = Ended(self.clone(), state.clone()); + let _ = start.send((ended, deadline)); + Ok(JobHandle::new(state)) + } + // The closure (native slot, hold, `done`) dropped with the failed + // spawn, releasing what was reserved. + Err(error) => Err(PipeErrorKind::SpawnFailed(error)), + } + } + + /// Run `work` as a job owned by this actor instance and deliver + /// `done(outcome)` as one message in a later turn (`handle_info`). A + /// panic arrives as `JobOutcome::Err(JobError::Panicked)`. Holds one + /// `MAX_OUTSTANDING` credit while it executes. Rejects at once at a + /// bound; never waits. + pub fn pipe(&self, work: W, done: D) -> Result + where + A: Handler, + M: Message, + T: Send + 'static, + W: Future + Send + 'static, + D: FnOnce(JobOutcome) -> M + Send + 'static, + { + self.pipe_with(JobOptions::default(), work, done) + } + + /// [`pipe`](Self::pipe) that first cancels the previous job with `key` + /// (whatever its notification state) and fences its queued messages. + pub fn pipe_keyed( + &self, + key: impl Into, + work: W, + done: D, + ) -> Result + where + A: Handler, + M: Message, + T: Send + 'static, + W: Future + Send + 'static, + D: FnOnce(JobOutcome) -> M + Send + 'static, + { + let opts = JobOptions { + key: Some(key.into()), + ..JobOptions::default() + }; + self.pipe_with(opts, work, done) + } + + /// [`pipe`](Self::pipe) with options (deadline, key, emit credits). + pub fn pipe_with( + &self, + opts: JobOptions, + work: W, + done: D, + ) -> Result + where + A: Handler, + M: Message, + T: Send + 'static, + W: Future + Send + 'static, + D: FnOnce(JobOutcome) -> M + Send + 'static, + { + let state = self.admit(&opts).map_err(|kind| PipeError { kind })?; + let credit = JobCredit::reserve(&self.shared.replies).ok_or(PipeError { + kind: PipeErrorKind::Overloaded, + })?; + Ok(self.launch_async::( + opts, + state, + Hold::Credit(credit), + work, + Box::new(done), + None, + )) + } + + /// Run blocking `work` on an owned native thread, under the actor's + /// `MAX_NATIVE_JOBS` budget. `work` gets a [`JobCtx`] to check + /// cancellation and `emit_blocking` fenced messages. + pub fn pipe_blocking_with( + &self, + opts: JobOptions, + work: W, + done: D, + ) -> Result + where + A: Handler, + M: Message, + T: Send + 'static, + W: FnOnce(&JobCtx) -> T + Send + 'static, + D: FnOnce(JobOutcome) -> M + Send + 'static, + { + let state = self.admit(&opts).map_err(|kind| PipeError { kind })?; + let credit = JobCredit::reserve(&self.shared.replies).ok_or(PipeError { + kind: PipeErrorKind::Overloaded, + })?; + self.launch_blocking::(opts, state, Hold::Credit(credit), work, Box::new(done)) + .map_err(|kind| PipeError { kind }) + } + + /// Delegate a deferred reply: run `work` as a job linked to `token`'s + /// request and deliver `done(outcome, token)` in a later turn. The + /// obligation and credit stay in this actor: the token and its first + /// linked job share one credit. On every error the token comes back. + pub fn pipe_reply( + &self, + token: ReplyTo, + work: W, + done: D, + ) -> Result> + where + A: Handler, + M: Message, + R: Send + 'static, + T: Send + 'static, + W: Future + Send + 'static, + D: FnOnce(JobOutcome, ReplyTo) -> M + Send + 'static, + { + self.pipe_reply_with(JobOptions::default(), token, work, done) + } + + /// [`pipe_reply`](Self::pipe_reply) with options. With + /// `cancel_on_caller_gone` the job is cancelled when the caller leaves + /// or the request's deadline passes (off by default, as in OTP). + pub fn pipe_reply_with( + &self, + opts: JobOptions, + token: ReplyTo, + work: W, + done: D, + ) -> Result> + where + A: Handler, + M: Message, + R: Send + 'static, + T: Send + 'static, + W: Future + Send + 'static, + D: FnOnce(JobOutcome, ReplyTo) -> M + Send + 'static, + { + let (state, slot, token) = self.link(&opts, token)?; + let watch = opts.cancel_on_caller_gone.then(|| slot.clone()); + let hold = Hold::Linked(LinkedJob(slot)); + Ok(self.launch_async(opts, state, hold, work, linked_done(token, done), watch)) + } + + /// Blocking [`pipe_reply`](Self::pipe_reply) on an owned native thread. + pub fn pipe_reply_blocking_with( + &self, + opts: JobOptions, + token: ReplyTo, + work: W, + done: D, + ) -> Result> + where + A: Handler, + M: Message, + R: Send + 'static, + T: Send + 'static, + W: FnOnce(&JobCtx) -> T + Send + 'static, + D: FnOnce(JobOutcome, ReplyTo) -> M + Send + 'static, + { + let (state, slot, token) = self.link(&opts, token)?; + let hold = Hold::Linked(LinkedJob(slot)); + let cell = Arc::new(Mutex::new(Some(token))); + let shared = cell.clone(); + let done: Done = Box::new(move |outcome| { + let token = shared + .lock() + .unwrap_or_else(|p| p.into_inner()) + .take() + .expect("the token is taken once, by done or by a spawn failure"); + done(outcome, token) + }); + self.launch_blocking(opts, state, hold, work, done) + .map_err(|kind| { + let token = cell + .lock() + .unwrap_or_else(|p| p.into_inner()) + .take() + .expect("done never ran"); + PipeReplyError::Failed { token, kind } + }) + } + + /// Checks shared by the `pipe_reply*` calls: owner, owner exit, and the + /// attach CAS on the token's slot (which reserves a credit only when a + /// linked job already runs). + #[allow(clippy::type_complexity)] + fn link( + &self, + opts: &JobOptions, + token: ReplyTo, + ) -> Result<(Arc, Arc>, ReplyTo), PipeReplyError> { + if token.owner() != self.id() { + return Err(PipeReplyError::WrongOwner(token)); + } + let state = match self.admit(opts) { + Ok(state) => state, + Err(kind) => return Err(PipeReplyError::Failed { token, kind }), + }; + let slot = token.slot_arc().clone(); + match slot.attach(&now_ticks) { + Ok(()) => Ok((state, slot, token)), + Err(Refused::Closed) => Err(PipeReplyError::Closed(token)), + Err(Refused::Overloaded) => Err(PipeReplyError::Failed { + token, + kind: PipeErrorKind::Overloaded, + }), + } + } +} + +fn linked_done(token: ReplyTo, done: D) -> Done +where + R: Send + 'static, + T: Send + 'static, + M: Message, + D: FnOnce(JobOutcome, ReplyTo) -> M + Send + 'static, +{ + Box::new(move |outcome| done(outcome, token)) +} From 6c80bca018fc6372b2a86036fd0fd38c9880c1db Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:41:33 -0300 Subject: [PATCH 022/208] fix(jobs): rename the pipe_reply link helper --- concurrency/src/tasks/jobs.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index 3b575a6d..b0e0d23e 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -519,7 +519,7 @@ impl Context { W: Future + Send + 'static, D: FnOnce(JobOutcome, ReplyTo) -> M + Send + 'static, { - let (state, slot, token) = self.link(&opts, token)?; + let (state, slot, token) = self.link_reply(&opts, token)?; let watch = opts.cancel_on_caller_gone.then(|| slot.clone()); let hold = Hold::Linked(LinkedJob(slot)); Ok(self.launch_async(opts, state, hold, work, linked_done(token, done), watch)) @@ -541,7 +541,7 @@ impl Context { W: FnOnce(&JobCtx) -> T + Send + 'static, D: FnOnce(JobOutcome, ReplyTo) -> M + Send + 'static, { - let (state, slot, token) = self.link(&opts, token)?; + let (state, slot, token) = self.link_reply(&opts, token)?; let hold = Hold::Linked(LinkedJob(slot)); let cell = Arc::new(Mutex::new(Some(token))); let shared = cell.clone(); @@ -568,7 +568,7 @@ impl Context { /// attach CAS on the token's slot (which reserves a credit only when a /// linked job already runs). #[allow(clippy::type_complexity)] - fn link( + fn link_reply( &self, opts: &JobOptions, token: ReplyTo, From e20b3dd8733ca47f908bcf5fe1c8d3647fc1cd2b Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:43:38 -0300 Subject: [PATCH 023/208] feat(jobs): threads-mode pipe_blocking_with and pipe_reply_blocking_with on owned native threads --- concurrency/src/tasks/actor.rs | 11 +- concurrency/src/tasks/mod.rs | 8 + concurrency/src/threads/actor.rs | 6 +- concurrency/src/threads/jobs.rs | 408 +++++++++++++++++++++++++++++++ concurrency/src/threads/mod.rs | 8 + 5 files changed, 426 insertions(+), 15 deletions(-) create mode 100644 concurrency/src/threads/jobs.rs diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 3c4fa7de..335ccca1 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -8,11 +8,7 @@ use crate::mailbox::tasks as mpsc; use crate::message::Message; use crate::monitor::{Down, MonitorCleanup, MonitorRef, MonitorTable}; use crate::reply::{registry::Registry, reply_slot, SlotHandle}; -#[doc(no_inline)] -#[allow(unused_imports)] // re-exported for `use spawned_concurrency::tasks::*` users -pub use crate::reply::{ - Deadline, DeferError, Delivery, PendingReply, ReplyReceiver, ReplyTo, DEFAULT_MAX_OUTSTANDING, -}; +use crate::reply::{PendingReply, ReplyReceiver, DEFAULT_MAX_OUTSTANDING}; use core::pin::pin; use futures::future::{self, FutureExt as _}; use spawned_rt::{ @@ -39,11 +35,6 @@ pub const DEFAULT_STOP_DEADLINE: Duration = Duration::from_secs(5); #[path = "jobs.rs"] mod jobs; pub use jobs::JobCtx; -#[doc(no_inline)] -pub use crate::reply::jobs::{ - JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, PipeError, PipeErrorKind, - PipeReplyError, -}; // --------------------------------------------------------------------------- // Backend diff --git a/concurrency/src/tasks/mod.rs b/concurrency/src/tasks/mod.rs index f3c9698f..e12c35f5 100644 --- a/concurrency/src/tasks/mod.rs +++ b/concurrency/src/tasks/mod.rs @@ -12,5 +12,13 @@ pub use actor::{ request, send_message_on, Actor, ActorRef, ActorStart, Backend, Context, Handler, Receiver, Recipient, DEFAULT_REQUEST_TIMEOUT, }; +pub use actor::{JobCtx, DEFAULT_STOP_DEADLINE}; +pub use crate::reply::jobs::{ + JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, PipeError, PipeErrorKind, + PipeReplyError, +}; +pub use crate::reply::{ + Deadline, DeferError, Delivery, PendingReply, ReplyReceiver, ReplyTo, DEFAULT_MAX_OUTSTANDING, +}; pub use stream::spawn_listener; pub use time::{send_after, send_interval, TimerHandle}; diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index bfe45631..c5a7d8ec 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -17,11 +17,7 @@ use crate::link::{ use crate::message::Message; use crate::monitor::{Down, MonitorCleanup, MonitorRef, MonitorTable}; use crate::reply::{registry::Registry, reply_slot, SlotHandle}; -#[doc(no_inline)] -#[allow(unused_imports)] // re-exported for `use spawned_concurrency::threads::*` users -pub use crate::reply::{ - Deadline, DeferError, Delivery, PendingReply, ReplyReceiver, ReplyTo, DEFAULT_MAX_OUTSTANDING, -}; +use crate::reply::{PendingReply, ReplyReceiver, DEFAULT_MAX_OUTSTANDING}; use std::sync::atomic::Ordering; pub use crate::response::DEFAULT_REQUEST_TIMEOUT; diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs new file mode 100644 index 00000000..0d9074d8 --- /dev/null +++ b/concurrency/src/threads/jobs.rs @@ -0,0 +1,408 @@ +//! `threads` mode pipe-to-self jobs (docs/design/deferred-replies.md §9): +//! one owned native thread per admitted job under `MAX_NATIVE_JOBS` +//! (`pipe_blocking_with`, `pipe_reply_blocking_with`), their envelopes and +//! `JobCtx`. The shared pool comes with rank 11b. +//! +//! A child module of `threads::actor`, so it reaches the private envelope and +//! context types. + +use super::{Actor, Context, Envelope, Handler, MailboxItem}; +use crate::message::Message; +use crate::reply::deadline::Expirable; +use crate::reply::jobs::{ + JobCredit, JobDeadline, JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, + JobState, PipeError, PipeErrorKind, PipeReplyError, +}; +use crate::reply::slot::{Refused, Slot}; +use crate::reply::{now_ticks, PendingReply, ReplyTo}; +use std::panic::AssertUnwindSafe; +use std::sync::{Arc, Mutex, Weak}; + +// --------------------------------------------------------------------------- +// Envelopes +// --------------------------------------------------------------------------- + +/// Returns one emit credit when the emitted message is dispatched, fenced or +/// dropped with the mailbox. +struct EmitCredit(Arc); +impl Drop for EmitCredit { + fn drop(&mut self) { + self.0.return_emit_credit(); + } +} + +/// A message emitted by a job; fenced at dispatch when the job is cancelled +/// or belongs to another instance. +struct EmitEnvelope { + msg: M, + credit: EmitCredit, +} + +impl Envelope for EmitEnvelope +where + A: Actor + Handler, + M: Message, +{ + fn handle(self: Box, actor: &mut A, ctx: &Context) { + let EmitEnvelope { msg, credit } = *self; + let fenced = credit.0.is_cancelled() || credit.0.owner() != ctx.id(); + // The credit returns at dispatch (or here, when fenced). + drop(credit); + if fenced { + drop(msg); + return; + } + actor.handle(msg, PendingReply::detached(), ctx); + } +} + +type Done = Box) -> M + Send>; + +/// A job's terminal message: `done(outcome)` runs in the owner's turn and the +/// produced message is dispatched in the same turn. +struct TerminalEnvelope { + job: Arc, + done: Done, + outcome: JobOutcome, +} + +impl Envelope for TerminalEnvelope +where + A: Actor + Handler, + M: Message, + T: Send + 'static, +{ + fn handle(self: Box, actor: &mut A, ctx: &Context) { + let TerminalEnvelope { job, done, outcome } = *self; + if job.is_cancelled() || job.owner() != ctx.id() { + // Fenced: `done` and its captures (a linked token) drop in the + // turn, by the drop rules. + drop((done, outcome)); + return; + } + let msg = done(outcome); + actor.handle(msg, PendingReply::detached(), ctx); + } +} + +// --------------------------------------------------------------------------- +// Job runs +// --------------------------------------------------------------------------- + +/// The parts shared by a job's completion path and its deadline. +struct Run { + state: Arc, + ctx: Context, + done: Mutex>>, +} + +impl Run +where + A: Actor + Handler, + M: Message, + T: Send + 'static, +{ + /// Take the terminal notification and enqueue `(done, outcome)`. + fn notify(&self, outcome: JobOutcome) { + let (_, unsent) = self.state.notify(|| { + let done = self.done.lock().unwrap_or_else(|p| p.into_inner()).take()?; + let envelope: Box + Send> = Box::new(TerminalEnvelope { + job: self.state.clone(), + done, + outcome, + }); + self.ctx.sender.send(MailboxItem::Message(envelope)).err() + }); + drop(unsent); + } +} + +/// Holds a `pipe_reply` job's place in its slot's `JOBS` count. +struct LinkedJob(Arc>); +impl Drop for LinkedJob { + fn drop(&mut self) { + self.0.job_end(); + } +} + +/// What a job holds while it executes; released when execution ends. +enum Hold { + Credit(#[allow(dead_code)] JobCredit), + Linked(#[allow(dead_code)] LinkedJob), +} + +/// Native slot (`MAX_NATIVE_JOBS`) held by a blocking job. +struct NativeSlot(Context); +impl Drop for NativeSlot { + fn drop(&mut self) { + self.0.jobs.release_native(); + } +} + +/// Marks the job ended in the owner's job set when execution ends. +struct Ended(Context, Arc); +impl Drop for Ended { + fn drop(&mut self) { + self.0.jobs.ended(&self.1); + } +} + +fn caught(result: std::thread::Result) -> JobOutcome { + match result { + Ok(value) => JobOutcome::Ok(value), + Err(panic) => { + let message = crate::error::panic_message(&*panic); + crate::mailbox::dispose_panic(panic); + JobOutcome::Err(JobError::Panicked(message)) + } + } +} + +/// The job's handle on its owner while it runs. +pub struct JobCtx { + state: Arc, + ctx: Context, +} + +impl std::fmt::Debug for JobCtx { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("JobCtx") + .field("cancelled", &self.state.is_cancelled()) + .finish_non_exhaustive() + } +} + +impl JobCtx { + /// Whether the job was cancelled (handle, key replacement, owner exit, + /// or opt-in caller give-up). Cancellation suppresses later results. + pub fn is_cancelled(&self) -> bool { + self.state.is_cancelled() + } + + /// Send `msg` to the owner as one of this job's messages, fenced like the + /// terminal one and never after it. Blocks while `emit_credits` messages + /// are in flight; fails with [`JobGone`] once the job is cancelled or its + /// terminal notification was taken. + pub fn emit_blocking(&self, msg: M) -> Result<(), JobGone> + where + A: Handler, + M: Message, + { + self.state.acquire_blocking()?; + self.enqueue(msg) + } + + fn enqueue(&self, msg: M) -> Result<(), JobGone> + where + A: Handler, + M: Message, + { + let credit = EmitCredit(self.state.clone()); + self.state + .enqueue_emit(|| { + let envelope: Box + Send> = Box::new(EmitEnvelope { msg, credit }); + self.ctx.sender.send(MailboxItem::Message(envelope)) + }) + .map_err(|(gone, unsent)| { + drop(unsent); + gone + }) + } +} + +// --------------------------------------------------------------------------- +// Context API +// --------------------------------------------------------------------------- + +impl Context { + fn admit(&self, opts: &JobOptions) -> Result, PipeErrorKind> { + if self.replies.is_exiting() { + return Err(PipeErrorKind::OwnerStopping); + } + Ok(JobState::new(self.id(), opts.emit_credits)) + } + + fn arm_deadline(&self, opts: &JobOptions, run: &Arc>) -> Option> + where + A: Handler, + M: Message, + T: Send + 'static, + { + let at = spawned_rt::time::to_ticks(opts.deadline?); + let weak = Arc::downgrade(run); + let deadline = Arc::new(JobDeadline { + at, + state: run.state.clone(), + notify: Mutex::new(Some(Box::new(move || { + if let Some(run) = weak.upgrade() { + run.notify(JobOutcome::Err(JobError::DeadlineExceeded)); + } + }))), + }); + if at <= now_ticks() { + deadline.expire(at); + } else { + let item: Weak = Arc::downgrade(&deadline) as Weak; + self.replies.track(at, item); + } + Some(deadline) + } + + fn launch_blocking( + &self, + opts: JobOptions, + state: Arc, + hold: Hold, + work: W, + done: Done, + ) -> Result + where + A: Handler, + M: Message, + T: Send + 'static, + R: Send + 'static, + W: FnOnce(&JobCtx) -> T + Send + 'static, + { + if !self.jobs.reserve_native() { + return Err(PipeErrorKind::NativeBudget); + } + let native = NativeSlot(self.clone()); + let run = Arc::new(Run { + state: state.clone(), + ctx: self.clone(), + done: Mutex::new(Some(done)), + }); + let thread_run = run.clone(); + let job = JobCtx { + state: state.clone(), + ctx: self.clone(), + }; + // Admission happens only once the thread exists; the thread waits + // for it, so a spawn failure leaves no trace in the job set. + type Start = (Ended, Option>); + let (start, started) = std::sync::mpsc::channel::>(); + let spawned = std::thread::Builder::new() + .name("spawned-job".into()) + .spawn(move || { + let _native = native; + let _hold = hold; + let Ok((ended, deadline)) = started.recv() else { + return; + }; + let _ended = ended; + let _deadline = deadline; + let result = std::panic::catch_unwind(AssertUnwindSafe(|| work(&job))); + thread_run.notify(caught(result)); + }); + match spawned { + Ok(_thread) => { + self.jobs.admit(&state, opts.key.clone()); + let deadline = self.arm_deadline(&opts, &run); + let ended = Ended(self.clone(), state.clone()); + let _ = start.send((ended, deadline)); + Ok(JobHandle::new(state)) + } + // The closure (native slot, hold, `done`) dropped with the failed + // spawn, releasing what was reserved. + Err(error) => Err(PipeErrorKind::SpawnFailed(error)), + } + } + + /// Run blocking `work` as a job on an owned native thread, under the + /// actor's `MAX_NATIVE_JOBS` budget, and deliver `done(outcome)` as one + /// message in a later turn. A panic arrives as + /// `JobOutcome::Err(JobError::Panicked)`. `work` gets a [`JobCtx`] to + /// check cancellation and `emit_blocking` fenced messages. Rejects at + /// once at a bound; never waits. + pub fn pipe_blocking_with( + &self, + opts: JobOptions, + work: W, + done: D, + ) -> Result + where + A: Handler, + M: Message, + T: Send + 'static, + W: FnOnce(&JobCtx) -> T + Send + 'static, + D: FnOnce(JobOutcome) -> M + Send + 'static, + { + let state = self.admit(&opts).map_err(|kind| PipeError { kind })?; + let credit = JobCredit::reserve(&self.replies).ok_or(PipeError { + kind: PipeErrorKind::Overloaded, + })?; + self.launch_blocking::(opts, state, Hold::Credit(credit), work, Box::new(done)) + .map_err(|kind| PipeError { kind }) + } + + /// Delegate a deferred reply: run blocking `work` on an owned native + /// thread, linked to `token`'s request, and deliver `done(outcome, token)` + /// in a later turn. The obligation and credit stay in this actor: the + /// token and its first linked job share one credit. On every error the + /// token comes back. + pub fn pipe_reply_blocking_with( + &self, + opts: JobOptions, + token: ReplyTo, + work: W, + done: D, + ) -> Result> + where + A: Handler, + M: Message, + R: Send + 'static, + T: Send + 'static, + W: FnOnce(&JobCtx) -> T + Send + 'static, + D: FnOnce(JobOutcome, ReplyTo) -> M + Send + 'static, + { + let (state, slot, token) = self.link_reply(&opts, token)?; + let hold = Hold::Linked(LinkedJob(slot)); + let cell = Arc::new(Mutex::new(Some(token))); + let shared = cell.clone(); + let done: Done = Box::new(move |outcome| { + let token = shared + .lock() + .unwrap_or_else(|p| p.into_inner()) + .take() + .expect("the token is taken once, by done or by a spawn failure"); + done(outcome, token) + }); + self.launch_blocking(opts, state, hold, work, done) + .map_err(|kind| { + let token = cell + .lock() + .unwrap_or_else(|p| p.into_inner()) + .take() + .expect("done never ran"); + PipeReplyError::Failed { token, kind } + }) + } + + /// Checks shared by the `pipe_reply*` calls: owner, owner exit, and the + /// attach CAS on the token's slot (which reserves a credit only when a + /// linked job already runs). + #[allow(clippy::type_complexity)] + fn link_reply( + &self, + opts: &JobOptions, + token: ReplyTo, + ) -> Result<(Arc, Arc>, ReplyTo), PipeReplyError> { + if token.owner() != self.id() { + return Err(PipeReplyError::WrongOwner(token)); + } + let state = match self.admit(opts) { + Ok(state) => state, + Err(kind) => return Err(PipeReplyError::Failed { token, kind }), + }; + let slot = token.slot_arc().clone(); + match slot.attach(&now_ticks) { + Ok(()) => Ok((state, slot, token)), + Err(Refused::Closed) => Err(PipeReplyError::Closed(token)), + Err(Refused::Overloaded) => Err(PipeReplyError::Failed { + token, + kind: PipeErrorKind::Overloaded, + }), + } + } +} + diff --git a/concurrency/src/threads/mod.rs b/concurrency/src/threads/mod.rs index ee1e94b1..cffef0be 100644 --- a/concurrency/src/threads/mod.rs +++ b/concurrency/src/threads/mod.rs @@ -13,5 +13,13 @@ pub use actor::{ request, send_message_on, Actor, ActorRef, ActorStart, Context, Handler, Receiver, Recipient, DEFAULT_REQUEST_TIMEOUT, }; +pub use actor::{DEFAULT_STOP_DEADLINE}; +pub use crate::reply::jobs::{ + JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, PipeError, PipeErrorKind, + PipeReplyError, +}; +pub use crate::reply::{ + Deadline, DeferError, Delivery, PendingReply, ReplyReceiver, ReplyTo, DEFAULT_MAX_OUTSTANDING, +}; pub use stream::spawn_listener; pub use time::{send_after, send_interval, TimerHandle}; From 9478d3771ae3364f473dff445cb439b458869a7b Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:44:05 -0300 Subject: [PATCH 024/208] feat(jobs): threads-mode pipe_blocking_with and pipe_reply_blocking_with on owned native threads --- concurrency/src/tasks/actor.rs | 2 +- concurrency/src/threads/actor.rs | 37 +++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 335ccca1..cc8d1b2d 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -8,7 +8,7 @@ use crate::mailbox::tasks as mpsc; use crate::message::Message; use crate::monitor::{Down, MonitorCleanup, MonitorRef, MonitorTable}; use crate::reply::{registry::Registry, reply_slot, SlotHandle}; -use crate::reply::{PendingReply, ReplyReceiver, DEFAULT_MAX_OUTSTANDING}; +use crate::reply::{Deadline, PendingReply, ReplyReceiver, DEFAULT_MAX_OUTSTANDING}; use core::pin::pin; use futures::future::{self, FutureExt as _}; use spawned_rt::{ diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index c5a7d8ec..48cb12a7 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -17,11 +17,19 @@ use crate::link::{ use crate::message::Message; use crate::monitor::{Down, MonitorCleanup, MonitorRef, MonitorTable}; use crate::reply::{registry::Registry, reply_slot, SlotHandle}; -use crate::reply::{PendingReply, ReplyReceiver, DEFAULT_MAX_OUTSTANDING}; +use crate::reply::{Deadline, PendingReply, ReplyReceiver, DEFAULT_MAX_OUTSTANDING}; use std::sync::atomic::Ordering; pub use crate::response::DEFAULT_REQUEST_TIMEOUT; +/// Cleanup deadline for actors without a supervisor: the bound on joining an +/// exiting actor's cancelled jobs. +pub const DEFAULT_STOP_DEADLINE: Duration = Duration::from_secs(5); + +#[path = "jobs.rs"] +mod jobs; +pub use jobs::JobCtx; + // --------------------------------------------------------------------------- // Actor trait // --------------------------------------------------------------------------- @@ -38,6 +46,10 @@ pub trait Actor: Send + Sized + 'static { /// Capped at 65535. const MAX_OUTSTANDING: usize = DEFAULT_MAX_OUTSTANDING; + /// Native threads this instance's jobs may run at once. `None`: + /// `min(4, available_parallelism)`. + const MAX_NATIVE_JOBS: Option = None; + fn started(&mut self, _ctx: &Context) {} fn stopped(&mut self, _ctx: &Context) {} @@ -186,6 +198,7 @@ pub struct Context { linked_reason: LinkedExitReason, mailbox: crate::instrumentation::Mailbox, replies: Arc, + jobs: Arc, } impl Clone for Context { @@ -201,6 +214,7 @@ impl Clone for Context { linked_reason: self.linked_reason.clone(), mailbox: self.mailbox.clone(), replies: self.replies.clone(), + jobs: self.jobs.clone(), } } } @@ -226,6 +240,7 @@ impl Context { linked_reason: actor_ref.linked_reason.clone(), mailbox: actor_ref.mailbox.clone(), replies: actor_ref.replies.clone(), + jobs: actor_ref.jobs.clone(), } } @@ -352,6 +367,7 @@ impl Context { linked_reason: self.linked_reason.clone(), mailbox: self.mailbox.clone(), replies: self.replies.clone(), + jobs: self.jobs.clone(), } } @@ -613,6 +629,7 @@ pub struct ActorRef { linked_reason: LinkedExitReason, mailbox: crate::instrumentation::Mailbox, replies: Arc, + jobs: Arc, } impl Debug for ActorRef { @@ -634,6 +651,7 @@ impl Clone for ActorRef { linked_reason: self.linked_reason.clone(), mailbox: self.mailbox.clone(), replies: self.replies.clone(), + jobs: self.jobs.clone(), } } } @@ -828,6 +846,10 @@ impl ActorRef { linked_reason: linked_reason.clone(), mailbox: crate::instrumentation::Mailbox::default(), replies: Registry::new(id, A::MAX_OUTSTANDING), + jobs: Arc::new(crate::reply::jobs::JobSet::new( + id, + A::MAX_NATIVE_JOBS.unwrap_or_else(crate::reply::jobs::default_max_native_jobs), + )), }; let ctx = Context { @@ -841,6 +863,7 @@ impl ActorRef { linked_reason: linked_reason.clone(), mailbox: actor_ref.mailbox.clone(), replies: actor_ref.replies.clone(), + jobs: actor_ref.jobs.clone(), }; if let Some(parent) = parent { @@ -861,6 +884,7 @@ impl ActorRef { let startup = crate::actor_cleanup::ActorStartup::new((actor, ctx, rx, cancellation_token), cleanup); let replies = actor_ref.replies.clone(); + let jobs = actor_ref.jobs.clone(); let _thread_handle = rt::spawn(move || { let ((mut actor, ctx, mut rx, cancellation_token), mut cleanup) = startup.into_parts(); let primary = match catch_unwind(AssertUnwindSafe(|| { @@ -887,6 +911,13 @@ impl ActorRef { Ok(reason) => cleanup.complete(reason), Err(reason) => cleanup.complete_secondary(reason), } + // Job cleanup (§5 step 4), after death: join the cancelled jobs + // that signal completion, bounded by the cleanup deadline. + if jobs.running() > 0 { + let unfinished = + jobs.join_blocking(std::time::Instant::now() + DEFAULT_STOP_DEADLINE); + crate::reply::jobs::report_unfinished(jobs.owner(), unfinished); + } }); actor_ref @@ -933,6 +964,7 @@ fn run_actor_receiving( if cancellation_token.is_cancelled() { ctx.replies.exiting(); + ctx.jobs.cancel_all(); if let Err(panic) = catch_unwind(AssertUnwindSafe(|| { fail_point!("actor.before_stopped"); actor.stopped(&ctx) @@ -1002,6 +1034,7 @@ fn run_actor_receiving( cancellation_token.cancel(); ctx.replies.exiting(); + ctx.jobs.cancel_all(); let stop_result = catch_unwind(AssertUnwindSafe(|| { fail_point!("actor.before_stopped"); actor.stopped(&ctx); @@ -1931,6 +1964,7 @@ mod idle_stop_regression { linked_reason: new_linked_exit_reason(), mailbox: crate::instrumentation::Mailbox::default(), replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), + jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), }; let count = Arc::new(AtomicUsize::new(0)); let seen = count.clone(); @@ -2053,6 +2087,7 @@ mod stop_wakeup_regression { linked_reason: new_linked_exit_reason(), mailbox: crate::instrumentation::Mailbox::default(), replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), + jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), }; let (entered_tx, entered) = mpsc::channel(); let (stopped_tx, stopped) = mpsc::channel(); From e256f90fb62762c4b613c9ce1b5edf333e18bb94 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:46:27 -0300 Subject: [PATCH 025/208] test: pipe-to-self jobs (results, panics, keyed fencing, deadlines, emits, delegation, bounds) and registry unit tests --- concurrency/src/reply/deadline.rs | 1 + concurrency/src/reply/registry.rs | 71 ++++ concurrency/src/reply/slot.rs | 1 + concurrency/src/threads/jobs.rs | 2 +- concurrency/src/threads/mod.rs | 2 +- concurrency/tests/deferred_jobs.rs | 522 +++++++++++++++++++++++++++++ 6 files changed, 597 insertions(+), 2 deletions(-) create mode 100644 concurrency/tests/deferred_jobs.rs diff --git a/concurrency/src/reply/deadline.rs b/concurrency/src/reply/deadline.rs index f393368f..0ff5b9d8 100644 --- a/concurrency/src/reply/deadline.rs +++ b/concurrency/src/reply/deadline.rs @@ -89,6 +89,7 @@ impl DeadlineHeap { self.heap.peek().map(|Reverse(e)| e.at) } + #[cfg(test)] pub(crate) fn len(&self) -> usize { self.heap.len() } diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index 22c9a1dc..f3460960 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -308,3 +308,74 @@ impl Drop for Registry { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::reply::slot::Slot; + + fn deferred(registry: &Arc, deadline: u64) -> Arc> { + let slot = Arc::new(Slot::new(deadline)); + slot.defer(registry.pool(), &|| 0).unwrap(); + registry.register(&slot); + registry.track_deadline(&slot); + slot + } + + #[test] + fn turn_drops_commit_dropped_or_stopped() { + let registry = Registry::new(ActorId::next(), 8); + for (panicked, expected) in [(false, Terminal::Dropped), (true, Terminal::Stopped)] { + let slot = deferred(®istry, NEVER); + registry.begin_turn(); + registry.dropped(slot.clone()); + assert!(!slot.is_terminal(), "an in-turn drop waits for the turn end"); + registry.end_turn(panicked); + assert_eq!(Terminal::of(crate::reply::slot::state(slot.word())), Some(expected)); + } + // Between turns a drop answers at once; while exiting, Stopped. + let slot = deferred(®istry, NEVER); + registry.dropped(slot.clone()); + assert_eq!(Terminal::of(crate::reply::slot::state(slot.word())), Some(Terminal::Dropped)); + registry.exiting(); + let slot = deferred(®istry, NEVER); + registry.dropped(slot.clone()); + assert_eq!(Terminal::of(crate::reply::slot::state(slot.word())), Some(Terminal::Stopped)); + assert_eq!(registry.outstanding(), 0); + } + + #[test] + fn close_stops_every_listed_open_token_once() { + let registry = Registry::new(ActorId::next(), 8); + let open = deferred(®istry, NEVER); + let answered = deferred(®istry, NEVER); + answered.reply(1, &|| 0); + assert_eq!(registry.close(), 1); + assert_eq!(Terminal::of(crate::reply::slot::state(open.word())), Some(Terminal::Stopped)); + assert_eq!(registry.outstanding(), 0); + assert_eq!(registry.heap_entries(), 0, "the heap is dropped at death"); + } + + #[test] + fn heap_stays_bounded_under_shortening_churn() { + let registry = Registry::new(ActorId::next(), 64); + let far = u64::MAX / 2; + let slots: Vec<_> = (0..32).map(|i| deferred(®istry, far + i)).collect(); + for round in 0..200u64 { + for slot in &slots { + // Each shortening supersedes the previous entry (stale). + slot.shorten(far - round - 1, &|| 0); + registry.track_deadline(slot); + } + assert!( + registry.heap_entries() <= 2 * slots.len() + 16 + slots.len(), + "heap grew to {} entries for {} live tokens", + registry.heap_entries(), + slots.len() + ); + } + // Expire them all: the service visit empties the heap of due entries. + assert_eq!(registry.service_deadlines(u64::MAX - 2), None); + assert_eq!(registry.outstanding(), 0); + } +} diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 52344c16..82d1cc81 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -156,6 +156,7 @@ impl CreditPool { pub(crate) fn used(&self) -> usize { self.used.load(Ordering::Acquire) } + #[cfg_attr(not(test), allow(dead_code))] // used by the Loom model pub(crate) fn max(&self) -> usize { self.max } diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index 0d9074d8..b998fdf8 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -10,7 +10,7 @@ use super::{Actor, Context, Envelope, Handler, MailboxItem}; use crate::message::Message; use crate::reply::deadline::Expirable; use crate::reply::jobs::{ - JobCredit, JobDeadline, JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, + JobCredit, JobDeadline, JobError, JobGone, JobHandle, JobOptions, JobOutcome, JobState, PipeError, PipeErrorKind, PipeReplyError, }; use crate::reply::slot::{Refused, Slot}; diff --git a/concurrency/src/threads/mod.rs b/concurrency/src/threads/mod.rs index cffef0be..9a62fad3 100644 --- a/concurrency/src/threads/mod.rs +++ b/concurrency/src/threads/mod.rs @@ -13,7 +13,7 @@ pub use actor::{ request, send_message_on, Actor, ActorRef, ActorStart, Context, Handler, Receiver, Recipient, DEFAULT_REQUEST_TIMEOUT, }; -pub use actor::{DEFAULT_STOP_DEADLINE}; +pub use actor::{JobCtx, DEFAULT_STOP_DEADLINE}; pub use crate::reply::jobs::{ JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, PipeError, PipeErrorKind, PipeReplyError, diff --git a/concurrency/tests/deferred_jobs.rs b/concurrency/tests/deferred_jobs.rs new file mode 100644 index 00000000..bd27d7a1 --- /dev/null +++ b/concurrency/tests/deferred_jobs.rs @@ -0,0 +1,522 @@ +//! Pipe-to-self jobs (docs/design/deferred-replies.md §9): results as +//! messages, panics, keyed replacement and fencing, cancellation, job +//! deadlines, emits, delegation with `pipe_reply`, bounds and owner exit. +use spawned_concurrency::message::Message; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +/// What the owner's handlers observed, in dispatch order. +#[derive(Default)] +pub struct Log(pub Mutex>); +impl Log { + pub fn push(&self, entry: impl Into) { + self.0.lock().unwrap().push(entry.into()); + } + pub fn take(&self) -> Vec { + std::mem::take(&mut self.0.lock().unwrap()) + } +} + +#[cfg(feature = "async")] +mod tasks_mode { + use super::*; + use spawned_concurrency::tasks::{ + Actor, ActorRef, ActorStart, Context, Deadline, Handler, JobError, JobGone, JobOptions, + JobOutcome, PendingReply, PipeErrorKind, PipeReplyError, ReplyTo, + }; + use spawned_concurrency::ActorError; + use spawned_rt::tasks::{oneshot, sleep}; + + struct Owner { + log: Arc, + gates: Vec>, + } + impl Actor for Owner { + const MAX_OUTSTANDING: usize = 4; + const MAX_NATIVE_JOBS: Option = Some(1); + } + + struct Done(String); + impl Message for Done { + type Result = (); + } + impl Handler for Owner { + async fn handle(&mut self, msg: Done, _: PendingReply<'_, ()>, _: &Context) { + self.log.push(msg.0); + } + } + + struct Open; + impl Message for Open { + type Result = (); + } + impl Handler for Owner { + async fn handle(&mut self, _: Open, reply: PendingReply<'_, ()>, _: &Context) { + for gate in self.gates.drain(..) { + let _ = gate.send(()); + } + reply.reply(()); + } + } + + struct Outstanding; + impl Message for Outstanding { + type Result = usize; + } + impl Handler for Owner { + async fn handle(&mut self, _: Outstanding, reply: PendingReply<'_, usize>, ctx: &Context) { + reply.reply(ctx.deferred_outstanding()); + } + } + + fn describe(name: &str, outcome: JobOutcome) -> Done { + Done(match outcome { + JobOutcome::Ok(v) => format!("{name}:ok:{v:?}"), + JobOutcome::Err(JobError::Panicked(m)) => format!("{name}:panicked:{m}"), + JobOutcome::Err(JobError::DeadlineExceeded) => format!("{name}:deadline"), + }) + } + + /// Start jobs from inside a turn, as handlers do. + enum Start { + Pipe(&'static str, u64), + Panics, + Gated(&'static str, Option), + Keyed(&'static str), + CancelAfterSpawn, + Blocking(u32), + NativeBudget, + Emits(&'static str, u32), + } + impl Message for Start { + type Result = Result<(), String>; + } + impl Handler for Owner { + async fn handle( + &mut self, + msg: Start, + reply: PendingReply<'_, Result<(), String>>, + ctx: &Context, + ) { + let result = match msg { + Start::Pipe(name, v) => ctx + .pipe(async move { v * 2 }, move |o| describe(name, o)) + .map(drop) + .map_err(|e| e.to_string()), + Start::Panics => ctx + .pipe(async { panic!("job boom") }, |o: JobOutcome<()>| { + describe("p", o) + }) + .map(drop) + .map_err(|e| e.to_string()), + Start::Gated(name, deadline) => { + let (open, gate) = oneshot::channel::<()>(); + self.gates.push(open); + let opts = JobOptions { + deadline: deadline.map(|d| spawned_rt::time::Instant::now() + d), + ..JobOptions::default() + }; + ctx.pipe_with(opts, async move { gate.await.is_ok() }, move |o| { + describe(name, o) + }) + .map(drop) + .map_err(|e| e.to_string()) + } + Start::Keyed(name) => { + let (open, gate) = oneshot::channel::<()>(); + self.gates.push(open); + ctx.pipe_keyed("k", async move { gate.await.is_ok() }, move |o| { + describe(name, o) + }) + .map(drop) + .map_err(|e| e.to_string()) + } + Start::CancelAfterSpawn => { + let (open, gate) = oneshot::channel::<()>(); + self.gates.push(open); + ctx.pipe(async move { gate.await.is_ok() }, |o| describe("cancelled", o)) + .map(|job| job.cancel()) + .map_err(|e| e.to_string()) + } + Start::Blocking(v) => ctx + .pipe_blocking_with(JobOptions::default(), move |_| v + 1, |o| { + describe("blocking", o) + }) + .map(drop) + .map_err(|e| e.to_string()), + Start::NativeBudget => { + let (hold, held) = std::sync::mpsc::channel::<()>(); + let first = ctx.pipe_blocking_with( + JobOptions::default(), + move |_| { + let _ = held.recv(); + }, + |o| describe("first", o), + ); + let second = ctx.pipe_blocking_with(JobOptions::default(), |_| (), |o| { + describe("second", o) + }); + drop(hold); + match (&first, &second) { + (Ok(_), Err(e)) if matches!(e.kind, PipeErrorKind::NativeBudget) => Ok(()), + _ => Err(format!("{first:?} / {second:?}")), + } + } + Start::Emits(name, n) => ctx + .pipe_blocking_with( + JobOptions { + key: Some("emits".into()), + emit_credits: 2, + ..JobOptions::default() + }, + move |job| { + for i in 0..n { + job.emit_blocking(Done(format!("{name}:emit:{i}")))?; + } + Ok::(n) + }, + move |o| describe(name, o), + ) + .map(drop) + .map_err(|e| e.to_string()), + }; + reply.reply(result); + } + } + + fn start() -> (ActorRef, Arc) { + let log = Arc::new(Log::default()); + let actor = Owner { + log: log.clone(), + gates: Vec::new(), + } + .start(); + (actor, log) + } + + /// Wait until the log has `n` entries (jobs report asynchronously). + async fn entries(actor: &ActorRef, log: &Log, n: usize) -> Vec { + for _ in 0..500 { + actor.request(Outstanding).await.unwrap(); + if log.0.lock().unwrap().len() >= n { + return log.take(); + } + sleep(Duration::from_millis(5)).await; + } + panic!("log never reached {n} entries: {:?}", log.take()); + } + + async fn stop(actor: ActorRef) { + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn job_results_and_panics_arrive_as_one_message() { + let (actor, log) = start(); + actor.request(Start::Pipe("a", 21)).await.unwrap().unwrap(); + assert_eq!(entries(&actor, &log, 1).await, ["a:ok:42"]); + actor.request(Start::Panics).await.unwrap().unwrap(); + assert_eq!(entries(&actor, &log, 1).await, ["p:panicked:job boom"]); + assert_eq!(actor.request(Outstanding).await.unwrap(), 0); + stop(actor).await; + } + + #[spawned_macros::sim_test] + async fn keyed_replacement_and_cancel_fence_results() { + let (actor, log) = start(); + actor.request(Start::Keyed("old")).await.unwrap().unwrap(); + actor.request(Start::Keyed("new")).await.unwrap().unwrap(); + actor.request(Start::CancelAfterSpawn).await.unwrap().unwrap(); + actor.request(Open).await.unwrap(); + // Only the replacing job reports; the replaced and cancelled ones are + // fenced even though their work finished. + assert_eq!(entries(&actor, &log, 1).await, ["new:ok:true"]); + sleep(Duration::from_millis(20)).await; + assert!(log.take().is_empty()); + assert_eq!(actor.request(Outstanding).await.unwrap(), 0); + stop(actor).await; + } + + #[spawned_macros::sim_test] + async fn job_deadline_notifies_while_the_work_keeps_its_credit() { + let (actor, log) = start(); + actor + .request(Start::Gated("slow", Some(Duration::from_millis(20)))) + .await + .unwrap() + .unwrap(); + assert_eq!(entries(&actor, &log, 1).await, ["slow:deadline"]); + assert_eq!( + actor.request(Outstanding).await.unwrap(), + 1, + "the hung work still holds its credit" + ); + actor.request(Open).await.unwrap(); + for _ in 0..200 { + if actor.request(Outstanding).await.unwrap() == 0 { + break; + } + sleep(Duration::from_millis(5)).await; + } + assert_eq!(actor.request(Outstanding).await.unwrap(), 0); + assert!(log.take().is_empty(), "the late result is dropped"); + stop(actor).await; + } + + #[spawned_macros::sim_test] + async fn pipe_rejects_at_the_credit_bound() { + let (actor, _) = start(); + for _ in 0..4 { + actor.request(Start::Gated("g", None)).await.unwrap().unwrap(); + } + let refused = actor.request(Start::Gated("g", None)).await.unwrap(); + assert!(refused.unwrap_err().contains("deferred credits")); + actor.request(Open).await.unwrap(); + stop(actor).await; + } + + #[cfg(not(spawned_sim))] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn blocking_jobs_emit_in_order_and_respect_the_native_budget() { + let (actor, log) = start(); + actor.request(Start::Blocking(1)).await.unwrap().unwrap(); + assert_eq!(entries(&actor, &log, 1).await, ["blocking:ok:2"]); + actor.request(Start::NativeBudget).await.unwrap().unwrap(); + assert_eq!(entries(&actor, &log, 1).await, ["first:ok:()"]); + actor.request(Start::Emits("e", 5)).await.unwrap().unwrap(); + // Emits are FIFO and the terminal message is last. + assert_eq!( + entries(&actor, &log, 6).await, + ["e:emit:0", "e:emit:1", "e:emit:2", "e:emit:3", "e:emit:4", "e:ok:Ok(5)"] + ); + stop(actor).await; + } + + // -- Delegation ----------------------------------------------------------- + + struct Backend; + impl Actor for Backend {} + struct Lookup(u64); + impl Message for Lookup { + type Result = u64; + } + impl Handler for Backend { + async fn handle(&mut self, msg: Lookup, reply: PendingReply<'_, u64>, _: &Context) { + sleep(Duration::from_millis(msg.0)).await; + reply.reply(msg.0 * 10); + } + } + + struct Router { + backend: ActorRef, + } + impl Actor for Router {} + struct Route(u64); + impl Message for Route { + type Result = u64; + } + struct Delegated { + res: JobOutcome>, + token: ReplyTo, + } + impl Message for Delegated { + type Result = (); + } + struct Ping; + impl Message for Ping { + type Result = &'static str; + } + struct Foreign(ReplyTo); + impl Message for Foreign { + type Result = bool; + } + impl Handler for Router { + async fn handle(&mut self, msg: Route, reply: PendingReply<'_, u64>, ctx: &Context) { + let Ok(token) = reply.defer() else { return }; + let downstream = self + .backend + .request_raw_until(Lookup(msg.0), token.deadline().map_or(Deadline::Never, Deadline::At)); + let work = async move { + match downstream { + Ok(rx) => rx.await, + Err(e) => Err(e), + } + }; + if let Err(e) = ctx.pipe_reply(token, work, |res, token| Delegated { res, token }) { + drop(e.into_token()); + } + } + } + impl Handler for Router { + async fn handle(&mut self, msg: Delegated, _: PendingReply<'_, ()>, _: &Context) { + match msg.res { + JobOutcome::Ok(Ok(v)) => { + msg.token.reply(v); + } + _ => msg.token.overloaded(), + } + } + } + impl Handler for Router { + async fn handle(&mut self, _: Ping, reply: PendingReply<'_, &'static str>, _: &Context) { + reply.reply("pong"); + } + } + impl Handler for Router { + async fn handle(&mut self, msg: Foreign, reply: PendingReply<'_, bool>, ctx: &Context) { + let refused = ctx.pipe_reply(msg.0, async { 1u64 }, |_, token| Delegated { + res: JobOutcome::Ok(Ok(0)), + token, + }); + let wrong_owner = matches!(refused, Err(PipeReplyError::WrongOwner(_))); + if let Err(e) = refused { + e.into_token().reply(7); + } + reply.reply(wrong_owner); + } + } + + /// Holds a deferred token so a test can hand it to another actor. + struct Holder(Option>); + impl Actor for Holder {} + struct Hold; + impl Message for Hold { + type Result = u64; + } + struct Give; + impl Message for Give { + type Result = Option>; + } + impl Handler for Holder { + async fn handle(&mut self, _: Hold, reply: PendingReply<'_, u64>, _: &Context) { + self.0 = reply.defer().ok(); + } + } + impl Handler for Holder { + async fn handle(&mut self, _: Give, reply: PendingReply<'_, Option>>, _: &Context) { + let token = self.0.take(); + reply.reply(token); + } + } + + #[spawned_macros::sim_test] + async fn router_delegates_with_pipe_reply_while_serving_its_mailbox() { + let backend = Backend.start(); + let router = Router { + backend: backend.clone(), + } + .start(); + let slow = router.request_raw(Route(200)).unwrap(); + // The router's mailbox is free while the slow request is delegated. + assert_eq!(router.request(Ping).await.unwrap(), "pong"); + assert_eq!(router.request(Route(1)).await.unwrap(), 10); + assert_eq!(slow.await.unwrap(), 2000); + router.context().stop(); + router.join().await; + backend.context().stop(); + backend.join().await; + } + + #[spawned_macros::sim_test] + async fn pipe_reply_refuses_a_token_of_another_actor() { + let backend = Backend.start(); + let router = Router { + backend: backend.clone(), + } + .start(); + let holder = Holder(None).start(); + let rx = holder.request_raw_until(Hold, Deadline::Never).unwrap(); + let token = holder.request(Give).await.unwrap().unwrap(); + assert!(router.request(Foreign(token)).await.unwrap()); + assert_eq!(rx.await.unwrap(), 7, "the refused token came back and was answered"); + for actor in [router.child_handle(), backend.child_handle(), holder.child_handle()] { + actor.stop(); + } + router.join().await; + } +} + +#[cfg(not(spawned_sim))] +mod threads_mode { + use super::*; + use spawned_concurrency::threads::{ + Actor, ActorRef, ActorStart, Context, Deadline, Handler, JobError, JobOptions, JobOutcome, + PendingReply, + }; + + struct Owner { + log: Arc, + } + impl Actor for Owner {} + + struct Done(String); + impl Message for Done { + type Result = (); + } + impl Handler for Owner { + fn handle(&mut self, msg: Done, _: PendingReply<'_, ()>, _: &Context) { + self.log.push(msg.0); + } + } + struct Compute(u64); + impl Message for Compute { + type Result = u64; + } + struct Computed { + res: JobOutcome, + token: spawned_concurrency::ReplyTo, + } + impl Message for Computed { + type Result = (); + } + impl Handler for Owner { + fn handle(&mut self, msg: Compute, reply: PendingReply<'_, u64>, ctx: &Context) { + let Ok(token) = reply.defer() else { return }; + let n = msg.0; + let _ = ctx.pipe_reply_blocking_with( + JobOptions::default(), + token, + move |job| { + job.emit_blocking(Done(format!("computing {n}"))).unwrap(); + if n == 0 { + panic!("zero"); + } + n * n + }, + |res, token| Computed { res, token }, + ); + } + } + impl Handler for Owner { + fn handle(&mut self, msg: Computed, _: PendingReply<'_, ()>, _: &Context) { + match msg.res { + JobOutcome::Ok(v) => { + msg.token.reply(v); + } + JobOutcome::Err(JobError::Panicked(m)) => { + self.log.push(format!("panicked {m}")); + msg.token.overloaded(); + } + JobOutcome::Err(JobError::DeadlineExceeded) => msg.token.overloaded(), + } + } + } + + #[test] + fn native_jobs_answer_deferred_requests() { + let log = Arc::new(Log::default()); + let actor: ActorRef = Owner { log: log.clone() }.start(); + let rx = actor.request_raw_until(Compute(7), Deadline::Never).unwrap(); + assert_eq!(rx.recv().unwrap(), 49); + assert!(matches!( + actor.request(Compute(0)), + Err(spawned_concurrency::ActorError::Overloaded) + )); + let entries = log.take(); + assert_eq!(entries[0], "computing 7"); + assert!(entries.contains(&"panicked zero".to_string()), "{entries:?}"); + actor.context().stop(); + actor.join(); + } +} From 0142c5a7532fd18380ac9ea9cd96a9b18c0f9b24 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:47:30 -0300 Subject: [PATCH 026/208] feat(reply): PendingReplies bounded token table --- concurrency/src/lib.rs | 4 +- concurrency/src/reply/mod.rs | 2 + concurrency/src/reply/pending.rs | 137 +++++++++++++++++++++++++++++++ concurrency/src/tasks/mod.rs | 3 +- concurrency/src/threads/mod.rs | 3 +- 5 files changed, 146 insertions(+), 3 deletions(-) create mode 100644 concurrency/src/reply/pending.rs diff --git a/concurrency/src/lib.rs b/concurrency/src/lib.rs index db02f7a5..a9943cce 100644 --- a/concurrency/src/lib.rs +++ b/concurrency/src/lib.rs @@ -110,7 +110,9 @@ pub use child_handle::{ActorId, ChildHandle}; pub use error::{ActorError, ExitReason}; pub use link::Exit; pub use monitor::{Down, MonitorRef}; -pub use reply::{Deadline, DeferError, Delivery, PendingReply, ReplyReceiver, ReplyTo}; +pub use reply::{ + Deadline, DeferError, Delivery, InsertError, PendingReplies, PendingReply, ReplyReceiver, ReplyTo, +}; pub use response::Response; pub use spawned_macros::{actor, protocol}; diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index e34329b2..2a06b13f 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -8,10 +8,12 @@ pub(crate) mod deadline; pub mod jobs; +mod pending; pub(crate) mod registry; pub(crate) mod slot; pub use registry::DEFAULT_MAX_OUTSTANDING; +pub use pending::{InsertError, PendingReplies}; pub use slot::Delivery; use crate::error::ActorError; diff --git a/concurrency/src/reply/pending.rs b/concurrency/src/reply/pending.rs new file mode 100644 index 00000000..2881d1c0 --- /dev/null +++ b/concurrency/src/reply/pending.rs @@ -0,0 +1,137 @@ +//! `PendingReplies`: a bounded table of deferred tokens by key +//! (docs/design/deferred-replies.md §7). + +use super::ReplyTo; +use std::collections::HashMap; +use std::hash::Hash; + +/// Why [`PendingReplies::insert`] handed the token back. The handler decides +/// the answer (typically `token.overloaded()` or a domain error) and still +/// owns the token. +#[derive(Debug)] +pub enum InsertError { + /// A live token is already stored under this key; it is kept. + Duplicate(T), + /// The table holds `cap` live tokens. + Full(T), +} + +impl InsertError { + pub fn into_inner(self) -> T { + match self { + Self::Duplicate(token) | Self::Full(token) => token, + } + } +} + +/// Deferred tokens by key, capped: retained entries never exceed `cap`, +/// however fast tokens expire. Terminal tokens (answered, expired, abandoned) +/// are purged on every insert and skipped by `take`. +pub struct PendingReplies { + cap: usize, + tokens: HashMap>, +} + +impl std::fmt::Debug for PendingReplies { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PendingReplies") + .field("cap", &self.cap) + .field("retained", &self.tokens.len()) + .finish() + } +} + +impl PendingReplies { + pub fn with_capacity(cap: usize) -> Self { + Self { + cap, + tokens: HashMap::new(), + } + } + + /// Purge terminal tokens, then insert. A duplicate key (with a live + /// token) or a full table hands the new token back. + pub fn insert(&mut self, key: K, token: ReplyTo) -> Result<(), InsertError>> { + self.purge(); + if self.tokens.contains_key(&key) { + return Err(InsertError::Duplicate(token)); + } + if self.tokens.len() >= self.cap { + return Err(InsertError::Full(token)); + } + self.tokens.insert(key, token); + Ok(()) + } + + /// Remove and return the live token under `key`; a terminal one is + /// removed and skipped. + pub fn take(&mut self, key: &K) -> Option> { + let token = self.tokens.remove(key)?; + (!token.is_closed()).then_some(token) + } + + /// Remove every terminal token; returns how many were removed. + pub fn purge(&mut self) -> usize { + let before = self.tokens.len(); + self.tokens.retain(|_, token| !token.is_closed()); + before - self.tokens.len() + } + + /// Entries held, without purging. + pub fn retained(&self) -> usize { + self.tokens.len() + } + + /// Non-terminal entries (a scan). + pub fn live(&self) -> usize { + self.tokens.values().filter(|t| !t.is_closed()).count() + } + + /// Take every live token (for example to answer them in `stopped()`). + pub fn drain(&mut self) -> impl Iterator)> + '_ { + self.tokens.drain().filter(|(_, token)| !token.is_closed()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::reply::{reply_pair, Deadline}; + use crate::ActorError; + + fn token() -> (ReplyTo, crate::reply::ReplyReceiver) { + let (reply, rx) = reply_pair::(Deadline::Never); + (reply.defer().unwrap(), rx) + } + + #[test] + fn bounded_with_duplicates_and_purge() { + let mut table = PendingReplies::with_capacity(2); + let (a, ra) = token(); + let (b, rb) = token(); + let (c, rc) = token(); + let (dup, rdup) = token(); + table.insert(1, a).unwrap(); + table.insert(2, b).unwrap(); + let full = table.insert(3, c).unwrap_err(); + assert!(matches!(full, InsertError::Full(_))); + full.into_inner().overloaded(); + assert!(matches!(rc.recv(), Err(ActorError::Overloaded))); + let duplicate = table.insert(1, dup).unwrap_err(); + assert!(matches!(duplicate, InsertError::Duplicate(_))); + drop(duplicate); + assert!(matches!(rdup.recv(), Err(ActorError::ReplyDropped))); + // A caller that left frees its entry on the next insert. + drop(rb); + assert_eq!(table.retained(), 2); + assert_eq!(table.live(), 1); + let (d, rd) = token(); + table.insert(4, d).unwrap(); + assert_eq!(table.retained(), 2); + assert!(table.take(&2).is_none()); + table.take(&1).unwrap().reply(10); + table.take(&4).unwrap().reply(40); + assert_eq!(ra.recv().unwrap(), 10); + assert_eq!(rd.recv().unwrap(), 40); + } +} diff --git a/concurrency/src/tasks/mod.rs b/concurrency/src/tasks/mod.rs index e12c35f5..49e5a78d 100644 --- a/concurrency/src/tasks/mod.rs +++ b/concurrency/src/tasks/mod.rs @@ -18,7 +18,8 @@ pub use crate::reply::jobs::{ PipeReplyError, }; pub use crate::reply::{ - Deadline, DeferError, Delivery, PendingReply, ReplyReceiver, ReplyTo, DEFAULT_MAX_OUTSTANDING, + Deadline, DeferError, Delivery, InsertError, PendingReplies, PendingReply, ReplyReceiver, + ReplyTo, DEFAULT_MAX_OUTSTANDING, }; pub use stream::spawn_listener; pub use time::{send_after, send_interval, TimerHandle}; diff --git a/concurrency/src/threads/mod.rs b/concurrency/src/threads/mod.rs index 9a62fad3..205490eb 100644 --- a/concurrency/src/threads/mod.rs +++ b/concurrency/src/threads/mod.rs @@ -19,7 +19,8 @@ pub use crate::reply::jobs::{ PipeReplyError, }; pub use crate::reply::{ - Deadline, DeferError, Delivery, PendingReply, ReplyReceiver, ReplyTo, DEFAULT_MAX_OUTSTANDING, + Deadline, DeferError, Delivery, InsertError, PendingReplies, PendingReply, ReplyReceiver, + ReplyTo, DEFAULT_MAX_OUTSTANDING, }; pub use stream::spawn_listener; pub use time::{send_after, send_interval, TimerHandle}; From 7e272fe96659263e8d26c0d686b244ec85e65237 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:48:36 -0300 Subject: [PATCH 027/208] chore: cfg-gate items unused in sim or thread-only builds --- concurrency/src/reply/registry.rs | 2 ++ concurrency/src/reply/slot.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index f3460960..d548024a 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -69,6 +69,7 @@ struct Lists { pub(crate) struct Registry { owner: ActorId, /// Service key, unique per registry. + #[cfg_attr(all(spawned_sim, feature = "async"), allow(dead_code))] key: u64, pool: Arc, phase: AtomicU8, @@ -133,6 +134,7 @@ impl Registry { lists.tokens.push(weak); } + #[cfg_attr(all(spawned_sim, feature = "async"), allow(dead_code))] pub(crate) fn key(&self) -> u64 { self.key } diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 82d1cc81..02644f17 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -504,6 +504,7 @@ impl Slot { /// `Pending` while the slot is open. The caller's timer expires the slot /// at its deadline (`ReplyReceiver` sleeps until it), so the poll itself /// reads no clock. + #[cfg_attr(not(feature = "async"), allow(dead_code))] pub(crate) fn poll(&self, waker: &Waker) -> Poll> { loop { if let Some(result) = self.take(state(self.word())) { @@ -529,6 +530,7 @@ impl Slot { /// Token side: ready once the slot is terminal (or its deadline passed, /// which expires it). Registers `waker` as the watcher otherwise. + #[cfg_attr(not(feature = "async"), allow(dead_code))] pub(crate) fn poll_closed(&self, waker: &Waker, now: &impl Fn() -> u64) -> Poll<()> { if !live(state(self.word())) || self.check_deadline(now) { return Poll::Ready(()); From 524f63cabff5f45c2fee201aa92040e61e5109c4 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:50:17 -0300 Subject: [PATCH 028/208] refactor: tidy migrated handlers (reply after the body, no wrapper blocks, diverging bodies) --- concurrency/benches/baseline.rs | 6 +- concurrency/benches/designs.rs | 9 +- concurrency/benches/lifecycle_soak.rs | 22 ++- concurrency/benches/metrics_overhead.rs | 6 +- concurrency/benches/pinned_pipeline.rs | 9 +- concurrency/benches/runtime_isolation.rs | 4 +- concurrency/benches/workload_research.rs | 7 +- concurrency/src/child_handle.rs | 10 +- concurrency/src/link_regression_tests.rs | 16 +-- concurrency/src/reply/slot.rs | 2 +- concurrency/src/tasks/actor.rs | 126 +++++++----------- concurrency/src/tasks/stream_tests.rs | 13 +- concurrency/src/tasks/timer_tests.rs | 23 ++-- concurrency/src/threads/actor.rs | 86 +++++------- concurrency/src/threads/stream_tests.rs | 8 +- concurrency/src/threads/timer_tests.rs | 23 ++-- concurrency/tests/actor_scenarios.rs | 24 ++-- concurrency/tests/backend_experiments.rs | 6 +- concurrency/tests/backend_resource_limits.rs | 2 +- concurrency/tests/borrowed_runtime.rs | 6 +- concurrency/tests/canary_feedback.rs | 10 +- concurrency/tests/cancellation_safety.rs | 9 +- concurrency/tests/cfg_gated_protocol.rs | 5 +- concurrency/tests/cohort_trace.rs | 14 +- concurrency/tests/context_exit.rs | 10 +- concurrency/tests/coverage_trace.rs | 14 +- concurrency/tests/cross_backend.rs | 33 ++--- concurrency/tests/dedicated_runtime.rs | 8 +- concurrency/tests/execution_groups.rs | 17 +-- concurrency/tests/execution_profiles.rs | 14 +- concurrency/tests/exit_signal_matrix.rs | 9 +- concurrency/tests/failpoints.rs | 34 ++--- concurrency/tests/foreign_panics.rs | 30 ++--- concurrency/tests/full_actor_shards.rs | 45 +++---- concurrency/tests/future_layout.rs | 5 +- concurrency/tests/kill_links.rs | 6 +- concurrency/tests/leaks.rs | 62 ++++----- concurrency/tests/mailbox_drop_panic.rs | 17 +-- concurrency/tests/metrics.rs | 29 ++-- concurrency/tests/metrics_composable.rs | 2 +- concurrency/tests/metrics_counts_only.rs | 2 +- concurrency/tests/metrics_sampling.rs | 2 +- concurrency/tests/native_dispatcher.rs | 2 +- concurrency/tests/nested_supervision.rs | 2 +- concurrency/tests/panic_safety.rs | 70 ++++------ concurrency/tests/protocol_timeouts.rs | 27 ++-- concurrency/tests/resource_cleanup.rs | 10 +- concurrency/tests/semantics_gaps.rs | 54 +++----- concurrency/tests/shutdown_publication.rs | 5 +- concurrency/tests/sim.rs | 7 +- concurrency/tests/sim_explore.rs | 22 ++- concurrency/tests/stack_budget.rs | 12 +- concurrency/tests/supervision.rs | 4 +- concurrency/tests/supervision_strategies.rs | 2 +- concurrency/tests/supervision_trace.rs | 36 +++-- concurrency/tests/supervisor_policies.rs | 3 +- concurrency/tests/support/failure_actor.rs | 14 +- concurrency/tests/support/store_actor.rs | 10 +- concurrency/tests/tree_trace.rs | 7 +- .../src/bin/byte_budget_pipeline.rs | 8 +- .../src/bin/coordinated_shutdown.rs | 15 +-- .../capabilities/src/bin/dedicated_thread.rs | 2 +- .../capabilities/src/bin/execution_groups.rs | 6 +- .../src/bin/execution_profiles.rs | 7 +- .../capabilities/src/bin/framework_metrics.rs | 2 +- .../capabilities/src/bin/graceful_drain.rs | 8 +- .../capabilities/src/bin/native_lifecycle.rs | 8 +- .../capabilities/src/bin/protocol_timeouts.rs | 8 +- .../capabilities/src/bin/runtime_isolation.rs | 5 +- .../src/bin/shutdown_diagnostics.rs | 10 +- .../src/bin/streams_and_recipients.rs | 16 +-- .../src/bin/supervision_recovery.rs | 2 +- .../capabilities/src/bin/timeout_outcomes.rs | 5 +- examples/exit_reason/src/main.rs | 20 ++- examples/thread_only/src/main.rs | 10 +- testing/src/shared.rs | 2 +- testing/src/tasks.rs | 2 +- testing/src/threads.rs | 2 +- testing/tests/tasks_kit.rs | 13 +- testing/tests/threads_kit.rs | 2 +- 80 files changed, 499 insertions(+), 726 deletions(-) diff --git a/concurrency/benches/baseline.rs b/concurrency/benches/baseline.rs index 3eaf94d4..265b363a 100644 --- a/concurrency/benches/baseline.rs +++ b/concurrency/benches/baseline.rs @@ -104,8 +104,7 @@ impl tasks::Handler for Sink { reply: spawned_concurrency::reply::PendingReply<'_, Ack>, _: &tasks::Context, ) { - let __reply_value: Ack = { process(work) }; - let _ = reply.reply(__reply_value); + reply.reply(process(work)); } } impl threads::Actor for Sink {} @@ -116,8 +115,7 @@ impl threads::Handler for Sink { reply: spawned_concurrency::reply::PendingReply<'_, Ack>, _: &threads::Context, ) { - let __reply_value: Ack = { process(work) }; - let _ = reply.reply(__reply_value); + reply.reply(process(work)); } } #[derive(Clone)] diff --git a/concurrency/benches/designs.rs b/concurrency/benches/designs.rs index 64532435..c8a5cf7d 100644 --- a/concurrency/benches/designs.rs +++ b/concurrency/benches/designs.rs @@ -158,7 +158,7 @@ impl tasks::Handler for Sink { } process_at(work, start) }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl threads::Actor for Sink {} @@ -169,8 +169,7 @@ impl threads::Handler for Sink { reply: spawned_concurrency::reply::PendingReply<'_, Ack>, _: &threads::Context, ) { - let __reply_value: Ack = { process(work) }; - let _ = reply.reply(__reply_value); + reply.reply(process(work)); } } enum ReplySender { @@ -210,7 +209,7 @@ impl tasks::Handler for Sink { { probe.reply.send(process(probe.work)); } - let _ = reply.reply(()); + reply.reply(()); } } impl threads::Handler for Sink { @@ -223,7 +222,7 @@ impl threads::Handler for Sink { { probe.reply.send(process(probe.work)); } - let _ = reply.reply(()); + reply.reply(()); } } enum TokioMessage { diff --git a/concurrency/benches/lifecycle_soak.rs b/concurrency/benches/lifecycle_soak.rs index e01e82cc..ce4893c3 100644 --- a/concurrency/benches/lifecycle_soak.rs +++ b/concurrency/benches/lifecycle_soak.rs @@ -85,7 +85,7 @@ impl Handler for Worker { self.counts.requests.fetch_add(1, Ordering::Relaxed); self.state }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } struct Crash; @@ -96,13 +96,12 @@ impl Handler for Worker { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + panic!("soak injected handler failure"); - } - let _ = reply.reply(()); + } } struct Watcher { @@ -121,12 +120,11 @@ impl Handler for Watcher { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + for (child, panic) in msg.0 { assert!(self.pending.insert(ctx.monitor(&child), panic).is_none()); } - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Watcher { @@ -136,7 +134,7 @@ impl Handler for Watcher { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + let panic = self .pending .remove(&msg.monitor_ref) @@ -146,8 +144,7 @@ impl Handler for Watcher { assert_eq!(msg.reason, ExitReason::Normal); } self.counts.down.fetch_add(1, Ordering::Release); - } - let _ = reply.reply(()); +reply.reply(()); } } struct Remaining; @@ -161,8 +158,7 @@ impl Handler for Watcher { reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context, ) { - let __reply_value: usize = { self.pending.len() }; - let _ = reply.reply(__reply_value); + reply.reply(self.pending.len()); } } fn pin(cpus: &str) { diff --git a/concurrency/benches/metrics_overhead.rs b/concurrency/benches/metrics_overhead.rs index 793a0d56..2648a176 100644 --- a/concurrency/benches/metrics_overhead.rs +++ b/concurrency/benches/metrics_overhead.rs @@ -24,8 +24,7 @@ impl tasks::Handler for Sink { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &tasks::Context, ) { - let __reply_value: u64 = { compute(work) }; - let _ = reply.reply(__reply_value); + reply.reply(compute(work)); } } impl threads::Actor for Sink {} @@ -36,8 +35,7 @@ impl threads::Handler for Sink { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &threads::Context, ) { - let __reply_value: u64 = { compute(work) }; - let _ = reply.reply(__reply_value); + reply.reply(compute(work)); } } fn main() { diff --git a/concurrency/benches/pinned_pipeline.rs b/concurrency/benches/pinned_pipeline.rs index a86b754e..6fbb287e 100644 --- a/concurrency/benches/pinned_pipeline.rs +++ b/concurrency/benches/pinned_pipeline.rs @@ -55,10 +55,9 @@ mod linux { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.next = m.0; - } - let _ = reply.reply(()); +reply.reply(()); } } struct Work { @@ -91,7 +90,7 @@ mod linux { .unwrap(); } } - let _ = reply.reply(()); + reply.reply(()); } } struct Audit; @@ -119,7 +118,7 @@ mod linux { self.handled, ) }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } fn pin(mask: &str) { diff --git a/concurrency/benches/runtime_isolation.rs b/concurrency/benches/runtime_isolation.rs index fcc1eaff..987c0f36 100644 --- a/concurrency/benches/runtime_isolation.rs +++ b/concurrency/benches/runtime_isolation.rs @@ -64,7 +64,7 @@ mod linux { (m.0, value, Instant::now()) } .await; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } struct Audit; @@ -90,7 +90,7 @@ mod linux { self.handled, ) }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } fn pin(mask: &str) { diff --git a/concurrency/benches/workload_research.rs b/concurrency/benches/workload_research.rs index c4a71002..d905eeae 100644 --- a/concurrency/benches/workload_research.rs +++ b/concurrency/benches/workload_research.rs @@ -236,7 +236,7 @@ impl Handler for Node { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + if std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").as_deref() == Ok("sharded") { assert_eq!( std::thread::current().name().unwrap(), @@ -244,8 +244,7 @@ impl Handler for Node { ); } self.children = msg.children; - } - let _ = reply.reply(()); +reply.reply(()); } } #[inline(never)] @@ -298,7 +297,7 @@ impl Handler for Node { service_ns: completed.duration_since(started).as_nanos() as u64, } }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } fn layout(c: &Config) -> (Vec, Vec) { diff --git a/concurrency/src/child_handle.rs b/concurrency/src/child_handle.rs index addb110e..d27e4d5d 100644 --- a/concurrency/src/child_handle.rs +++ b/concurrency/src/child_handle.rs @@ -478,10 +478,9 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.stop(); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -517,10 +516,9 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.stop(); - } - let _ = reply.reply(()); +reply.reply(()); } } diff --git a/concurrency/src/link_regression_tests.rs b/concurrency/src/link_regression_tests.rs index 354f2ed5..e14b2715 100644 --- a/concurrency/src/link_regression_tests.rs +++ b/concurrency/src/link_regression_tests.rs @@ -125,7 +125,7 @@ impl crate::tasks::Handler for Idler { reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } @@ -170,10 +170,9 @@ impl crate::tasks::Handler for MonitorObserver { reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0.take().unwrap().send(down).unwrap(); - } - let _ = reply.reply(()); +reply.reply(()); } } #[test] @@ -248,10 +247,9 @@ impl crate::tasks::Handler for Stopping { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.stop(); - } - let _ = reply.reply(()); +reply.reply(()); } } #[test] @@ -502,7 +500,7 @@ fn linked_runtime_abort_cancels_an_idle_threads_actor() { reply: crate::reply::PendingReply<'_, ()>, _: &crate::threads::Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } let owner = runtime(); @@ -551,7 +549,7 @@ fn dropping_runtime_with_large_polled_link_cycle_reclaims_every_actor() { reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } impl Drop for Node { diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 02644f17..4e83366b 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -156,7 +156,7 @@ impl CreditPool { pub(crate) fn used(&self) -> usize { self.used.load(Ordering::Acquire) } - #[cfg_attr(not(test), allow(dead_code))] // used by the Loom model + #[allow(dead_code)] // used by the Loom model pub(crate) fn max(&self) -> usize { self.max } diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index cc8d1b2d..7ad7b646 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -1660,8 +1660,7 @@ mod tests { reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context, ) { - let __reply_value: u64 = { self.count }; - let _ = reply.reply(__reply_value); + reply.reply(self.count); } } @@ -1676,7 +1675,7 @@ mod tests { self.count += 1; self.count }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } @@ -1691,7 +1690,7 @@ mod tests { ctx.stop(); self.count }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } @@ -1838,12 +1837,11 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - { + // Never replies, so the timeout is the only possible outcome // however late the runtime polls it. std::future::pending::<()>().await; - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -1896,10 +1894,9 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.stop(); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -1944,10 +1941,9 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.stop(); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -1958,11 +1954,10 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + self.hold(); ctx.stop(); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -2041,12 +2036,11 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + rt::sleep(Duration::from_millis(20)).await; thread::sleep(Duration::from_secs(2)); ctx.stop(); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -2068,8 +2062,7 @@ mod tests { reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context, ) { - let __reply_value: u64 = { self.count }; - let _ = reply.reply(__reply_value); + reply.reply(self.count); } } @@ -2084,7 +2077,7 @@ mod tests { ctx.stop(); self.count }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } @@ -2095,12 +2088,11 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + self.count += 1; use crate::tasks::send_after; send_after(Duration::from_millis(100), ctx.clone(), IncrementWell); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -2196,7 +2188,7 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } @@ -2240,8 +2232,7 @@ mod tests { reply: crate::reply::PendingReply<'_, u32>, _ctx: &Context, ) { - let __reply_value: u32 = { 42 }; - let _ = reply.reply(__reply_value); + reply.reply(42); } } @@ -2274,10 +2265,9 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.stop(); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -2350,7 +2340,7 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } @@ -2439,8 +2429,7 @@ mod tests { reply: crate::reply::PendingReply<'_, crate::monitor::MonitorRef>, ctx: &Context, ) { - let __reply_value: crate::monitor::MonitorRef = { ctx.monitor(&msg.0) }; - let _ = reply.reply(__reply_value); + reply.reply(ctx.monitor(&msg.0)); } } @@ -2451,10 +2440,9 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.demonitor(msg.0); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -2465,10 +2453,9 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - { + self.downs.lock().unwrap().push(msg); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -2479,8 +2466,7 @@ mod tests { reply: crate::reply::PendingReply<'_, Vec>, _ctx: &Context, ) { - let __reply_value: Vec = { self.downs.lock().unwrap().clone() }; - let _ = reply.reply(__reply_value); + reply.reply(self.downs.lock().unwrap().clone()); } } @@ -2614,15 +2600,12 @@ mod tests { reply: crate::reply::PendingReply<'_, usize>, ctx: &Context, ) { - let __reply_value: usize = { - ctx.shared + reply.reply(ctx.shared .monitors .lock() .unwrap() .as_ref() - .map_or(0, HashMap::len) - }; - let _ = reply.reply(__reply_value); + .map_or(0, HashMap::len)); } } @@ -2765,10 +2748,9 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.link(&msg.0); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -2779,8 +2761,7 @@ mod tests { reply: crate::reply::PendingReply<'_, Vec>, _ctx: &Context, ) { - let __reply_value: Vec = { self.exits.lock().unwrap().clone() }; - let _ = reply.reply(__reply_value); + reply.reply(self.exits.lock().unwrap().clone()); } } @@ -2966,10 +2947,9 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.unlink(&msg.0); - } - let _ = reply.reply(()); +reply.reply(()); } } b.request(UnlinkFrom(a.child_handle())).await.unwrap(); @@ -2986,9 +2966,7 @@ mod tests { _reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context, ) { - let _reply_value: u64 = { - panic!("boom4"); - }; + panic!("boom4"); } } // Counter's Boom4 returns u64; just use send with no expectation of reply @@ -3026,9 +3004,7 @@ mod tests { _reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context, ) { - let _reply_value: u64 = { - panic!("boom5"); - }; + panic!("boom5"); } } let _ = a.send(Boom5); @@ -3208,7 +3184,7 @@ mod monitor_cleanup_regressions { reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } #[test] @@ -3356,10 +3332,9 @@ mod turn_regressions { reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0.fetch_add(1, Ordering::SeqCst); - } - let _ = reply.reply(()); +reply.reply(()); } } rt::Runtime::new().unwrap().block_on(async { @@ -3432,10 +3407,9 @@ mod idle_stop_regression { reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0.fetch_add(1, Ordering::SeqCst); - } - let _ = reply.reply(()); +reply.reply(()); } } #[test] @@ -3525,8 +3499,7 @@ mod api_coverage { reply: crate::reply::PendingReply<'_, u32>, _: &Context, ) { - let __reply_value: u32 = { msg.0 }; - let _ = reply.reply(__reply_value); + reply.reply(msg.0); } } impl Handler for Echo { @@ -3536,11 +3509,10 @@ mod api_coverage { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + let _ = msg.0.await; ctx.stop(); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Echo { @@ -3550,8 +3522,7 @@ mod api_coverage { reply: crate::reply::PendingReply<'_, (Recipient, ActorId)>, ctx: &Context, ) { - let __reply_value: (Recipient, ActorId) = { (ctx.recipient::(), ctx.id()) }; - let _ = reply.reply(__reply_value); + reply.reply((ctx.recipient::(), ctx.id())); } } impl Handler for Echo { @@ -3650,8 +3621,7 @@ mod api_coverage { reply: crate::reply::PendingReply<'_, u32>, _: &Context, ) { - let __reply_value: u32 = { msg.0 }; - let _ = reply.reply(__reply_value); + reply.reply(msg.0); } } rt::Runtime::new().unwrap().block_on(async { @@ -3708,7 +3678,7 @@ mod shared_state_release { reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } diff --git a/concurrency/src/tasks/stream_tests.rs b/concurrency/src/tasks/stream_tests.rs index 167515d9..81af2e27 100644 --- a/concurrency/src/tasks/stream_tests.rs +++ b/concurrency/src/tasks/stream_tests.rs @@ -49,13 +49,12 @@ impl Handler for Summatory { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + match msg { StreamMsg::Add(val) => self.count += val, StreamMsg::Error => ctx.stop(), } - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -66,10 +65,9 @@ impl Handler for Summatory { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.stop(); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -80,8 +78,7 @@ impl Handler for Summatory { reply: crate::reply::PendingReply<'_, u16>, _ctx: &Context, ) { - let __reply_value: u16 = { self.count }; - let _ = reply.reply(__reply_value); + reply.reply(self.count); } } diff --git a/concurrency/src/tasks/timer_tests.rs b/concurrency/src/tasks/timer_tests.rs index b542a2d8..eb572bae 100644 --- a/concurrency/src/tasks/timer_tests.rs +++ b/concurrency/src/tasks/timer_tests.rs @@ -52,10 +52,9 @@ impl Handler for Repeater { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - { + self.count += 1; - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -66,12 +65,11 @@ impl Handler for Repeater { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - { + if let Some(ct) = self.cancellation_token.clone() { ct.cancel(); } - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -82,8 +80,7 @@ impl Handler for Repeater { reply: crate::reply::PendingReply<'_, i32>, _ctx: &Context, ) { - let __reply_value: i32 = { self.count }; - let _ = reply.reply(__reply_value); + reply.reply(self.count); } } @@ -150,10 +147,9 @@ impl Handler for Delayed { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - { + self.count += 1; - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -164,8 +160,7 @@ impl Handler for Delayed { reply: crate::reply::PendingReply<'_, i32>, _ctx: &Context, ) { - let __reply_value: i32 = { self.count }; - let _ = reply.reply(__reply_value); + reply.reply(self.count); } } @@ -180,7 +175,7 @@ impl Handler for Delayed { ctx.stop(); self.count }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 48cb12a7..e48c71b3 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -1136,8 +1136,7 @@ mod tests { reply: crate::reply::PendingReply<'_, u64>, _ctx: &Context, ) { - let __reply_value: u64 = { self.count }; - let _ = reply.reply(__reply_value); + reply.reply(self.count); } } @@ -1152,7 +1151,7 @@ mod tests { self.count += 1; self.count }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } @@ -1167,7 +1166,7 @@ mod tests { ctx.stop(); self.count }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } @@ -1201,10 +1200,9 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.stop(); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -1233,10 +1231,9 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.stop(); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -1275,7 +1272,7 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } @@ -1316,8 +1313,7 @@ mod tests { reply: crate::reply::PendingReply<'_, u32>, _ctx: &Context, ) { - let __reply_value: u32 = { 42 }; - let _ = reply.reply(__reply_value); + reply.reply(42); } } @@ -1347,10 +1343,9 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.stop(); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -1406,7 +1401,7 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } @@ -1480,8 +1475,7 @@ mod tests { reply: crate::reply::PendingReply<'_, crate::monitor::MonitorRef>, ctx: &Context, ) { - let __reply_value: crate::monitor::MonitorRef = { ctx.monitor(&msg.0) }; - let _ = reply.reply(__reply_value); + reply.reply(ctx.monitor(&msg.0)); } } @@ -1492,10 +1486,9 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.demonitor(msg.0); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -1506,10 +1499,9 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - { + self.downs.lock().unwrap().push(msg); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -1520,8 +1512,7 @@ mod tests { reply: crate::reply::PendingReply<'_, Vec>, _ctx: &Context, ) { - let __reply_value: Vec = { self.downs.lock().unwrap().clone() }; - let _ = reply.reply(__reply_value); + reply.reply(self.downs.lock().unwrap().clone()); } } @@ -1680,10 +1671,9 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.link(&msg.0); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -1694,8 +1684,7 @@ mod tests { reply: crate::reply::PendingReply<'_, Vec>, _ctx: &Context, ) { - let __reply_value: Vec = { self.exits.lock().unwrap().clone() }; - let _ = reply.reply(__reply_value); + reply.reply(self.exits.lock().unwrap().clone()); } } @@ -1823,7 +1812,7 @@ mod monitor_cleanup_regressions { reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } #[test] @@ -1940,10 +1929,9 @@ mod idle_stop_regression { reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0.fetch_add(1, Ordering::SeqCst); - } - let _ = reply.reply(()); +reply.reply(()); } } #[test] @@ -2038,7 +2026,7 @@ mod stop_wakeup_regression { reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } @@ -2188,12 +2176,11 @@ mod stop_wakeup_regression { reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.handled.fetch_add(1, Ordering::SeqCst); let _ = self.entered.send(()); self.release.recv_timeout(Duration::from_secs(5)).unwrap(); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -2269,8 +2256,7 @@ mod api_coverage { reply: crate::reply::PendingReply<'_, u32>, _: &Context, ) { - let __reply_value: u32 = { msg.0 }; - let _ = reply.reply(__reply_value); + reply.reply(msg.0); } } impl Handler for Echo { @@ -2280,11 +2266,10 @@ mod api_coverage { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + let _ = msg.0.recv_timeout(WAIT); ctx.stop(); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Echo { @@ -2294,8 +2279,7 @@ mod api_coverage { reply: crate::reply::PendingReply<'_, (Recipient, ActorId)>, ctx: &Context, ) { - let __reply_value: (Recipient, ActorId) = { (ctx.recipient::(), ctx.id()) }; - let _ = reply.reply(__reply_value); + reply.reply((ctx.recipient::(), ctx.id())); } } impl Handler for Echo { @@ -2391,8 +2375,7 @@ mod api_coverage { reply: crate::reply::PendingReply<'_, u32>, _: &Context, ) { - let __reply_value: u32 = { msg.0 }; - let _ = reply.reply(__reply_value); + reply.reply(msg.0); } } impl Handler for Trapper { @@ -2402,10 +2385,9 @@ mod api_coverage { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.unlink(&msg.0); - } - let _ = reply.reply(()); +reply.reply(()); } } diff --git a/concurrency/src/threads/stream_tests.rs b/concurrency/src/threads/stream_tests.rs index b059da36..680fefb0 100644 --- a/concurrency/src/threads/stream_tests.rs +++ b/concurrency/src/threads/stream_tests.rs @@ -34,11 +34,10 @@ impl Handler for Collector { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - { + self.items.push(msg.value); self.done.fetch_add(1, Ordering::SeqCst); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -49,8 +48,7 @@ impl Handler for Collector { reply: crate::reply::PendingReply<'_, Vec>, _ctx: &Context, ) { - let __reply_value: Vec = { self.items.clone() }; - let _ = reply.reply(__reply_value); + reply.reply(self.items.clone()); } } diff --git a/concurrency/src/threads/timer_tests.rs b/concurrency/src/threads/timer_tests.rs index ecfa34f9..5c0e795c 100644 --- a/concurrency/src/threads/timer_tests.rs +++ b/concurrency/src/threads/timer_tests.rs @@ -52,10 +52,9 @@ impl Handler for Repeater { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - { + self.count += 1; - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -66,12 +65,11 @@ impl Handler for Repeater { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - { + if let Some(ct) = self.cancellation_token.clone() { ct.cancel(); } - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -82,8 +80,7 @@ impl Handler for Repeater { reply: crate::reply::PendingReply<'_, i32>, _ctx: &Context, ) { - let __reply_value: i32 = { self.count }; - let _ = reply.reply(__reply_value); + reply.reply(self.count); } } @@ -147,10 +144,9 @@ impl Handler for Delayed { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - { + self.count += 1; - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -161,8 +157,7 @@ impl Handler for Delayed { reply: crate::reply::PendingReply<'_, i32>, _ctx: &Context, ) { - let __reply_value: i32 = { self.count }; - let _ = reply.reply(__reply_value); + reply.reply(self.count); } } @@ -177,7 +172,7 @@ impl Handler for Delayed { ctx.stop(); self.count }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } diff --git a/concurrency/tests/actor_scenarios.rs b/concurrency/tests/actor_scenarios.rs index 59cd899c..d0359d73 100644 --- a/concurrency/tests/actor_scenarios.rs +++ b/concurrency/tests/actor_scenarios.rs @@ -43,8 +43,7 @@ mod tasks { reply: spawned_concurrency::reply::PendingReply<'_, Reply>, ctx: &Context, ) { - let __reply_value: Reply = { - match command { + reply.reply(match command { Command::Work(id) => { self.0 .processed @@ -59,9 +58,7 @@ mod tasks { Reply::Unwatched } Command::Boom => panic!("scenario injected panic"), - } - }; - let _ = reply.reply(__reply_value); + }); } } impl Handler for TestActor { @@ -71,10 +68,9 @@ mod tasks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + let _ = self.0.events.send(event); - } - let _ = reply.reply(()); +reply.reply(()); } } } @@ -91,8 +87,7 @@ mod threads { reply: spawned_concurrency::reply::PendingReply<'_, Reply>, ctx: &Context, ) { - let __reply_value: Reply = { - match command { + reply.reply(match command { Command::Work(id) => { self.0 .processed @@ -107,9 +102,7 @@ mod threads { Reply::Unwatched } Command::Boom => panic!("scenario injected panic"), - } - }; - let _ = reply.reply(__reply_value); + }); } } impl Handler for TestActor { @@ -119,10 +112,9 @@ mod threads { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + let _ = self.0.events.send(event); - } - let _ = reply.reply(()); +reply.reply(()); } } } diff --git a/concurrency/tests/backend_experiments.rs b/concurrency/tests/backend_experiments.rs index 087786a5..bba0d989 100644 --- a/concurrency/tests/backend_experiments.rs +++ b/concurrency/tests/backend_experiments.rs @@ -30,7 +30,7 @@ impl Handler for Worker { }); child.await.unwrap() }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for Worker { @@ -48,7 +48,7 @@ impl Handler for Worker { // Characterizes progress of a detached helper while its actor blocks. rx.recv_timeout(Duration::from_millis(100)).is_ok() }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } #[test] @@ -106,7 +106,7 @@ impl Handler for Worker { server.join().unwrap(); u64::from_le_bytes(data) }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } #[test] diff --git a/concurrency/tests/backend_resource_limits.rs b/concurrency/tests/backend_resource_limits.rs index d802965f..97f926a1 100644 --- a/concurrency/tests/backend_resource_limits.rs +++ b/concurrency/tests/backend_resource_limits.rs @@ -36,7 +36,7 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } // Not simulated: Blocking backend workers are OS threads. diff --git a/concurrency/tests/borrowed_runtime.rs b/concurrency/tests/borrowed_runtime.rs index 3557d7b3..dc23b504 100644 --- a/concurrency/tests/borrowed_runtime.rs +++ b/concurrency/tests/borrowed_runtime.rs @@ -26,8 +26,7 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ThreadId>, _: &Context, ) { - let __reply_value: ThreadId = { std::thread::current().id() }; - let _ = reply.reply(__reply_value); + reply.reply(std::thread::current().id()); } } impl Handler for Worker { @@ -37,8 +36,7 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ActorRef>, _: &Context, ) { - let __reply_value: ActorRef = { Worker.start() }; - let _ = reply.reply(__reply_value); + reply.reply(Worker.start()); } } diff --git a/concurrency/tests/canary_feedback.rs b/concurrency/tests/canary_feedback.rs index c40d8f9a..f71a465d 100644 --- a/concurrency/tests/canary_feedback.rs +++ b/concurrency/tests/canary_feedback.rs @@ -39,11 +39,10 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + let _ = hold.entered.send(()); let _ = hold.release.await; - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Worker { @@ -53,10 +52,9 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.stop_with_reason(quit.0); - } - let _ = reply.reply(()); +reply.reply(()); } } diff --git a/concurrency/tests/cancellation_safety.rs b/concurrency/tests/cancellation_safety.rs index e09e8efd..858aaaaf 100644 --- a/concurrency/tests/cancellation_safety.rs +++ b/concurrency/tests/cancellation_safety.rs @@ -223,7 +223,7 @@ impl Handler for Probe { } } }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for Probe { @@ -233,10 +233,9 @@ impl Handler for Probe { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.counters.downs.fetch_add(1, Ordering::SeqCst); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -253,7 +252,7 @@ impl Handler for Probe { ) { let __reply_value: u64 = { self.counters.effects.fetch_add(1, Ordering::SeqCst) as u64 + 1 }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } diff --git a/concurrency/tests/cfg_gated_protocol.rs b/concurrency/tests/cfg_gated_protocol.rs index 25f6f309..f85b271a 100644 --- a/concurrency/tests/cfg_gated_protocol.rs +++ b/concurrency/tests/cfg_gated_protocol.rs @@ -29,8 +29,7 @@ impl Handler for MixedActor { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _ctx: &Context, ) { - let __reply_value: u64 = { self.value }; - let _ = reply.reply(__reply_value); + reply.reply(self.value); } } @@ -46,7 +45,7 @@ impl Handler for MixedActor { self.value += 1; self.value }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } diff --git a/concurrency/tests/cohort_trace.rs b/concurrency/tests/cohort_trace.rs index 84d7d12a..9c4421f0 100644 --- a/concurrency/tests/cohort_trace.rs +++ b/concurrency/tests/cohort_trace.rs @@ -18,13 +18,12 @@ impl Handler for Worker { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + panic!("trace cohort failure"); - } - let _ = reply.reply(()); + } } fn write_trace(name: &str, id: u64, strategy: SupervisorStrategy, modes: &[&str]) { @@ -279,13 +278,12 @@ impl Handler for MaybeStop { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + panic!("trace cohort failure"); - } - let _ = reply.reply(()); + } } /// A replacement stopping before readiness inside a round requests a new diff --git a/concurrency/tests/context_exit.rs b/concurrency/tests/context_exit.rs index a5e540ee..bd0cf436 100644 --- a/concurrency/tests/context_exit.rs +++ b/concurrency/tests/context_exit.rs @@ -26,11 +26,10 @@ fn native_explicit_exit_is_cooperative_and_first_reason_wins() { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.stop_with_reason(exit.0); ctx.stop(); - } - let _ = reply.reply(()); +reply.reply(()); } } for reason in [ExitReason::Shutdown, ExitReason::Kill] { @@ -64,11 +63,10 @@ async fn async_explicit_exit_is_cooperative_and_first_reason_wins() { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.stop_with_reason(exit.0); ctx.stop(); - } - let _ = reply.reply(()); +reply.reply(()); } } for reason in [ExitReason::Shutdown, ExitReason::Kill] { diff --git a/concurrency/tests/coverage_trace.rs b/concurrency/tests/coverage_trace.rs index 896e6d80..af23e6a2 100644 --- a/concurrency/tests/coverage_trace.rs +++ b/concurrency/tests/coverage_trace.rs @@ -94,13 +94,12 @@ impl Handler for Worker { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + panic!("injected child failure"); - } - let _ = reply.reply(()); + } } @@ -301,13 +300,12 @@ impl Handler for QuitInStarted { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + panic!("injected child failure"); - } - let _ = reply.reply(()); + } } diff --git a/concurrency/tests/cross_backend.rs b/concurrency/tests/cross_backend.rs index 70448e8f..11e6a23d 100644 --- a/concurrency/tests/cross_backend.rs +++ b/concurrency/tests/cross_backend.rs @@ -221,7 +221,7 @@ mod tasks_node { } } .await; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for Node { @@ -231,14 +231,13 @@ mod tasks_node { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.state.log.push(Obs::Down { observer: self.state.index, monitor: down.monitor_ref, reason: down.reason, }); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Node { @@ -248,7 +247,7 @@ mod tasks_node { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + self.state.ticks_seen += 1; self.state.log.push(Obs::Tick { node: self.state.index, @@ -257,8 +256,7 @@ mod tasks_node { if self.state.timers_done() { ctx.stop(); } - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Node { @@ -286,7 +284,7 @@ mod tasks_node { } } .await; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } } @@ -386,7 +384,7 @@ mod threads_node { } } })(); - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for Node { @@ -396,14 +394,13 @@ mod threads_node { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.state.log.push(Obs::Down { observer: self.state.index, monitor: down.monitor_ref, reason: down.reason, }); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Node { @@ -413,7 +410,7 @@ mod threads_node { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + self.state.ticks_seen += 1; self.state.log.push(Obs::Tick { node: self.state.index, @@ -422,8 +419,7 @@ mod threads_node { if self.state.timers_done() { ctx.stop(); } - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Node { @@ -450,7 +446,7 @@ mod threads_node { ctx.stop(); } })(); - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } } @@ -1357,12 +1353,11 @@ mod early_stop { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &spawned_concurrency::tasks::Context, ) { - { + // Deliberately blocks the only async worker of the group. let _ = block.1.send(()); let _ = block.0.lock().unwrap().recv_timeout(LIMIT); - } - let _ = reply.reply(()); +reply.reply(()); } } impl spawned_concurrency::threads::Actor for Hooks { diff --git a/concurrency/tests/dedicated_runtime.rs b/concurrency/tests/dedicated_runtime.rs index 3b9b958f..57335d05 100644 --- a/concurrency/tests/dedicated_runtime.rs +++ b/concurrency/tests/dedicated_runtime.rs @@ -144,7 +144,7 @@ impl Handler for Probe { flavor, } }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } @@ -165,7 +165,7 @@ impl Handler for Child { tokio::time::sleep(Duration::from_millis(1)).await; 42 }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } struct Nested; @@ -190,7 +190,7 @@ impl Handler for Probe { self.assert_root(); answer }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } @@ -220,7 +220,7 @@ impl Handler for Probe { self.assert_root(); answer }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } diff --git a/concurrency/tests/execution_groups.rs b/concurrency/tests/execution_groups.rs index 1502aebd..4ee107cb 100644 --- a/concurrency/tests/execution_groups.rs +++ b/concurrency/tests/execution_groups.rs @@ -29,13 +29,10 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, String>, _: &Context, ) { - let __reply_value: String = { - std::thread::current() + reply.reply(std::thread::current() .name() .unwrap_or_default() - .to_string() - }; - let _ = reply.reply(__reply_value); + .to_string()); } } impl Handler for Worker { @@ -45,7 +42,7 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } @@ -359,11 +356,10 @@ impl Handler for FailedStartup { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.handled.fetch_add(1, Ordering::SeqCst); drop(msg.0); - } - let _ = reply.reply(()); +reply.reply(()); } } #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] @@ -591,8 +587,7 @@ impl Handler for HelperAware { reply: spawned_concurrency::reply::PendingReply<'_, bool>, ctx: &Context, ) { - let __reply_value: bool = { ctx.spawn_helper(async {}).is_err() }; - let _ = reply.reply(__reply_value); + reply.reply(ctx.spawn_helper(async {}).is_err()); } } struct Dependent { diff --git a/concurrency/tests/execution_profiles.rs b/concurrency/tests/execution_profiles.rs index 6931ab12..04761757 100644 --- a/concurrency/tests/execution_profiles.rs +++ b/concurrency/tests/execution_profiles.rs @@ -35,7 +35,7 @@ impl Handler for Probe { spawned_rt::tasks::affinity::current_cpu().ok(), ) }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } struct StartPeer; @@ -51,7 +51,7 @@ impl Handler for Probe { ) { let __reply_value: ::Result = { ctx.execution_group().unwrap().start(Probe) }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } // Not simulated: native execution groups need OS threads. @@ -100,13 +100,12 @@ impl Handler for Probe { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + tokio::task::block_in_place(|| { let _ = msg.entered.send(()); msg.release.recv_timeout(LIMIT).unwrap(); }); - } - let _ = reply.reply(()); +reply.reply(()); } } struct Crash; @@ -376,10 +375,9 @@ mod tls_join { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + PROBE.with(|p| *p.borrow_mut() = Some(msg.0)); - } - let _ = reply.reply(()); +reply.reply(()); } } #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] diff --git a/concurrency/tests/exit_signal_matrix.rs b/concurrency/tests/exit_signal_matrix.rs index d32ad6cb..16b499a7 100644 --- a/concurrency/tests/exit_signal_matrix.rs +++ b/concurrency/tests/exit_signal_matrix.rs @@ -29,7 +29,7 @@ macro_rules! actors { impl Actor for Idle {} impl Handler for Idle { $($asynchronous)? fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { - let _ = reply.reply(()); + reply.reply(()); } } struct Observer(mpsc::Sender); @@ -40,15 +40,14 @@ macro_rules! actors { } impl Handler for Observer { $($asynchronous)? fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { - let _ = reply.reply(()); + reply.reply(()); } } impl Handler for Observer { $($asynchronous)? fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { - { + self.0.send(Notice::Down(down.reason)).unwrap(); - } - let _ = reply.reply(()); +reply.reply(()); } } }; diff --git a/concurrency/tests/failpoints.rs b/concurrency/tests/failpoints.rs index 38e47f59..eb52aaa3 100644 --- a/concurrency/tests/failpoints.rs +++ b/concurrency/tests/failpoints.rs @@ -96,13 +96,12 @@ impl Handler for Worker { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + panic!("injected child failure"); - } - let _ = reply.reply(()); + } } impl Handler for Worker { @@ -112,7 +111,7 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } @@ -669,10 +668,9 @@ impl Handler for Linker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.link(&msg.0); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Linker { @@ -682,10 +680,9 @@ impl Handler for Linker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.monitor(&msg.0); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Linker { @@ -695,7 +692,7 @@ impl Handler for Linker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } impl Handler for Linker { @@ -705,10 +702,9 @@ impl Handler for Linker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + let _ = self.downs.send(msg.reason); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -926,15 +922,19 @@ mod native { struct Target; impl NativeActor for Target {} impl NativeHandler for Target { - fn handle(&mut self, _: Crash, _: &NativeContext) { + fn handle(&mut self, _: Crash, _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &NativeContext) { + panic!("injected native failure"); + } } struct Peer; impl NativeActor for Peer {} impl NativeHandler for Peer { - fn handle(&mut self, msg: LinkTo, ctx: &NativeContext) { + fn handle(&mut self, msg: LinkTo, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &NativeContext) { + ctx.link(&msg.0); +reply.reply(()); } } diff --git a/concurrency/tests/foreign_panics.rs b/concurrency/tests/foreign_panics.rs index 8e26c08a..744eb0b9 100644 --- a/concurrency/tests/foreign_panics.rs +++ b/concurrency/tests/foreign_panics.rs @@ -188,10 +188,9 @@ mod task_actors { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0.on_crash(); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for TaskVictim { @@ -201,8 +200,7 @@ mod task_actors { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { 7 }; - let _ = reply.reply(__reply_value); + reply.reply(7); } } impl Handler for TaskVictim { @@ -216,7 +214,7 @@ mod task_actors { self.0.gate.wait(); Bomb(Some(REPLY_PANIC)) }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for TaskVictim { @@ -226,10 +224,9 @@ mod task_actors { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0.stash.push(Bomb(Some(REPLY_PANIC))); - } - let _ = reply.reply(()); +reply.reply(()); } } } @@ -253,10 +250,9 @@ mod thread_actors { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0.on_crash(); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for ThreadVictim { @@ -266,8 +262,7 @@ mod thread_actors { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { 7 }; - let _ = reply.reply(__reply_value); + reply.reply(7); } } impl Handler for ThreadVictim { @@ -281,7 +276,7 @@ mod thread_actors { self.0.gate.wait(); Bomb(Some(REPLY_PANIC)) }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for ThreadVictim { @@ -291,10 +286,9 @@ mod thread_actors { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0.stash.push(Bomb(Some(REPLY_PANIC))); - } - let _ = reply.reply(()); +reply.reply(()); } } } diff --git a/concurrency/tests/full_actor_shards.rs b/concurrency/tests/full_actor_shards.rs index e059d556..8f2e36ea 100644 --- a/concurrency/tests/full_actor_shards.rs +++ b/concurrency/tests/full_actor_shards.rs @@ -39,13 +39,10 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, String>, _: &Context, ) { - let __reply_value: String = { - std::thread::current() + reply.reply(std::thread::current() .name() .unwrap_or("unnamed") - .to_owned() - }; - let _ = reply.reply(__reply_value); + .to_owned()); } } struct Hold { @@ -67,7 +64,7 @@ impl Handler for Worker { msg.release.await.unwrap(); 42 }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } struct Nested(std::net::SocketAddr); @@ -100,7 +97,7 @@ impl Handler for Worker { socket.read_exact(&mut reply).await.unwrap(); u64::from_le_bytes(reply) }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } fn runtime() -> tokio::runtime::Runtime { @@ -269,11 +266,10 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + let _ = msg.started.send(()); msg.release.recv_timeout(LIMIT).unwrap(); - } - let _ = reply.reply(()); +reply.reply(()); } } #[test] @@ -357,7 +353,7 @@ impl Handler for Worker { tokio::time::sleep(Duration::from_millis(1)).await; 42 }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } struct SpawnBlockingChild; @@ -374,11 +370,8 @@ impl Handler for Worker { >, _: &Context, ) { - let __reply_value: spawned_concurrency::tasks::ActorRef = { - Worker(Default::default()) - .start_with_backend(spawned_concurrency::tasks::Backend::Blocking) - }; - let _ = reply.reply(__reply_value); + reply.reply(Worker(Default::default()) + .start_with_backend(spawned_concurrency::tasks::Backend::Blocking)); } } struct Helper(tokio::sync::oneshot::Receiver<()>); @@ -392,14 +385,11 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, tokio::task::JoinHandle>, _: &Context, ) { - let __reply_value: tokio::task::JoinHandle = { - tokio::spawn(async move { + reply.reply(tokio::spawn(async move { msg.0.await.unwrap(); tokio::time::sleep(Duration::from_millis(1)).await; 42 - }) - }; - let _ = reply.reply(__reply_value); + })); } } #[test] @@ -444,10 +434,9 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + msg.observer.monitor(&msg.target); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Worker { @@ -457,10 +446,9 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0.lock().unwrap().push("down"); - } - let _ = reply.reply(()); +reply.reply(()); } } struct Events; @@ -474,8 +462,7 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, Vec<&'static str>>, _: &Context, ) { - let __reply_value: Vec<&'static str> = { self.0.lock().unwrap().clone() }; - let _ = reply.reply(__reply_value); + reply.reply(self.0.lock().unwrap().clone()); } } #[test] diff --git a/concurrency/tests/future_layout.rs b/concurrency/tests/future_layout.rs index eb5aa363..3483134f 100644 --- a/concurrency/tests/future_layout.rs +++ b/concurrency/tests/future_layout.rs @@ -29,7 +29,7 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } impl Handler for Worker { @@ -39,8 +39,7 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { query.value }; - let _ = reply.reply(__reply_value); + reply.reply(query.value); } } const BASELINE_WORDS: usize = 1; diff --git a/concurrency/tests/kill_links.rs b/concurrency/tests/kill_links.rs index b29394df..8e8eaf59 100644 --- a/concurrency/tests/kill_links.rs +++ b/concurrency/tests/kill_links.rs @@ -28,8 +28,7 @@ fn native_propagated_kill_is_trappable_but_direct_kill_is_not() { reply: spawned_concurrency::reply::PendingReply<'_, u8>, _: &Context, ) { - let __reply_value: u8 = { 7 }; - let _ = reply.reply(__reply_value); + reply.reply(7); } } fn wait(actor: &ActorRef) -> ExitReason { @@ -90,8 +89,7 @@ async fn async_propagated_kill_is_trappable_but_direct_kill_is_not() { reply: spawned_concurrency::reply::PendingReply<'_, u8>, _: &Context, ) { - let __reply_value: u8 = { 7 }; - let _ = reply.reply(__reply_value); + reply.reply(7); } } let (send, events) = mpsc::channel(); diff --git a/concurrency/tests/leaks.rs b/concurrency/tests/leaks.rs index fc1bca91..81183305 100644 --- a/concurrency/tests/leaks.rs +++ b/concurrency/tests/leaks.rs @@ -70,7 +70,7 @@ impl threads::Handler for NativeProbe { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } @@ -115,10 +115,9 @@ impl threads::Handler>> for NativeA { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context, ) { - { + self.peer = Some(msg.0); - } - let _ = reply.reply(()); +reply.reply(()); } } impl threads::Handler>> for NativeB { @@ -128,10 +127,9 @@ impl threads::Handler>> for NativeB { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context, ) { - { + self.peer = Some(msg.0); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -228,7 +226,7 @@ mod async_leaks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } struct Crash; @@ -254,7 +252,7 @@ mod async_leaks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } @@ -349,7 +347,7 @@ mod async_leaks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } impl Handler for AsyncB { @@ -359,7 +357,7 @@ mod async_leaks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } impl Handler, tasks::Recipient)>> for AsyncA { @@ -369,11 +367,10 @@ mod async_leaks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.peer = Some(msg.0 .0); self.recipient = Some(msg.0 .1); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler, tasks::Recipient)>> for AsyncB { @@ -383,11 +380,10 @@ mod async_leaks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.peer = Some(msg.0 .0); self.recipient = Some(msg.0 .1); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -641,7 +637,7 @@ mod sim_leaks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } impl Handler for Probe { @@ -651,7 +647,7 @@ mod sim_leaks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } @@ -674,7 +670,7 @@ mod sim_leaks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } impl Handler for B { @@ -684,7 +680,7 @@ mod sim_leaks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } impl Handler, tasks::Recipient)>> for A { @@ -694,10 +690,9 @@ mod sim_leaks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.peer = Some(msg.0); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler, tasks::Recipient)>> for B { @@ -707,10 +702,9 @@ mod sim_leaks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.peer = Some(msg.0); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -844,13 +838,12 @@ mod sim_leaks { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + panic!("injected failure for leak test"); - } - let _ = reply.reply(()); + } } @@ -960,13 +953,12 @@ mod sim_leaks { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + panic!("injected failure for leak test"); - } - let _ = reply.reply(()); + } } } diff --git a/concurrency/tests/mailbox_drop_panic.rs b/concurrency/tests/mailbox_drop_panic.rs index 017723e9..63cd00bf 100644 --- a/concurrency/tests/mailbox_drop_panic.rs +++ b/concurrency/tests/mailbox_drop_panic.rs @@ -48,12 +48,11 @@ impl Handler for Victim { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + msg.0.send(()).unwrap(); let _ = msg.1.await; assert!(!msg.2, "original handler failure"); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Victim { @@ -63,10 +62,9 @@ impl Handler for Victim { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0.fetch_add(1, Ordering::SeqCst); - } - let _ = reply.reply(()); +reply.reply(()); } } struct Ping; @@ -86,10 +84,9 @@ impl Handler for Observer { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0.send((false, down.reason)).unwrap(); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Observer { @@ -99,7 +96,7 @@ impl Handler for Observer { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } diff --git a/concurrency/tests/metrics.rs b/concurrency/tests/metrics.rs index 67c71b9b..ffdc3e46 100644 --- a/concurrency/tests/metrics.rs +++ b/concurrency/tests/metrics.rs @@ -45,8 +45,7 @@ impl Handler for Probe { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { 9 }; - let _ = reply.reply(__reply_value); + reply.reply(9); } } impl Handler for Probe { @@ -56,20 +55,19 @@ impl Handler for Probe { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } impl Handler for Probe { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + panic!("metric panic regression"); - } - let _ = reply.reply(()); + } } impl Handler for Probe { @@ -79,11 +77,10 @@ impl Handler for Probe { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + let _ = hold.entered.send(()); let _ = hold.release.await; - } - let _ = reply.reply(()); +reply.reply(()); } } fn runtime() -> tokio::runtime::Runtime { @@ -238,16 +235,20 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { struct Native; impl ThreadActor for Native {} impl ThreadHandler for Native { - fn handle(&mut self, _: metrics_protocol::Query, _: &ThreadContext) -> u64 { - 9 + fn handle(&mut self, _: metrics_protocol::Query, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &ThreadContext) { + reply.reply(9); } } impl ThreadHandler for Native { - fn handle(&mut self, _: Ping, _: &ThreadContext) {} + fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &ThreadContext) { + reply.reply(()); + } } impl ThreadHandler for Native { - fn handle(&mut self, _: Crash, _: &ThreadContext) { + fn handle(&mut self, _: Crash, _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &ThreadContext) { + panic!("native metric panic"); + } } let actor = Native.start(); diff --git a/concurrency/tests/metrics_composable.rs b/concurrency/tests/metrics_composable.rs index daf222ea..5bcdf9fd 100644 --- a/concurrency/tests/metrics_composable.rs +++ b/concurrency/tests/metrics_composable.rs @@ -19,7 +19,7 @@ impl Handler for Probe { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } diff --git a/concurrency/tests/metrics_counts_only.rs b/concurrency/tests/metrics_counts_only.rs index 4be78f6d..44fe4c79 100644 --- a/concurrency/tests/metrics_counts_only.rs +++ b/concurrency/tests/metrics_counts_only.rs @@ -17,7 +17,7 @@ impl Handler for Probe { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } #[test] diff --git a/concurrency/tests/metrics_sampling.rs b/concurrency/tests/metrics_sampling.rs index 5294ff2e..44f02eca 100644 --- a/concurrency/tests/metrics_sampling.rs +++ b/concurrency/tests/metrics_sampling.rs @@ -18,7 +18,7 @@ impl Handler for Probe { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } #[test] diff --git a/concurrency/tests/native_dispatcher.rs b/concurrency/tests/native_dispatcher.rs index 2e25b384..035ab9f1 100644 --- a/concurrency/tests/native_dispatcher.rs +++ b/concurrency/tests/native_dispatcher.rs @@ -26,7 +26,7 @@ impl Handler for Healthy { tokio::task::yield_now().await; 42 }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } struct DropPanic; diff --git a/concurrency/tests/nested_supervision.rs b/concurrency/tests/nested_supervision.rs index ef4e75e0..79866908 100644 --- a/concurrency/tests/nested_supervision.rs +++ b/concurrency/tests/nested_supervision.rs @@ -54,7 +54,7 @@ impl Handler for Counter { self.0 += 1; self.0 }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for Counter { diff --git a/concurrency/tests/panic_safety.rs b/concurrency/tests/panic_safety.rs index 4c951509..41e5817a 100644 --- a/concurrency/tests/panic_safety.rs +++ b/concurrency/tests/panic_safety.rs @@ -201,10 +201,9 @@ mod tasks_actors { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0.fault(Fault::SendHandler, "injected handler panic"); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for TaskVictim { @@ -214,8 +213,7 @@ mod tasks_actors { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { self.0.ask() }; - let _ = reply.reply(__reply_value); + reply.reply(self.0.ask()); } } impl Handler for TaskVictim { @@ -225,7 +223,7 @@ mod tasks_actors { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } impl Handler for TaskVictim { @@ -235,8 +233,7 @@ mod tasks_actors { reply: spawned_concurrency::reply::PendingReply<'_, Bomb>, _: &Context, ) { - let __reply_value: Bomb = { self.0.gift() }; - let _ = reply.reply(__reply_value); + reply.reply(self.0.gift()); } } impl Handler for TaskVictim { @@ -246,10 +243,9 @@ mod tasks_actors { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0.gate.wait(); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for TaskVictim { @@ -268,7 +264,7 @@ mod tasks_actors { None } .await; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for TaskVictim { @@ -278,10 +274,9 @@ mod tasks_actors { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0.fault(Fault::DownHandler, "injected Down panic"); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -301,12 +296,11 @@ mod tasks_actors { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0 .events .push(Event::Down(down.monitor_ref, down.reason)); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for TaskObserver { @@ -325,7 +319,7 @@ mod tasks_actors { None } .await; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for TaskObserver { @@ -335,8 +329,7 @@ mod tasks_actors { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { 7 }; - let _ = reply.reply(__reply_value); + reply.reply(7); } } } @@ -367,10 +360,9 @@ mod thread_actors { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0.fault(Fault::SendHandler, "injected handler panic"); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for ThreadVictim { @@ -380,8 +372,7 @@ mod thread_actors { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { self.0.ask() }; - let _ = reply.reply(__reply_value); + reply.reply(self.0.ask()); } } impl Handler for ThreadVictim { @@ -391,7 +382,7 @@ mod thread_actors { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } impl Handler for ThreadVictim { @@ -401,8 +392,7 @@ mod thread_actors { reply: spawned_concurrency::reply::PendingReply<'_, Bomb>, _: &Context, ) { - let __reply_value: Bomb = { self.0.gift() }; - let _ = reply.reply(__reply_value); + reply.reply(self.0.gift()); } } impl Handler for ThreadVictim { @@ -412,10 +402,9 @@ mod thread_actors { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0.gate.wait(); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for ThreadVictim { @@ -433,7 +422,7 @@ mod thread_actors { } None })(); - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for ThreadVictim { @@ -443,10 +432,9 @@ mod thread_actors { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0.fault(Fault::DownHandler, "injected Down panic"); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -466,12 +454,11 @@ mod thread_actors { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0 .events .push(Event::Down(down.monitor_ref, down.reason)); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for ThreadObserver { @@ -489,7 +476,7 @@ mod thread_actors { } None })(); - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for ThreadObserver { @@ -499,8 +486,7 @@ mod thread_actors { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { 7 }; - let _ = reply.reply(__reply_value); + reply.reply(7); } } } diff --git a/concurrency/tests/protocol_timeouts.rs b/concurrency/tests/protocol_timeouts.rs index f8105ad3..a470ed7b 100644 --- a/concurrency/tests/protocol_timeouts.rs +++ b/concurrency/tests/protocol_timeouts.rs @@ -68,7 +68,7 @@ mod asynchronous { spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; msg.millis }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for Worker { @@ -82,7 +82,7 @@ mod asynchronous { spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; msg.millis }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for Worker { @@ -96,7 +96,7 @@ mod asynchronous { spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; msg.millis }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } // Tokio's paused clock normally; the simulator's clock is always virtual. @@ -164,7 +164,7 @@ mod synchronous { std::thread::sleep(Duration::from_millis(msg.millis)); msg.millis }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for Worker { @@ -178,7 +178,7 @@ mod synchronous { std::thread::sleep(Duration::from_millis(msg.millis)); msg.millis }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } #[test] @@ -213,7 +213,7 @@ mod synchronous { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } impl Handler for Worker { @@ -223,8 +223,7 @@ mod synchronous { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { 31 }; - let _ = reply.reply(__reply_value); + reply.reply(31); } } impl Handler for Worker { @@ -234,8 +233,7 @@ mod synchronous { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { 32 }; - let _ = reply.reply(__reply_value); + reply.reply(32); } } impl Handler for Worker { @@ -245,8 +243,7 @@ mod synchronous { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { 33 }; - let _ = reply.reply(__reply_value); + reply.reply(33); } } impl Handler for Worker { @@ -256,8 +253,7 @@ mod synchronous { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { 34 }; - let _ = reply.reply(__reply_value); + reply.reply(34); } } #[test] @@ -282,8 +278,7 @@ mod synchronous { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { msg.__spawned_request_timeout.value }; - let _ = reply.reply(__reply_value); + reply.reply(msg.__spawned_request_timeout.value); } } #[test] diff --git a/concurrency/tests/resource_cleanup.rs b/concurrency/tests/resource_cleanup.rs index 25a45868..867702f2 100644 --- a/concurrency/tests/resource_cleanup.rs +++ b/concurrency/tests/resource_cleanup.rs @@ -21,10 +21,9 @@ impl tasks::Handler for Sink { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &tasks::Context, ) { - { + drop(msg.0); - } - let _ = reply.reply(()); +reply.reply(()); } } impl threads::Actor for Sink {} @@ -35,10 +34,9 @@ impl threads::Handler for Sink { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context, ) { - { + drop(msg.0); - } - let _ = reply.reply(()); +reply.reply(()); } } fn released(weak: &Weak<()>) { diff --git a/concurrency/tests/semantics_gaps.rs b/concurrency/tests/semantics_gaps.rs index 8c79d330..900a86bf 100644 --- a/concurrency/tests/semantics_gaps.rs +++ b/concurrency/tests/semantics_gaps.rs @@ -107,12 +107,11 @@ mod native { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.trap_exit(true); ctx.link(&watch.0); ctx.monitor(&watch.0); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Observer { @@ -122,10 +121,9 @@ mod native { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + let _ = self.0.send(Event::Ping(ping.0)); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Observer { @@ -135,10 +133,9 @@ mod native { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + let _ = self.0.send(Event::Down(down.reason)); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -151,7 +148,7 @@ mod native { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + for i in 0..burst.count { burst.to.send(Ping(i)).unwrap(); } @@ -159,8 +156,7 @@ mod native { Some(reason) => ctx.stop_with_reason(reason), None => panic!("source panicked"), } - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -201,7 +197,7 @@ mod native { _reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let _reply_value: u64 = { panic!("request handler panic") }; + panic!("request handler panic"); } } @@ -271,12 +267,11 @@ mod tasks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.trap_exit(true); ctx.link(&watch.0); ctx.monitor(&watch.0); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Observer { @@ -286,10 +281,9 @@ mod tasks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + let _ = self.0.send(Event::Ping(ping.0)); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Observer { @@ -299,10 +293,9 @@ mod tasks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + let _ = self.0.send(Event::Down(down.reason)); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -315,7 +308,7 @@ mod tasks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + for i in 0..burst.count { burst.to.send(Ping(i)).unwrap(); } @@ -323,8 +316,7 @@ mod tasks { Some(reason) => ctx.stop_with_reason(reason), None => panic!("source panicked"), } - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -375,10 +367,9 @@ mod tasks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + let _ = self.0.send(Instant::now()); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -467,10 +458,9 @@ mod tasks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + let _ = self.log.send((self.generation, stamp.0)); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Clocked { @@ -563,7 +553,7 @@ mod tasks { _reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let _reply_value: u64 = { panic!("request handler panic") }; + panic!("request handler panic"); } } diff --git a/concurrency/tests/shutdown_publication.rs b/concurrency/tests/shutdown_publication.rs index 0354d9c9..8f01bbab 100644 --- a/concurrency/tests/shutdown_publication.rs +++ b/concurrency/tests/shutdown_publication.rs @@ -72,7 +72,7 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + match msg { Command::Hold { entered, release } => { let _ = entered.send(()); @@ -80,8 +80,7 @@ impl Handler for Worker { } Command::Work => panic!("queued work must be discarded after stop"), } - } - let _ = reply.reply(()); +reply.reply(()); } } struct Cleanup(ChildHandle); diff --git a/concurrency/tests/sim.rs b/concurrency/tests/sim.rs index 1d116da3..5dc1f3f1 100644 --- a/concurrency/tests/sim.rs +++ b/concurrency/tests/sim.rs @@ -88,13 +88,12 @@ impl Handler for Node { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + panic!("injected crash after readiness"); - } - let _ = reply.reply(()); + } } diff --git a/concurrency/tests/sim_explore.rs b/concurrency/tests/sim_explore.rs index 6a1ab361..10faa202 100644 --- a/concurrency/tests/sim_explore.rs +++ b/concurrency/tests/sim_explore.rs @@ -66,7 +66,7 @@ impl Handler for Store { self.log.push(Access::Read(self.value)); self.value }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for Store { @@ -76,11 +76,10 @@ impl Handler for Store { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.log.push(Access::Write(value)); self.value = value; - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Store { @@ -90,8 +89,7 @@ impl Handler for Store { reply: spawned_concurrency::reply::PendingReply<'_, Vec>, _: &Context, ) { - let __reply_value: Vec = { self.log.clone() }; - let _ = reply.reply(__reply_value); + reply.reply(self.log.clone()); } } @@ -115,14 +113,13 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + for _ in 0..self.noise { spawned_rt::tasks::yield_now().await; } let value = self.store.request(Get).await.unwrap(); self.store.request(Set(value + 1)).await.unwrap(); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -233,13 +230,12 @@ impl Handler for Node { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + panic!("injected crash after readiness"); - } - let _ = reply.reply(()); + } } diff --git a/concurrency/tests/stack_budget.rs b/concurrency/tests/stack_budget.rs index e518a776..06a806b3 100644 --- a/concurrency/tests/stack_budget.rs +++ b/concurrency/tests/stack_budget.rs @@ -131,7 +131,7 @@ mod tasks_chain { None => block, } }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } @@ -229,12 +229,11 @@ mod tasks_chain { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + if self.generation == 1 { panic!("injected stack-budget crash"); } - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Flaky { @@ -244,8 +243,7 @@ mod tasks_chain { reply: spawned_concurrency::reply::PendingReply<'_, Block>, _: &Context, ) { - let __reply_value: Block = { msg.block }; - let _ = reply.reply(__reply_value); + reply.reply(msg.block); } } @@ -331,7 +329,7 @@ mod threads_chain { None => block, } }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } diff --git a/concurrency/tests/supervision.rs b/concurrency/tests/supervision.rs index 3cf93828..fbcc409c 100644 --- a/concurrency/tests/supervision.rs +++ b/concurrency/tests/supervision.rs @@ -50,7 +50,7 @@ impl Handler for Counter { self.0 += 1; self.0 }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for Counter { @@ -578,7 +578,7 @@ impl Handler for RaceActor { self.count += 1; self.count }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] diff --git a/concurrency/tests/supervision_strategies.rs b/concurrency/tests/supervision_strategies.rs index 570288ce..a93941e9 100644 --- a/concurrency/tests/supervision_strategies.rs +++ b/concurrency/tests/supervision_strategies.rs @@ -80,7 +80,7 @@ impl Handler for Probe { self.value += 1; self.value }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for Probe { diff --git a/concurrency/tests/supervision_trace.rs b/concurrency/tests/supervision_trace.rs index b63312bf..960fdde4 100644 --- a/concurrency/tests/supervision_trace.rs +++ b/concurrency/tests/supervision_trace.rs @@ -104,11 +104,10 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + msg.0.send(()).unwrap(); let _ = msg.1.await; - } - let _ = reply.reply(()); +reply.reply(()); } } struct DropBomb; @@ -124,13 +123,12 @@ impl Handler for Worker { async fn handle( &mut self, _: DropBomb, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + panic!("discarded message must not be handled"); - } - let _ = reply.reply(()); + } } @@ -216,13 +214,12 @@ impl Handler for Worker { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + panic!("injected child failure"); - } - let _ = reply.reply(()); + } } impl Handler for Worker { @@ -232,10 +229,9 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.stop(); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Worker { @@ -245,10 +241,9 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.stop_with_reason(msg.0); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -541,13 +536,12 @@ impl Handler for ExitOnceDuringRestart { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + panic!("injected child failure"); - } - let _ = reply.reply(()); + } } diff --git a/concurrency/tests/supervisor_policies.rs b/concurrency/tests/supervisor_policies.rs index 819ddbc4..96ff440e 100644 --- a/concurrency/tests/supervisor_policies.rs +++ b/concurrency/tests/supervisor_policies.rs @@ -31,8 +31,7 @@ impl Handler for Probe { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { self.0 }; - let _ = reply.reply(__reply_value); + reply.reply(self.0); } } impl Handler for Probe { diff --git a/concurrency/tests/support/failure_actor.rs b/concurrency/tests/support/failure_actor.rs index 8e414340..94d69f86 100644 --- a/concurrency/tests/support/failure_actor.rs +++ b/concurrency/tests/support/failure_actor.rs @@ -188,7 +188,7 @@ impl spawned_concurrency::tasks::Handler for AsyncWorker { } } }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } #[cfg(feature = "async")] @@ -199,12 +199,11 @@ impl spawned_concurrency::tasks::Handler for AsyncWorker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &spawned_concurrency::tasks::Context, ) { - { + if let Some(sender) = &self.0.down { let _ = sender.send(down); } - } - let _ = reply.reply(()); +reply.reply(()); } } pub struct NativeWorker(pub Worker); @@ -254,7 +253,7 @@ impl spawned_concurrency::threads::Handler for NativeWorker { } } }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl spawned_concurrency::threads::Handler for NativeWorker { @@ -264,12 +263,11 @@ impl spawned_concurrency::threads::Handler for NativeWorker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &spawned_concurrency::threads::Context, ) { - { + if let Some(sender) = &self.0.down { let _ = sender.send(down); } - } - let _ = reply.reply(()); +reply.reply(()); } } #[derive(Clone)] diff --git a/concurrency/tests/support/store_actor.rs b/concurrency/tests/support/store_actor.rs index 344a13c3..1b6355a6 100644 --- a/concurrency/tests/support/store_actor.rs +++ b/concurrency/tests/support/store_actor.rs @@ -118,14 +118,11 @@ impl spawned_concurrency::tasks::Handler for AsyncStore { reply: spawned_concurrency::reply::PendingReply<'_, Option>, _: &spawned_concurrency::tasks::Context, ) { - let __reply_value: Option = { - // Yield inside the turn so concurrent requests really queue up. + reply.reply(// Yield inside the turn so concurrent requests really queue up. if op.key().1.is_multiple_of(2) { tokio::task::yield_now().await; } - self.0.apply(op) - }; - let _ = reply.reply(__reply_value); + self.0.apply(op)); } } @@ -140,8 +137,7 @@ impl spawned_concurrency::threads::Handler for NativeStore { reply: spawned_concurrency::reply::PendingReply<'_, Option>, _: &spawned_concurrency::threads::Context, ) { - let __reply_value: Option = { self.0.apply(op) }; - let _ = reply.reply(__reply_value); + reply.reply(self.0.apply(op)); } } diff --git a/concurrency/tests/tree_trace.rs b/concurrency/tests/tree_trace.rs index 32318c0e..1b58691a 100644 --- a/concurrency/tests/tree_trace.rs +++ b/concurrency/tests/tree_trace.rs @@ -23,13 +23,12 @@ impl Handler for Worker { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + panic!("nested trace failure"); - } - let _ = reply.reply(()); + } } fn config(strategy: SupervisorStrategy) -> SupervisorConfig { diff --git a/examples/capabilities/src/bin/byte_budget_pipeline.rs b/examples/capabilities/src/bin/byte_budget_pipeline.rs index a7e556d5..a8aa6b1f 100644 --- a/examples/capabilities/src/bin/byte_budget_pipeline.rs +++ b/examples/capabilities/src/bin/byte_budget_pipeline.rs @@ -32,7 +32,7 @@ impl Handler for Sink { drop(credit); // Capacity returns after the payload is released. size }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } struct Total; @@ -46,8 +46,7 @@ impl Handler for Sink { reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context, ) { - let __reply_value: usize = { self.0 }; - let _ = reply.reply(__reply_value); + reply.reply(self.0); } } struct Forwarder(ActorRef); @@ -59,8 +58,7 @@ impl Handler for Forwarder { reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context, ) { - let __reply_value: usize = { self.0.request(packet).await.unwrap() }; - let _ = reply.reply(__reply_value); + reply.reply(self.0.request(packet).await.unwrap()); } } diff --git a/examples/capabilities/src/bin/coordinated_shutdown.rs b/examples/capabilities/src/bin/coordinated_shutdown.rs index 6cb12fd6..c8108c6b 100644 --- a/examples/capabilities/src/bin/coordinated_shutdown.rs +++ b/examples/capabilities/src/bin/coordinated_shutdown.rs @@ -42,10 +42,9 @@ impl Handler for Peers { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.peers.push(msg.0); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Peers { @@ -55,8 +54,7 @@ impl Handler for Peers { reply: spawned_concurrency::reply::PendingReply<'_, Vec>, _: &Context, ) { - let __reply_value: Vec = { self.peers.clone() }; - let _ = reply.reply(__reply_value); + reply.reply(self.peers.clone()); } } struct Importer { @@ -75,10 +73,9 @@ impl Handler for Importer { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.peers.send(Peer(msg.0)).unwrap(); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Importer { @@ -88,7 +85,7 @@ impl Handler for Importer { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } #[tokio::main(flavor = "current_thread")] diff --git a/examples/capabilities/src/bin/dedicated_thread.rs b/examples/capabilities/src/bin/dedicated_thread.rs index 86c837d3..f505f486 100644 --- a/examples/capabilities/src/bin/dedicated_thread.rs +++ b/examples/capabilities/src/bin/dedicated_thread.rs @@ -34,7 +34,7 @@ impl Handler for Worker { assert_eq!(root, thread::current().id()); (workers, answer) }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } diff --git a/examples/capabilities/src/bin/execution_groups.rs b/examples/capabilities/src/bin/execution_groups.rs index 0d02234d..ab170220 100644 --- a/examples/capabilities/src/bin/execution_groups.rs +++ b/examples/capabilities/src/bin/execution_groups.rs @@ -18,8 +18,7 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { msg.0 * 2 }; - let _ = reply.reply(__reply_value); + reply.reply(msg.0 * 2); } } struct Router(ActorRef); @@ -31,8 +30,7 @@ impl Handler for Router { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { self.0.request(msg).await.unwrap() }; - let _ = reply.reply(__reply_value); + reply.reply(self.0.request(msg).await.unwrap()); } } #[tokio::main(flavor = "current_thread")] diff --git a/examples/capabilities/src/bin/execution_profiles.rs b/examples/capabilities/src/bin/execution_profiles.rs index c8c61f7b..6e6746bf 100644 --- a/examples/capabilities/src/bin/execution_profiles.rs +++ b/examples/capabilities/src/bin/execution_profiles.rs @@ -18,13 +18,10 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ::Result>, _: &Context, ) { - let __reply_value: ::Result = { - ( + reply.reply(( std::thread::current().name().unwrap_or_default().to_owned(), spawned_rt::tasks::affinity::current_cpu().ok(), - ) - }; - let _ = reply.reply(__reply_value); + )); } } #[tokio::main(flavor = "multi_thread", worker_threads = 2)] diff --git a/examples/capabilities/src/bin/framework_metrics.rs b/examples/capabilities/src/bin/framework_metrics.rs index a1c8dbf4..e389a941 100644 --- a/examples/capabilities/src/bin/framework_metrics.rs +++ b/examples/capabilities/src/bin/framework_metrics.rs @@ -17,7 +17,7 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } #[tokio::main(flavor = "multi_thread", worker_threads = 2)] diff --git a/examples/capabilities/src/bin/graceful_drain.rs b/examples/capabilities/src/bin/graceful_drain.rs index f3c28391..cc6b32de 100644 --- a/examples/capabilities/src/bin/graceful_drain.rs +++ b/examples/capabilities/src/bin/graceful_drain.rs @@ -16,10 +16,9 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0 += 1; - } - let _ = reply.reply(()); +reply.reply(()); } } struct Fence; @@ -33,8 +32,7 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context, ) { - let __reply_value: usize = { self.0 }; - let _ = reply.reply(__reply_value); + reply.reply(self.0); } } #[tokio::main(flavor = "multi_thread", worker_threads = 2)] diff --git a/examples/capabilities/src/bin/native_lifecycle.rs b/examples/capabilities/src/bin/native_lifecycle.rs index db9b1cd9..e43a2ef1 100644 --- a/examples/capabilities/src/bin/native_lifecycle.rs +++ b/examples/capabilities/src/bin/native_lifecycle.rs @@ -25,11 +25,10 @@ impl Handler for Observer { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.counts.1 += 1; self.downs.send(event).unwrap(); - } - let _ = reply.reply(()); +reply.reply(()); } } struct Counts; @@ -43,8 +42,7 @@ impl Handler for Observer { reply: spawned_concurrency::reply::PendingReply<'_, (usize, usize)>, _: &Context, ) { - let __reply_value: (usize, usize) = { self.counts }; - let _ = reply.reply(__reply_value); + reply.reply(self.counts); } } struct Failing; diff --git a/examples/capabilities/src/bin/protocol_timeouts.rs b/examples/capabilities/src/bin/protocol_timeouts.rs index ecb0f6f6..aa2f6952 100644 --- a/examples/capabilities/src/bin/protocol_timeouts.rs +++ b/examples/capabilities/src/bin/protocol_timeouts.rs @@ -33,7 +33,7 @@ mod asynchronous { self.1.fetch_add(1, Ordering::SeqCst); 42 }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for Worker { @@ -47,7 +47,7 @@ mod asynchronous { self.1.fetch_add(1, Ordering::SeqCst); 42 }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } pub async fn run() { @@ -89,7 +89,7 @@ mod synchronous { self.1.fetch_add(1, Ordering::SeqCst); 42 }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for Worker { @@ -103,7 +103,7 @@ mod synchronous { self.1.fetch_add(1, Ordering::SeqCst); 42 }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } pub fn run() { diff --git a/examples/capabilities/src/bin/runtime_isolation.rs b/examples/capabilities/src/bin/runtime_isolation.rs index a852427c..40876284 100644 --- a/examples/capabilities/src/bin/runtime_isolation.rs +++ b/examples/capabilities/src/bin/runtime_isolation.rs @@ -17,8 +17,7 @@ impl Handler for Critical { reply: spawned_concurrency::reply::PendingReply<'_, String>, _: &Context, ) { - let __reply_value: String = { std::thread::current().name().unwrap().to_owned() }; - let _ = reply.reply(__reply_value); + reply.reply(std::thread::current().name().unwrap().to_owned()); } } struct Router(ActorRef); @@ -34,7 +33,7 @@ impl Handler for Router { assert_eq!(std::thread::current().name(), Some("background-worker")); self.0.request(Ping).await.unwrap() }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } diff --git a/examples/capabilities/src/bin/shutdown_diagnostics.rs b/examples/capabilities/src/bin/shutdown_diagnostics.rs index 3db8eb3b..5142644c 100644 --- a/examples/capabilities/src/bin/shutdown_diagnostics.rs +++ b/examples/capabilities/src/bin/shutdown_diagnostics.rs @@ -22,11 +22,10 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + let _ = msg.0.send(()); let _ = msg.1.await; - } - let _ = reply.reply(()); +reply.reply(()); } } struct Unhealthy; @@ -40,11 +39,10 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + // Cooperative abnormal exit, without a panic or forced interruption. ctx.stop_with_reason(ExitReason::Panic("reported unhealthy worker".into())); - } - let _ = reply.reply(()); +reply.reply(()); } } diff --git a/examples/capabilities/src/bin/streams_and_recipients.rs b/examples/capabilities/src/bin/streams_and_recipients.rs index 2dd4d44c..d28c680d 100644 --- a/examples/capabilities/src/bin/streams_and_recipients.rs +++ b/examples/capabilities/src/bin/streams_and_recipients.rs @@ -25,10 +25,9 @@ mod asynchronous { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0 += N * msg.0; - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Counter { @@ -38,8 +37,7 @@ mod asynchronous { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { self.0 }; - let _ = reply.reply(__reply_value); + reply.reply(self.0); } } pub async fn run() { @@ -93,10 +91,9 @@ mod synchronous { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0 += N * msg.0; - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Counter { @@ -106,8 +103,7 @@ mod synchronous { reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let __reply_value: u64 = { self.0 }; - let _ = reply.reply(__reply_value); + reply.reply(self.0); } } struct Source { diff --git a/examples/capabilities/src/bin/supervision_recovery.rs b/examples/capabilities/src/bin/supervision_recovery.rs index ed73fad3..f80578b9 100644 --- a/examples/capabilities/src/bin/supervision_recovery.rs +++ b/examples/capabilities/src/bin/supervision_recovery.rs @@ -27,7 +27,7 @@ impl Handler for Counter { self.0 += 1; self.0 }; - let _ = reply.reply(__reply_value); + reply.reply(__reply_value); } } impl Handler for Counter { diff --git a/examples/capabilities/src/bin/timeout_outcomes.rs b/examples/capabilities/src/bin/timeout_outcomes.rs index a36932ae..4cd0215f 100644 --- a/examples/capabilities/src/bin/timeout_outcomes.rs +++ b/examples/capabilities/src/bin/timeout_outcomes.rs @@ -28,7 +28,7 @@ impl Handler for Counter { msg.entered.send(()).unwrap(); let _ = msg.release.await; } - let _ = reply.reply(()); + reply.reply(()); } } struct Read; @@ -42,8 +42,7 @@ impl Handler for Counter { reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context, ) { - let __reply_value: usize = { self.0 }; - let _ = reply.reply(__reply_value); + reply.reply(self.0); } } diff --git a/examples/exit_reason/src/main.rs b/examples/exit_reason/src/main.rs index 7418ac46..f853a5b8 100644 --- a/examples/exit_reason/src/main.rs +++ b/examples/exit_reason/src/main.rs @@ -100,7 +100,7 @@ fn main() { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } let worker3 = PanicOnStart.start(); @@ -183,8 +183,7 @@ fn main() { reply: spawned_concurrency::reply::PendingReply<'_, MonitorRef>, ctx: &Context, ) { - let __reply_value: MonitorRef = { ctx.monitor(&msg.0) }; - let _ = reply.reply(__reply_value); + reply.reply(ctx.monitor(&msg.0)); } } impl Handler for Observer { @@ -194,10 +193,9 @@ fn main() { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.demonitor(msg.0); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Observer { @@ -207,15 +205,14 @@ fn main() { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - { + tracing::info!( "[observer] received Down for {} ({})", msg.monitor_ref, msg.reason ); self.log.lock().unwrap().push(msg); - } - let _ = reply.reply(()); +reply.reply(()); } } @@ -294,10 +291,9 @@ fn main() { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - { + ctx.link(&msg.0); - } - let _ = reply.reply(()); +reply.reply(()); } } diff --git a/examples/thread_only/src/main.rs b/examples/thread_only/src/main.rs index 0e5ec5ed..3ebca555 100644 --- a/examples/thread_only/src/main.rs +++ b/examples/thread_only/src/main.rs @@ -33,18 +33,16 @@ struct Observer(Arc); impl Actor for Observer {} impl Handler for Observer { fn handle(&mut self, _: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { - { + self.0.fetch_add(1, Ordering::SeqCst); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Observer { fn handle(&mut self, _: Tick, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { - { + self.0.fetch_add(1, Ordering::SeqCst); - } - let _ = reply.reply(()); +reply.reply(()); } } fn main() { diff --git a/testing/src/shared.rs b/testing/src/shared.rs index d8a441ad..4572e510 100644 --- a/testing/src/shared.rs +++ b/testing/src/shared.rs @@ -160,7 +160,7 @@ macro_rules! probe_common { let (reply, rx) = spawned_concurrency::reply::reply_pair(deadline); match self.request(msg)? { Some((value, record)) => { - let _ = reply.reply(value); + reply.reply(value); // Logged `Handled` only once the reply is in the slot. record.finish(); } diff --git a/testing/src/tasks.rs b/testing/src/tasks.rs index cfd3cd16..c57a6d63 100644 --- a/testing/src/tasks.rs +++ b/testing/src/tasks.rs @@ -114,7 +114,7 @@ impl Handler for Mock { if let Some(record) = record { record.finish(); } - let _ = reply.reply(value); + reply.reply(value); } Plan::Crash(reason) => panic!("{reason}"), } diff --git a/testing/src/threads.rs b/testing/src/threads.rs index 70f77f11..45189e13 100644 --- a/testing/src/threads.rs +++ b/testing/src/threads.rs @@ -65,7 +65,7 @@ impl Handler for Mock { if let Some(record) = record { record.finish(); } - let _ = reply.reply(value); + reply.reply(value); } Plan::Crash(reason) => panic!("{reason}"), } diff --git a/testing/tests/tasks_kit.rs b/testing/tests/tasks_kit.rs index bd49ea8c..2d64ed87 100644 --- a/testing/tests/tasks_kit.rs +++ b/testing/tests/tasks_kit.rs @@ -231,10 +231,9 @@ impl Handler for Counter { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0.fetch_add(1, Ordering::SeqCst); - } - let _ = reply.reply(()); +reply.reply(()); } } impl Handler for Counter { @@ -244,8 +243,7 @@ impl Handler for Counter { reply: spawned_concurrency::reply::PendingReply<'_, usize>, _: &Context, ) { - let __reply_value: usize = { self.0.load(Ordering::SeqCst) }; - let _ = reply.reply(__reply_value); + reply.reply(self.0.load(Ordering::SeqCst)); } } @@ -350,12 +348,11 @@ impl Handler for Stuck { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - { + self.0.store(true, Ordering::SeqCst); // Blocking code is never preempted, so shutdown cannot finish in time. std::thread::sleep(Duration::from_millis(500)); - } - let _ = reply.reply(()); +reply.reply(()); } } diff --git a/testing/tests/threads_kit.rs b/testing/tests/threads_kit.rs index 9e2c51e5..4727d4f3 100644 --- a/testing/tests/threads_kit.rs +++ b/testing/tests/threads_kit.rs @@ -156,7 +156,7 @@ impl Handler for Idle { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - let _ = reply.reply(()); + reply.reply(()); } } From eb5878e3d51b73058668b50126a049304a111ab8 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:50:59 -0300 Subject: [PATCH 029/208] fix: tidy leftovers (store actor, panic example), test cfgs --- concurrency/examples/panic_abort.rs | 4 ++-- concurrency/tests/deferred_jobs.rs | 3 ++- concurrency/tests/support/store_actor.rs | 10 +++++----- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/concurrency/examples/panic_abort.rs b/concurrency/examples/panic_abort.rs index aacb7f55..81263ee0 100644 --- a/concurrency/examples/panic_abort.rs +++ b/concurrency/examples/panic_abort.rs @@ -49,7 +49,7 @@ mod native { _reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let _reply_value: u64 = { panic!("{PANIC_TEXT}") }; + panic!("{PANIC_TEXT}"); } } pub fn handler_panic() { @@ -77,7 +77,7 @@ mod tasks { _reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &Context, ) { - let _reply_value: u64 = { panic!("{PANIC_TEXT}") }; + panic!("{PANIC_TEXT}"); } } pub fn handler_panic() { diff --git a/concurrency/tests/deferred_jobs.rs b/concurrency/tests/deferred_jobs.rs index bd27d7a1..98b5cc33 100644 --- a/concurrency/tests/deferred_jobs.rs +++ b/concurrency/tests/deferred_jobs.rs @@ -3,7 +3,6 @@ //! deadlines, emits, delegation with `pipe_reply`, bounds and owner exit. use spawned_concurrency::message::Message; use std::sync::{Arc, Mutex}; -use std::time::Duration; /// What the owner's handlers observed, in dispatch order. #[derive(Default)] @@ -26,6 +25,7 @@ mod tasks_mode { }; use spawned_concurrency::ActorError; use spawned_rt::tasks::{oneshot, sleep}; + use std::time::Duration; struct Owner { log: Arc, @@ -78,6 +78,7 @@ mod tasks_mode { } /// Start jobs from inside a turn, as handlers do. + #[cfg_attr(spawned_sim, allow(dead_code))] // native jobs are not simulated enum Start { Pipe(&'static str, u64), Panics, diff --git a/concurrency/tests/support/store_actor.rs b/concurrency/tests/support/store_actor.rs index 1b6355a6..520a1d89 100644 --- a/concurrency/tests/support/store_actor.rs +++ b/concurrency/tests/support/store_actor.rs @@ -118,11 +118,11 @@ impl spawned_concurrency::tasks::Handler for AsyncStore { reply: spawned_concurrency::reply::PendingReply<'_, Option>, _: &spawned_concurrency::tasks::Context, ) { - reply.reply(// Yield inside the turn so concurrent requests really queue up. - if op.key().1.is_multiple_of(2) { - tokio::task::yield_now().await; - } - self.0.apply(op)); + // Yield inside the turn so concurrent requests really queue up. + if op.key().1.is_multiple_of(2) { + tokio::task::yield_now().await; + } + reply.reply(self.0.apply(op)); } } From 6da7347b03eeb89b020060e125de407274972046 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:51:33 -0300 Subject: [PATCH 030/208] fix(tests): metrics test uses send-time deadlines; rustfmt --- concurrency/benches/lifecycle_soak.rs | 34 +++--- concurrency/benches/pinned_pipeline.rs | 5 +- concurrency/benches/workload_research.rs | 17 ++- concurrency/src/child_handle.rs | 10 +- concurrency/src/lib.rs | 3 +- concurrency/src/link_regression_tests.rs | 10 +- concurrency/src/reply/deadline.rs | 5 +- concurrency/src/reply/jobs.rs | 10 +- concurrency/src/reply/mod.rs | 2 +- concurrency/src/reply/registry.rs | 25 ++++- concurrency/src/tasks/actor.rs | 92 +++++++--------- concurrency/src/tasks/jobs.rs | 10 +- concurrency/src/tasks/mod.rs | 12 +-- concurrency/src/tasks/stream_tests.rs | 16 ++- concurrency/src/tasks/timer_tests.rs | 19 ++-- concurrency/src/threads/actor.rs | 56 ++++------ concurrency/src/threads/jobs.rs | 11 +- concurrency/src/threads/mod.rs | 12 +-- concurrency/src/threads/stream_tests.rs | 7 +- concurrency/src/threads/timer_tests.rs | 19 ++-- concurrency/tests/actor_scenarios.rs | 70 ++++++------ concurrency/tests/canary_feedback.rs | 12 +-- concurrency/tests/cancellation_safety.rs | 5 +- concurrency/tests/cohort_trace.rs | 8 +- concurrency/tests/context_exit.rs | 14 ++- concurrency/tests/coverage_trace.rs | 8 +- concurrency/tests/cross_backend.rs | 73 ++++++------- concurrency/tests/deferred_jobs.rs | 101 ++++++++++++++---- concurrency/tests/deferred_replies.rs | 5 +- concurrency/tests/execution_groups.rs | 13 +-- concurrency/tests/execution_profiles.rs | 16 ++- concurrency/tests/exit_signal_matrix.rs | 2 +- concurrency/tests/failpoints.rs | 38 +++---- concurrency/tests/foreign_panics.rs | 20 ++-- concurrency/tests/full_actor_shards.rs | 37 +++---- concurrency/tests/leaks.rs | 42 +++----- concurrency/tests/mailbox_drop_panic.rs | 19 ++-- concurrency/tests/metrics.rs | 40 ++++--- concurrency/tests/panic_safety.rs | 48 ++++----- concurrency/tests/resource_cleanup.rs | 10 +- concurrency/tests/semantics_gaps.rs | 82 +++++++------- concurrency/tests/shutdown_publication.rs | 15 ++- concurrency/tests/sim.rs | 4 +- concurrency/tests/sim_explore.rs | 24 ++--- concurrency/tests/stack_budget.rs | 9 +- concurrency/tests/supervision_trace.rs | 29 ++--- concurrency/tests/support/failure_actor.rs | 18 ++-- concurrency/tests/tree_trace.rs | 4 +- .../src/bin/coordinated_shutdown.rs | 10 +- .../src/bin/execution_profiles.rs | 6 +- .../capabilities/src/bin/graceful_drain.rs | 5 +- .../capabilities/src/bin/native_lifecycle.rs | 7 +- .../src/bin/shutdown_diagnostics.rs | 14 ++- .../src/bin/streams_and_recipients.rs | 10 +- examples/exit_reason/src/main.rs | 25 ++--- examples/thread_only/src/main.rs | 20 ++-- testing/tests/tasks_kit.rs | 14 ++- 57 files changed, 612 insertions(+), 640 deletions(-) diff --git a/concurrency/benches/lifecycle_soak.rs b/concurrency/benches/lifecycle_soak.rs index ce4893c3..a0cc5dea 100644 --- a/concurrency/benches/lifecycle_soak.rs +++ b/concurrency/benches/lifecycle_soak.rs @@ -99,9 +99,7 @@ impl Handler for Worker { _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - panic!("soak injected handler failure"); - + panic!("soak injected handler failure"); } } struct Watcher { @@ -120,11 +118,10 @@ impl Handler for Watcher { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - for (child, panic) in msg.0 { - assert!(self.pending.insert(ctx.monitor(&child), panic).is_none()); - } -reply.reply(()); + for (child, panic) in msg.0 { + assert!(self.pending.insert(ctx.monitor(&child), panic).is_none()); + } + reply.reply(()); } } impl Handler for Watcher { @@ -134,17 +131,16 @@ impl Handler for Watcher { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - let panic = self - .pending - .remove(&msg.monitor_ref) - .expect("duplicate/unknown monitor notification"); - assert_eq!(matches!(msg.reason, ExitReason::Panic(_)), panic); - if !panic { - assert_eq!(msg.reason, ExitReason::Normal); - } - self.counts.down.fetch_add(1, Ordering::Release); -reply.reply(()); + let panic = self + .pending + .remove(&msg.monitor_ref) + .expect("duplicate/unknown monitor notification"); + assert_eq!(matches!(msg.reason, ExitReason::Panic(_)), panic); + if !panic { + assert_eq!(msg.reason, ExitReason::Normal); + } + self.counts.down.fetch_add(1, Ordering::Release); + reply.reply(()); } } struct Remaining; diff --git a/concurrency/benches/pinned_pipeline.rs b/concurrency/benches/pinned_pipeline.rs index 6fbb287e..65d012b6 100644 --- a/concurrency/benches/pinned_pipeline.rs +++ b/concurrency/benches/pinned_pipeline.rs @@ -55,9 +55,8 @@ mod linux { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.next = m.0; -reply.reply(()); + self.next = m.0; + reply.reply(()); } } struct Work { diff --git a/concurrency/benches/workload_research.rs b/concurrency/benches/workload_research.rs index d905eeae..e9af06ef 100644 --- a/concurrency/benches/workload_research.rs +++ b/concurrency/benches/workload_research.rs @@ -236,15 +236,14 @@ impl Handler for Node { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - if std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").as_deref() == Ok("sharded") { - assert_eq!( - std::thread::current().name().unwrap(), - format!("spawned-shard-{}", msg.expected_shard) - ); - } - self.children = msg.children; -reply.reply(()); + if std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").as_deref() == Ok("sharded") { + assert_eq!( + std::thread::current().name().unwrap(), + format!("spawned-shard-{}", msg.expected_shard) + ); + } + self.children = msg.children; + reply.reply(()); } } #[inline(never)] diff --git a/concurrency/src/child_handle.rs b/concurrency/src/child_handle.rs index d27e4d5d..9d052d5e 100644 --- a/concurrency/src/child_handle.rs +++ b/concurrency/src/child_handle.rs @@ -478,9 +478,8 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.stop(); -reply.reply(()); + ctx.stop(); + reply.reply(()); } } @@ -516,9 +515,8 @@ reply.reply(()); reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.stop(); -reply.reply(()); + ctx.stop(); + reply.reply(()); } } diff --git a/concurrency/src/lib.rs b/concurrency/src/lib.rs index a9943cce..f3ce8471 100644 --- a/concurrency/src/lib.rs +++ b/concurrency/src/lib.rs @@ -111,7 +111,8 @@ pub use error::{ActorError, ExitReason}; pub use link::Exit; pub use monitor::{Down, MonitorRef}; pub use reply::{ - Deadline, DeferError, Delivery, InsertError, PendingReplies, PendingReply, ReplyReceiver, ReplyTo, + Deadline, DeferError, Delivery, InsertError, PendingReplies, PendingReply, ReplyReceiver, + ReplyTo, }; pub use response::Response; pub use spawned_macros::{actor, protocol}; diff --git a/concurrency/src/link_regression_tests.rs b/concurrency/src/link_regression_tests.rs index e14b2715..f6bbeebc 100644 --- a/concurrency/src/link_regression_tests.rs +++ b/concurrency/src/link_regression_tests.rs @@ -170,9 +170,8 @@ impl crate::tasks::Handler for MonitorObserver { reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0.take().unwrap().send(down).unwrap(); -reply.reply(()); + self.0.take().unwrap().send(down).unwrap(); + reply.reply(()); } } #[test] @@ -247,9 +246,8 @@ impl crate::tasks::Handler for Stopping { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.stop(); -reply.reply(()); + ctx.stop(); + reply.reply(()); } } #[test] diff --git a/concurrency/src/reply/deadline.rs b/concurrency/src/reply/deadline.rs index 0ff5b9d8..330b60da 100644 --- a/concurrency/src/reply/deadline.rs +++ b/concurrency/src/reply/deadline.rs @@ -97,7 +97,10 @@ impl DeadlineHeap { /// Rebuild without stale entries. fn compact(&mut self) { let entries = std::mem::take(&mut self.heap).into_vec(); - self.heap = entries.into_iter().filter(|Reverse(e)| e.is_live()).collect(); + self.heap = entries + .into_iter() + .filter(|Reverse(e)| e.is_live()) + .collect(); self.live_at_rebuild = self.heap.len(); } diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 7e4b0efa..e49dfdba 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -292,16 +292,16 @@ impl JobState { inner.emit_credits -= 1; return Ok(()); } - inner = self - .emitters - .wait(inner) - .unwrap_or_else(|p| p.into_inner()); + inner = self.emitters.wait(inner).unwrap_or_else(|p| p.into_inner()); } } /// Emit step 1 (async). #[cfg(feature = "async")] - pub(crate) fn poll_acquire(&self, cx: &mut std::task::Context<'_>) -> std::task::Poll> { + pub(crate) fn poll_acquire( + &self, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { let mut inner = lock(&self.inner); if inner.cancelled || inner.notification != Notification::Pending { return std::task::Poll::Ready(Err(JobGone)); diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 2a06b13f..f65742eb 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -12,8 +12,8 @@ mod pending; pub(crate) mod registry; pub(crate) mod slot; -pub use registry::DEFAULT_MAX_OUTSTANDING; pub use pending::{InsertError, PendingReplies}; +pub use registry::DEFAULT_MAX_OUTSTANDING; pub use slot::Delivery; use crate::error::ActorError; diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index d548024a..bea0469e 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -331,18 +331,30 @@ mod tests { let slot = deferred(®istry, NEVER); registry.begin_turn(); registry.dropped(slot.clone()); - assert!(!slot.is_terminal(), "an in-turn drop waits for the turn end"); + assert!( + !slot.is_terminal(), + "an in-turn drop waits for the turn end" + ); registry.end_turn(panicked); - assert_eq!(Terminal::of(crate::reply::slot::state(slot.word())), Some(expected)); + assert_eq!( + Terminal::of(crate::reply::slot::state(slot.word())), + Some(expected) + ); } // Between turns a drop answers at once; while exiting, Stopped. let slot = deferred(®istry, NEVER); registry.dropped(slot.clone()); - assert_eq!(Terminal::of(crate::reply::slot::state(slot.word())), Some(Terminal::Dropped)); + assert_eq!( + Terminal::of(crate::reply::slot::state(slot.word())), + Some(Terminal::Dropped) + ); registry.exiting(); let slot = deferred(®istry, NEVER); registry.dropped(slot.clone()); - assert_eq!(Terminal::of(crate::reply::slot::state(slot.word())), Some(Terminal::Stopped)); + assert_eq!( + Terminal::of(crate::reply::slot::state(slot.word())), + Some(Terminal::Stopped) + ); assert_eq!(registry.outstanding(), 0); } @@ -353,7 +365,10 @@ mod tests { let answered = deferred(®istry, NEVER); answered.reply(1, &|| 0); assert_eq!(registry.close(), 1); - assert_eq!(Terminal::of(crate::reply::slot::state(open.word())), Some(Terminal::Stopped)); + assert_eq!( + Terminal::of(crate::reply::slot::state(open.word())), + Some(Terminal::Stopped) + ); assert_eq!(registry.outstanding(), 0); assert_eq!(registry.heap_entries(), 0, "the heap is dropped at death"); } diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 7ad7b646..8ee3051b 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -1837,11 +1837,10 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - - // Never replies, so the timeout is the only possible outcome - // however late the runtime polls it. - std::future::pending::<()>().await; -reply.reply(()); + // Never replies, so the timeout is the only possible outcome + // however late the runtime polls it. + std::future::pending::<()>().await; + reply.reply(()); } } @@ -1894,9 +1893,8 @@ reply.reply(()); reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.stop(); -reply.reply(()); + ctx.stop(); + reply.reply(()); } } @@ -1941,9 +1939,8 @@ reply.reply(()); reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.stop(); -reply.reply(()); + ctx.stop(); + reply.reply(()); } } @@ -1954,10 +1951,9 @@ reply.reply(()); reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - self.hold(); - ctx.stop(); -reply.reply(()); + self.hold(); + ctx.stop(); + reply.reply(()); } } @@ -2036,11 +2032,10 @@ reply.reply(()); reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - rt::sleep(Duration::from_millis(20)).await; - thread::sleep(Duration::from_secs(2)); - ctx.stop(); -reply.reply(()); + rt::sleep(Duration::from_millis(20)).await; + thread::sleep(Duration::from_secs(2)); + ctx.stop(); + reply.reply(()); } } @@ -2088,11 +2083,10 @@ reply.reply(()); reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - self.count += 1; - use crate::tasks::send_after; - send_after(Duration::from_millis(100), ctx.clone(), IncrementWell); -reply.reply(()); + self.count += 1; + use crate::tasks::send_after; + send_after(Duration::from_millis(100), ctx.clone(), IncrementWell); + reply.reply(()); } } @@ -2265,9 +2259,8 @@ reply.reply(()); reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.stop(); -reply.reply(()); + ctx.stop(); + reply.reply(()); } } @@ -2440,9 +2433,8 @@ reply.reply(()); reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.demonitor(msg.0); -reply.reply(()); + ctx.demonitor(msg.0); + reply.reply(()); } } @@ -2453,9 +2445,8 @@ reply.reply(()); reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - - self.downs.lock().unwrap().push(msg); -reply.reply(()); + self.downs.lock().unwrap().push(msg); + reply.reply(()); } } @@ -2600,12 +2591,14 @@ reply.reply(()); reply: crate::reply::PendingReply<'_, usize>, ctx: &Context, ) { - reply.reply(ctx.shared + reply.reply( + ctx.shared .monitors .lock() .unwrap() .as_ref() - .map_or(0, HashMap::len)); + .map_or(0, HashMap::len), + ); } } @@ -2748,9 +2741,8 @@ reply.reply(()); reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.link(&msg.0); -reply.reply(()); + ctx.link(&msg.0); + reply.reply(()); } } @@ -2947,9 +2939,8 @@ reply.reply(()); reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.unlink(&msg.0); -reply.reply(()); + ctx.unlink(&msg.0); + reply.reply(()); } } b.request(UnlinkFrom(a.child_handle())).await.unwrap(); @@ -3332,9 +3323,8 @@ mod turn_regressions { reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0.fetch_add(1, Ordering::SeqCst); -reply.reply(()); + self.0.fetch_add(1, Ordering::SeqCst); + reply.reply(()); } } rt::Runtime::new().unwrap().block_on(async { @@ -3407,9 +3397,8 @@ mod idle_stop_regression { reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0.fetch_add(1, Ordering::SeqCst); -reply.reply(()); + self.0.fetch_add(1, Ordering::SeqCst); + reply.reply(()); } } #[test] @@ -3509,10 +3498,9 @@ mod api_coverage { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - let _ = msg.0.await; - ctx.stop(); -reply.reply(()); + let _ = msg.0.await; + ctx.stop(); + reply.reply(()); } } impl Handler for Echo { diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index b0e0d23e..5300a70a 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -8,8 +8,8 @@ use super::{Actor, Context, Envelope, Handler}; use crate::message::Message; use crate::reply::deadline::Expirable; use crate::reply::jobs::{ - JobCredit, JobDeadline, JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, - JobState, PipeError, PipeErrorKind, PipeReplyError, + JobCredit, JobDeadline, JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, JobState, + PipeError, PipeErrorKind, PipeReplyError, }; use crate::reply::slot::{Refused, Slot}; use crate::reply::{now_ticks, PendingReply, ReplyTo}; @@ -254,7 +254,11 @@ impl Context { Ok(JobState::new(self.id(), opts.emit_credits)) } - fn arm_deadline(&self, opts: &JobOptions, run: &Arc>) -> Option> + fn arm_deadline( + &self, + opts: &JobOptions, + run: &Arc>, + ) -> Option> where A: Handler, M: Message, diff --git a/concurrency/src/tasks/mod.rs b/concurrency/src/tasks/mod.rs index 49e5a78d..ae802182 100644 --- a/concurrency/src/tasks/mod.rs +++ b/concurrency/src/tasks/mod.rs @@ -7,12 +7,6 @@ mod stream_tests; #[cfg(test)] mod timer_tests; -pub use crate::response::Response; -pub use actor::{ - request, send_message_on, Actor, ActorRef, ActorStart, Backend, Context, Handler, Receiver, - Recipient, DEFAULT_REQUEST_TIMEOUT, -}; -pub use actor::{JobCtx, DEFAULT_STOP_DEADLINE}; pub use crate::reply::jobs::{ JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, PipeError, PipeErrorKind, PipeReplyError, @@ -21,5 +15,11 @@ pub use crate::reply::{ Deadline, DeferError, Delivery, InsertError, PendingReplies, PendingReply, ReplyReceiver, ReplyTo, DEFAULT_MAX_OUTSTANDING, }; +pub use crate::response::Response; +pub use actor::{ + request, send_message_on, Actor, ActorRef, ActorStart, Backend, Context, Handler, Receiver, + Recipient, DEFAULT_REQUEST_TIMEOUT, +}; +pub use actor::{JobCtx, DEFAULT_STOP_DEADLINE}; pub use stream::spawn_listener; pub use time::{send_after, send_interval, TimerHandle}; diff --git a/concurrency/src/tasks/stream_tests.rs b/concurrency/src/tasks/stream_tests.rs index 81af2e27..51217963 100644 --- a/concurrency/src/tasks/stream_tests.rs +++ b/concurrency/src/tasks/stream_tests.rs @@ -49,12 +49,11 @@ impl Handler for Summatory { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - match msg { - StreamMsg::Add(val) => self.count += val, - StreamMsg::Error => ctx.stop(), - } -reply.reply(()); + match msg { + StreamMsg::Add(val) => self.count += val, + StreamMsg::Error => ctx.stop(), + } + reply.reply(()); } } @@ -65,9 +64,8 @@ impl Handler for Summatory { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.stop(); -reply.reply(()); + ctx.stop(); + reply.reply(()); } } diff --git a/concurrency/src/tasks/timer_tests.rs b/concurrency/src/tasks/timer_tests.rs index eb572bae..f3d50582 100644 --- a/concurrency/src/tasks/timer_tests.rs +++ b/concurrency/src/tasks/timer_tests.rs @@ -52,9 +52,8 @@ impl Handler for Repeater { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - - self.count += 1; -reply.reply(()); + self.count += 1; + reply.reply(()); } } @@ -65,11 +64,10 @@ impl Handler for Repeater { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - - if let Some(ct) = self.cancellation_token.clone() { - ct.cancel(); - } -reply.reply(()); + if let Some(ct) = self.cancellation_token.clone() { + ct.cancel(); + } + reply.reply(()); } } @@ -147,9 +145,8 @@ impl Handler for Delayed { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - - self.count += 1; -reply.reply(()); + self.count += 1; + reply.reply(()); } } diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index e48c71b3..f3bdcd95 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -1200,9 +1200,8 @@ mod tests { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.stop(); -reply.reply(()); + ctx.stop(); + reply.reply(()); } } @@ -1231,9 +1230,8 @@ reply.reply(()); reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.stop(); -reply.reply(()); + ctx.stop(); + reply.reply(()); } } @@ -1343,9 +1341,8 @@ reply.reply(()); reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.stop(); -reply.reply(()); + ctx.stop(); + reply.reply(()); } } @@ -1486,9 +1483,8 @@ reply.reply(()); reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.demonitor(msg.0); -reply.reply(()); + ctx.demonitor(msg.0); + reply.reply(()); } } @@ -1499,9 +1495,8 @@ reply.reply(()); reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - - self.downs.lock().unwrap().push(msg); -reply.reply(()); + self.downs.lock().unwrap().push(msg); + reply.reply(()); } } @@ -1671,9 +1666,8 @@ reply.reply(()); reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.link(&msg.0); -reply.reply(()); + ctx.link(&msg.0); + reply.reply(()); } } @@ -1929,9 +1923,8 @@ mod idle_stop_regression { reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0.fetch_add(1, Ordering::SeqCst); -reply.reply(()); + self.0.fetch_add(1, Ordering::SeqCst); + reply.reply(()); } } #[test] @@ -2176,11 +2169,10 @@ mod stop_wakeup_regression { reply: crate::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.handled.fetch_add(1, Ordering::SeqCst); - let _ = self.entered.send(()); - self.release.recv_timeout(Duration::from_secs(5)).unwrap(); -reply.reply(()); + self.handled.fetch_add(1, Ordering::SeqCst); + let _ = self.entered.send(()); + self.release.recv_timeout(Duration::from_secs(5)).unwrap(); + reply.reply(()); } } @@ -2266,10 +2258,9 @@ mod api_coverage { reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - let _ = msg.0.recv_timeout(WAIT); - ctx.stop(); -reply.reply(()); + let _ = msg.0.recv_timeout(WAIT); + ctx.stop(); + reply.reply(()); } } impl Handler for Echo { @@ -2385,9 +2376,8 @@ reply.reply(()); reply: crate::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.unlink(&msg.0); -reply.reply(()); + ctx.unlink(&msg.0); + reply.reply(()); } } diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index b998fdf8..425ba807 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -10,8 +10,8 @@ use super::{Actor, Context, Envelope, Handler, MailboxItem}; use crate::message::Message; use crate::reply::deadline::Expirable; use crate::reply::jobs::{ - JobCredit, JobDeadline, JobError, JobGone, JobHandle, JobOptions, JobOutcome, - JobState, PipeError, PipeErrorKind, PipeReplyError, + JobCredit, JobDeadline, JobError, JobGone, JobHandle, JobOptions, JobOutcome, JobState, + PipeError, PipeErrorKind, PipeReplyError, }; use crate::reply::slot::{Refused, Slot}; use crate::reply::{now_ticks, PendingReply, ReplyTo}; @@ -222,7 +222,11 @@ impl Context { Ok(JobState::new(self.id(), opts.emit_credits)) } - fn arm_deadline(&self, opts: &JobOptions, run: &Arc>) -> Option> + fn arm_deadline( + &self, + opts: &JobOptions, + run: &Arc>, + ) -> Option> where A: Handler, M: Message, @@ -405,4 +409,3 @@ impl Context { } } } - diff --git a/concurrency/src/threads/mod.rs b/concurrency/src/threads/mod.rs index 205490eb..bc8480fe 100644 --- a/concurrency/src/threads/mod.rs +++ b/concurrency/src/threads/mod.rs @@ -8,12 +8,6 @@ mod timer_tests; #[cfg(test)] mod stream_tests; -pub use crate::response::Response; -pub use actor::{ - request, send_message_on, Actor, ActorRef, ActorStart, Context, Handler, Receiver, Recipient, - DEFAULT_REQUEST_TIMEOUT, -}; -pub use actor::{JobCtx, DEFAULT_STOP_DEADLINE}; pub use crate::reply::jobs::{ JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, PipeError, PipeErrorKind, PipeReplyError, @@ -22,5 +16,11 @@ pub use crate::reply::{ Deadline, DeferError, Delivery, InsertError, PendingReplies, PendingReply, ReplyReceiver, ReplyTo, DEFAULT_MAX_OUTSTANDING, }; +pub use crate::response::Response; +pub use actor::{ + request, send_message_on, Actor, ActorRef, ActorStart, Context, Handler, Receiver, Recipient, + DEFAULT_REQUEST_TIMEOUT, +}; +pub use actor::{JobCtx, DEFAULT_STOP_DEADLINE}; pub use stream::spawn_listener; pub use time::{send_after, send_interval, TimerHandle}; diff --git a/concurrency/src/threads/stream_tests.rs b/concurrency/src/threads/stream_tests.rs index 680fefb0..cd83a74d 100644 --- a/concurrency/src/threads/stream_tests.rs +++ b/concurrency/src/threads/stream_tests.rs @@ -34,10 +34,9 @@ impl Handler for Collector { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - - self.items.push(msg.value); - self.done.fetch_add(1, Ordering::SeqCst); -reply.reply(()); + self.items.push(msg.value); + self.done.fetch_add(1, Ordering::SeqCst); + reply.reply(()); } } diff --git a/concurrency/src/threads/timer_tests.rs b/concurrency/src/threads/timer_tests.rs index 5c0e795c..a1e7bda9 100644 --- a/concurrency/src/threads/timer_tests.rs +++ b/concurrency/src/threads/timer_tests.rs @@ -52,9 +52,8 @@ impl Handler for Repeater { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - - self.count += 1; -reply.reply(()); + self.count += 1; + reply.reply(()); } } @@ -65,11 +64,10 @@ impl Handler for Repeater { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - - if let Some(ct) = self.cancellation_token.clone() { - ct.cancel(); - } -reply.reply(()); + if let Some(ct) = self.cancellation_token.clone() { + ct.cancel(); + } + reply.reply(()); } } @@ -144,9 +142,8 @@ impl Handler for Delayed { reply: crate::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - - self.count += 1; -reply.reply(()); + self.count += 1; + reply.reply(()); } } diff --git a/concurrency/tests/actor_scenarios.rs b/concurrency/tests/actor_scenarios.rs index d0359d73..b5a42f9b 100644 --- a/concurrency/tests/actor_scenarios.rs +++ b/concurrency/tests/actor_scenarios.rs @@ -44,21 +44,21 @@ mod tasks { ctx: &Context, ) { reply.reply(match command { - Command::Work(id) => { - self.0 - .processed - .lock() - .unwrap_or_else(|p| p.into_inner()) - .push(id); - Reply::Work(id) - } - Command::Watch(target) => Reply::Watch(ctx.monitor(&target)), - Command::Unwatch(id) => { - ctx.demonitor(id); - Reply::Unwatched - } - Command::Boom => panic!("scenario injected panic"), - }); + Command::Work(id) => { + self.0 + .processed + .lock() + .unwrap_or_else(|p| p.into_inner()) + .push(id); + Reply::Work(id) + } + Command::Watch(target) => Reply::Watch(ctx.monitor(&target)), + Command::Unwatch(id) => { + ctx.demonitor(id); + Reply::Unwatched + } + Command::Boom => panic!("scenario injected panic"), + }); } } impl Handler for TestActor { @@ -68,9 +68,8 @@ mod tasks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - let _ = self.0.events.send(event); -reply.reply(()); + let _ = self.0.events.send(event); + reply.reply(()); } } } @@ -88,21 +87,21 @@ mod threads { ctx: &Context, ) { reply.reply(match command { - Command::Work(id) => { - self.0 - .processed - .lock() - .unwrap_or_else(|p| p.into_inner()) - .push(id); - Reply::Work(id) - } - Command::Watch(target) => Reply::Watch(ctx.monitor(&target)), - Command::Unwatch(id) => { - ctx.demonitor(id); - Reply::Unwatched - } - Command::Boom => panic!("scenario injected panic"), - }); + Command::Work(id) => { + self.0 + .processed + .lock() + .unwrap_or_else(|p| p.into_inner()) + .push(id); + Reply::Work(id) + } + Command::Watch(target) => Reply::Watch(ctx.monitor(&target)), + Command::Unwatch(id) => { + ctx.demonitor(id); + Reply::Unwatched + } + Command::Boom => panic!("scenario injected panic"), + }); } } impl Handler for TestActor { @@ -112,9 +111,8 @@ mod threads { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - let _ = self.0.events.send(event); -reply.reply(()); + let _ = self.0.events.send(event); + reply.reply(()); } } } diff --git a/concurrency/tests/canary_feedback.rs b/concurrency/tests/canary_feedback.rs index f71a465d..790e0ae0 100644 --- a/concurrency/tests/canary_feedback.rs +++ b/concurrency/tests/canary_feedback.rs @@ -39,10 +39,9 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - let _ = hold.entered.send(()); - let _ = hold.release.await; -reply.reply(()); + let _ = hold.entered.send(()); + let _ = hold.release.await; + reply.reply(()); } } impl Handler for Worker { @@ -52,9 +51,8 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.stop_with_reason(quit.0); -reply.reply(()); + ctx.stop_with_reason(quit.0); + reply.reply(()); } } diff --git a/concurrency/tests/cancellation_safety.rs b/concurrency/tests/cancellation_safety.rs index 858aaaaf..85baab53 100644 --- a/concurrency/tests/cancellation_safety.rs +++ b/concurrency/tests/cancellation_safety.rs @@ -233,9 +233,8 @@ impl Handler for Probe { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.counters.downs.fetch_add(1, Ordering::SeqCst); -reply.reply(()); + self.counters.downs.fetch_add(1, Ordering::SeqCst); + reply.reply(()); } } diff --git a/concurrency/tests/cohort_trace.rs b/concurrency/tests/cohort_trace.rs index 9c4421f0..6323c54e 100644 --- a/concurrency/tests/cohort_trace.rs +++ b/concurrency/tests/cohort_trace.rs @@ -21,9 +21,7 @@ impl Handler for Worker { _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - panic!("trace cohort failure"); - + panic!("trace cohort failure"); } } fn write_trace(name: &str, id: u64, strategy: SupervisorStrategy, modes: &[&str]) { @@ -281,9 +279,7 @@ impl Handler for MaybeStop { _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - panic!("trace cohort failure"); - + panic!("trace cohort failure"); } } /// A replacement stopping before readiness inside a round requests a new diff --git a/concurrency/tests/context_exit.rs b/concurrency/tests/context_exit.rs index bd0cf436..9cb59432 100644 --- a/concurrency/tests/context_exit.rs +++ b/concurrency/tests/context_exit.rs @@ -26,10 +26,9 @@ fn native_explicit_exit_is_cooperative_and_first_reason_wins() { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.stop_with_reason(exit.0); - ctx.stop(); -reply.reply(()); + ctx.stop_with_reason(exit.0); + ctx.stop(); + reply.reply(()); } } for reason in [ExitReason::Shutdown, ExitReason::Kill] { @@ -63,10 +62,9 @@ async fn async_explicit_exit_is_cooperative_and_first_reason_wins() { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.stop_with_reason(exit.0); - ctx.stop(); -reply.reply(()); + ctx.stop_with_reason(exit.0); + ctx.stop(); + reply.reply(()); } } for reason in [ExitReason::Shutdown, ExitReason::Kill] { diff --git a/concurrency/tests/coverage_trace.rs b/concurrency/tests/coverage_trace.rs index af23e6a2..1abefb58 100644 --- a/concurrency/tests/coverage_trace.rs +++ b/concurrency/tests/coverage_trace.rs @@ -97,9 +97,7 @@ impl Handler for Worker { _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - panic!("injected child failure"); - + panic!("injected child failure"); } } @@ -303,9 +301,7 @@ impl Handler for QuitInStarted { _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - panic!("injected child failure"); - + panic!("injected child failure"); } } diff --git a/concurrency/tests/cross_backend.rs b/concurrency/tests/cross_backend.rs index 11e6a23d..fb335bf2 100644 --- a/concurrency/tests/cross_backend.rs +++ b/concurrency/tests/cross_backend.rs @@ -231,13 +231,12 @@ mod tasks_node { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.state.log.push(Obs::Down { - observer: self.state.index, - monitor: down.monitor_ref, - reason: down.reason, - }); -reply.reply(()); + self.state.log.push(Obs::Down { + observer: self.state.index, + monitor: down.monitor_ref, + reason: down.reason, + }); + reply.reply(()); } } impl Handler for Node { @@ -247,16 +246,15 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - self.state.ticks_seen += 1; - self.state.log.push(Obs::Tick { - node: self.state.index, - n, - }); - if self.state.timers_done() { - ctx.stop(); - } -reply.reply(()); + self.state.ticks_seen += 1; + self.state.log.push(Obs::Tick { + node: self.state.index, + n, + }); + if self.state.timers_done() { + ctx.stop(); + } + reply.reply(()); } } impl Handler for Node { @@ -394,13 +392,12 @@ mod threads_node { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.state.log.push(Obs::Down { - observer: self.state.index, - monitor: down.monitor_ref, - reason: down.reason, - }); -reply.reply(()); + self.state.log.push(Obs::Down { + observer: self.state.index, + monitor: down.monitor_ref, + reason: down.reason, + }); + reply.reply(()); } } impl Handler for Node { @@ -410,16 +407,15 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - self.state.ticks_seen += 1; - self.state.log.push(Obs::Tick { - node: self.state.index, - n, - }); - if self.state.timers_done() { - ctx.stop(); - } -reply.reply(()); + self.state.ticks_seen += 1; + self.state.log.push(Obs::Tick { + node: self.state.index, + n, + }); + if self.state.timers_done() { + ctx.stop(); + } + reply.reply(()); } } impl Handler for Node { @@ -1353,11 +1349,10 @@ mod early_stop { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &spawned_concurrency::tasks::Context, ) { - - // Deliberately blocks the only async worker of the group. - let _ = block.1.send(()); - let _ = block.0.lock().unwrap().recv_timeout(LIMIT); -reply.reply(()); + // Deliberately blocks the only async worker of the group. + let _ = block.1.send(()); + let _ = block.0.lock().unwrap().recv_timeout(LIMIT); + reply.reply(()); } } impl spawned_concurrency::threads::Actor for Hooks { diff --git a/concurrency/tests/deferred_jobs.rs b/concurrency/tests/deferred_jobs.rs index 98b5cc33..f4abfb84 100644 --- a/concurrency/tests/deferred_jobs.rs +++ b/concurrency/tests/deferred_jobs.rs @@ -64,7 +64,12 @@ mod tasks_mode { type Result = usize; } impl Handler for Owner { - async fn handle(&mut self, _: Outstanding, reply: PendingReply<'_, usize>, ctx: &Context) { + async fn handle( + &mut self, + _: Outstanding, + reply: PendingReply<'_, usize>, + ctx: &Context, + ) { reply.reply(ctx.deferred_outstanding()); } } @@ -135,14 +140,18 @@ mod tasks_mode { Start::CancelAfterSpawn => { let (open, gate) = oneshot::channel::<()>(); self.gates.push(open); - ctx.pipe(async move { gate.await.is_ok() }, |o| describe("cancelled", o)) - .map(|job| job.cancel()) - .map_err(|e| e.to_string()) + ctx.pipe(async move { gate.await.is_ok() }, |o| { + describe("cancelled", o) + }) + .map(|job| job.cancel()) + .map_err(|e| e.to_string()) } Start::Blocking(v) => ctx - .pipe_blocking_with(JobOptions::default(), move |_| v + 1, |o| { - describe("blocking", o) - }) + .pipe_blocking_with( + JobOptions::default(), + move |_| v + 1, + |o| describe("blocking", o), + ) .map(drop) .map_err(|e| e.to_string()), Start::NativeBudget => { @@ -154,9 +163,11 @@ mod tasks_mode { }, |o| describe("first", o), ); - let second = ctx.pipe_blocking_with(JobOptions::default(), |_| (), |o| { - describe("second", o) - }); + let second = ctx.pipe_blocking_with( + JobOptions::default(), + |_| (), + |o| describe("second", o), + ); drop(hold); match (&first, &second) { (Ok(_), Err(e)) if matches!(e.kind, PipeErrorKind::NativeBudget) => Ok(()), @@ -228,7 +239,11 @@ mod tasks_mode { let (actor, log) = start(); actor.request(Start::Keyed("old")).await.unwrap().unwrap(); actor.request(Start::Keyed("new")).await.unwrap().unwrap(); - actor.request(Start::CancelAfterSpawn).await.unwrap().unwrap(); + actor + .request(Start::CancelAfterSpawn) + .await + .unwrap() + .unwrap(); actor.request(Open).await.unwrap(); // Only the replacing job reports; the replaced and cancelled ones are // fenced even though their work finished. @@ -269,7 +284,11 @@ mod tasks_mode { async fn pipe_rejects_at_the_credit_bound() { let (actor, _) = start(); for _ in 0..4 { - actor.request(Start::Gated("g", None)).await.unwrap().unwrap(); + actor + .request(Start::Gated("g", None)) + .await + .unwrap() + .unwrap(); } let refused = actor.request(Start::Gated("g", None)).await.unwrap(); assert!(refused.unwrap_err().contains("deferred credits")); @@ -289,7 +308,14 @@ mod tasks_mode { // Emits are FIFO and the terminal message is last. assert_eq!( entries(&actor, &log, 6).await, - ["e:emit:0", "e:emit:1", "e:emit:2", "e:emit:3", "e:emit:4", "e:ok:Ok(5)"] + [ + "e:emit:0", + "e:emit:1", + "e:emit:2", + "e:emit:3", + "e:emit:4", + "e:ok:Ok(5)" + ] ); stop(actor).await; } @@ -335,9 +361,10 @@ mod tasks_mode { impl Handler for Router { async fn handle(&mut self, msg: Route, reply: PendingReply<'_, u64>, ctx: &Context) { let Ok(token) = reply.defer() else { return }; - let downstream = self - .backend - .request_raw_until(Lookup(msg.0), token.deadline().map_or(Deadline::Never, Deadline::At)); + let downstream = self.backend.request_raw_until( + Lookup(msg.0), + token.deadline().map_or(Deadline::Never, Deadline::At), + ); let work = async move { match downstream { Ok(rx) => rx.await, @@ -360,12 +387,22 @@ mod tasks_mode { } } impl Handler for Router { - async fn handle(&mut self, _: Ping, reply: PendingReply<'_, &'static str>, _: &Context) { + async fn handle( + &mut self, + _: Ping, + reply: PendingReply<'_, &'static str>, + _: &Context, + ) { reply.reply("pong"); } } impl Handler for Router { - async fn handle(&mut self, msg: Foreign, reply: PendingReply<'_, bool>, ctx: &Context) { + async fn handle( + &mut self, + msg: Foreign, + reply: PendingReply<'_, bool>, + ctx: &Context, + ) { let refused = ctx.pipe_reply(msg.0, async { 1u64 }, |_, token| Delegated { res: JobOutcome::Ok(Ok(0)), token, @@ -395,7 +432,12 @@ mod tasks_mode { } } impl Handler for Holder { - async fn handle(&mut self, _: Give, reply: PendingReply<'_, Option>>, _: &Context) { + async fn handle( + &mut self, + _: Give, + reply: PendingReply<'_, Option>>, + _: &Context, + ) { let token = self.0.take(); reply.reply(token); } @@ -430,8 +472,16 @@ mod tasks_mode { let rx = holder.request_raw_until(Hold, Deadline::Never).unwrap(); let token = holder.request(Give).await.unwrap().unwrap(); assert!(router.request(Foreign(token)).await.unwrap()); - assert_eq!(rx.await.unwrap(), 7, "the refused token came back and was answered"); - for actor in [router.child_handle(), backend.child_handle(), holder.child_handle()] { + assert_eq!( + rx.await.unwrap(), + 7, + "the refused token came back and was answered" + ); + for actor in [ + router.child_handle(), + backend.child_handle(), + holder.child_handle(), + ] { actor.stop(); } router.join().await; @@ -508,7 +558,9 @@ mod threads_mode { fn native_jobs_answer_deferred_requests() { let log = Arc::new(Log::default()); let actor: ActorRef = Owner { log: log.clone() }.start(); - let rx = actor.request_raw_until(Compute(7), Deadline::Never).unwrap(); + let rx = actor + .request_raw_until(Compute(7), Deadline::Never) + .unwrap(); assert_eq!(rx.recv().unwrap(), 49); assert!(matches!( actor.request(Compute(0)), @@ -516,7 +568,10 @@ mod threads_mode { )); let entries = log.take(); assert_eq!(entries[0], "computing 7"); - assert!(entries.contains(&"panicked zero".to_string()), "{entries:?}"); + assert!( + entries.contains(&"panicked zero".to_string()), + "{entries:?}" + ); actor.context().stop(); actor.join(); } diff --git a/concurrency/tests/deferred_replies.rs b/concurrency/tests/deferred_replies.rs index f6891d19..f68eec6c 100644 --- a/concurrency/tests/deferred_replies.rs +++ b/concurrency/tests/deferred_replies.rs @@ -547,7 +547,10 @@ mod threads_mode { .unwrap(); let began = std::time::Instant::now(); while actor.request(Outstanding).unwrap() != 0 { - assert!(began.elapsed() < Duration::from_secs(5), "token never expired"); + assert!( + began.elapsed() < Duration::from_secs(5), + "token never expired" + ); std::thread::sleep(Duration::from_millis(5)); } actor.request(Release(0)).unwrap(); diff --git a/concurrency/tests/execution_groups.rs b/concurrency/tests/execution_groups.rs index 4ee107cb..2f1e5f61 100644 --- a/concurrency/tests/execution_groups.rs +++ b/concurrency/tests/execution_groups.rs @@ -29,10 +29,12 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, String>, _: &Context, ) { - reply.reply(std::thread::current() + reply.reply( + std::thread::current() .name() .unwrap_or_default() - .to_string()); + .to_string(), + ); } } impl Handler for Worker { @@ -356,10 +358,9 @@ impl Handler for FailedStartup { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.handled.fetch_add(1, Ordering::SeqCst); - drop(msg.0); -reply.reply(()); + self.handled.fetch_add(1, Ordering::SeqCst); + drop(msg.0); + reply.reply(()); } } #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] diff --git a/concurrency/tests/execution_profiles.rs b/concurrency/tests/execution_profiles.rs index 04761757..d508e20f 100644 --- a/concurrency/tests/execution_profiles.rs +++ b/concurrency/tests/execution_profiles.rs @@ -100,12 +100,11 @@ impl Handler for Probe { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - tokio::task::block_in_place(|| { - let _ = msg.entered.send(()); - msg.release.recv_timeout(LIMIT).unwrap(); - }); -reply.reply(()); + tokio::task::block_in_place(|| { + let _ = msg.entered.send(()); + msg.release.recv_timeout(LIMIT).unwrap(); + }); + reply.reply(()); } } struct Crash; @@ -375,9 +374,8 @@ mod tls_join { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - PROBE.with(|p| *p.borrow_mut() = Some(msg.0)); -reply.reply(()); + PROBE.with(|p| *p.borrow_mut() = Some(msg.0)); + reply.reply(()); } } #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] diff --git a/concurrency/tests/exit_signal_matrix.rs b/concurrency/tests/exit_signal_matrix.rs index 16b499a7..b721b750 100644 --- a/concurrency/tests/exit_signal_matrix.rs +++ b/concurrency/tests/exit_signal_matrix.rs @@ -45,7 +45,7 @@ macro_rules! actors { } impl Handler for Observer { $($asynchronous)? fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { - + self.0.send(Notice::Down(down.reason)).unwrap(); reply.reply(()); } diff --git a/concurrency/tests/failpoints.rs b/concurrency/tests/failpoints.rs index eb52aaa3..04687211 100644 --- a/concurrency/tests/failpoints.rs +++ b/concurrency/tests/failpoints.rs @@ -99,9 +99,7 @@ impl Handler for Worker { _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - panic!("injected child failure"); - + panic!("injected child failure"); } } impl Handler for Worker { @@ -668,9 +666,8 @@ impl Handler for Linker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.link(&msg.0); -reply.reply(()); + ctx.link(&msg.0); + reply.reply(()); } } impl Handler for Linker { @@ -680,9 +677,8 @@ impl Handler for Linker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.monitor(&msg.0); -reply.reply(()); + ctx.monitor(&msg.0); + reply.reply(()); } } impl Handler for Linker { @@ -702,9 +698,8 @@ impl Handler for Linker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - let _ = self.downs.send(msg.reason); -reply.reply(()); + let _ = self.downs.send(msg.reason); + reply.reply(()); } } @@ -922,19 +917,26 @@ mod native { struct Target; impl NativeActor for Target {} impl NativeHandler for Target { - fn handle(&mut self, _: Crash, _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &NativeContext) { - + fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &NativeContext, + ) { panic!("injected native failure"); - } } struct Peer; impl NativeActor for Peer {} impl NativeHandler for Peer { - fn handle(&mut self, msg: LinkTo, reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &NativeContext) { - + fn handle( + &mut self, + msg: LinkTo, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &NativeContext, + ) { ctx.link(&msg.0); -reply.reply(()); + reply.reply(()); } } diff --git a/concurrency/tests/foreign_panics.rs b/concurrency/tests/foreign_panics.rs index 744eb0b9..909bbdfa 100644 --- a/concurrency/tests/foreign_panics.rs +++ b/concurrency/tests/foreign_panics.rs @@ -188,9 +188,8 @@ mod task_actors { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0.on_crash(); -reply.reply(()); + self.0.on_crash(); + reply.reply(()); } } impl Handler for TaskVictim { @@ -224,9 +223,8 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0.stash.push(Bomb(Some(REPLY_PANIC))); -reply.reply(()); + self.0.stash.push(Bomb(Some(REPLY_PANIC))); + reply.reply(()); } } } @@ -250,9 +248,8 @@ mod thread_actors { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0.on_crash(); -reply.reply(()); + self.0.on_crash(); + reply.reply(()); } } impl Handler for ThreadVictim { @@ -286,9 +283,8 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0.stash.push(Bomb(Some(REPLY_PANIC))); -reply.reply(()); + self.0.stash.push(Bomb(Some(REPLY_PANIC))); + reply.reply(()); } } } diff --git a/concurrency/tests/full_actor_shards.rs b/concurrency/tests/full_actor_shards.rs index 8f2e36ea..cf6acbd7 100644 --- a/concurrency/tests/full_actor_shards.rs +++ b/concurrency/tests/full_actor_shards.rs @@ -39,10 +39,12 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, String>, _: &Context, ) { - reply.reply(std::thread::current() + reply.reply( + std::thread::current() .name() .unwrap_or("unnamed") - .to_owned()); + .to_owned(), + ); } } struct Hold { @@ -266,10 +268,9 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - let _ = msg.started.send(()); - msg.release.recv_timeout(LIMIT).unwrap(); -reply.reply(()); + let _ = msg.started.send(()); + msg.release.recv_timeout(LIMIT).unwrap(); + reply.reply(()); } } #[test] @@ -370,8 +371,10 @@ impl Handler for Worker { >, _: &Context, ) { - reply.reply(Worker(Default::default()) - .start_with_backend(spawned_concurrency::tasks::Backend::Blocking)); + reply.reply( + Worker(Default::default()) + .start_with_backend(spawned_concurrency::tasks::Backend::Blocking), + ); } } struct Helper(tokio::sync::oneshot::Receiver<()>); @@ -386,10 +389,10 @@ impl Handler for Worker { _: &Context, ) { reply.reply(tokio::spawn(async move { - msg.0.await.unwrap(); - tokio::time::sleep(Duration::from_millis(1)).await; - 42 - })); + msg.0.await.unwrap(); + tokio::time::sleep(Duration::from_millis(1)).await; + 42 + })); } } #[test] @@ -434,9 +437,8 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - msg.observer.monitor(&msg.target); -reply.reply(()); + msg.observer.monitor(&msg.target); + reply.reply(()); } } impl Handler for Worker { @@ -446,9 +448,8 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0.lock().unwrap().push("down"); -reply.reply(()); + self.0.lock().unwrap().push("down"); + reply.reply(()); } } struct Events; diff --git a/concurrency/tests/leaks.rs b/concurrency/tests/leaks.rs index 81183305..d5a615a9 100644 --- a/concurrency/tests/leaks.rs +++ b/concurrency/tests/leaks.rs @@ -115,9 +115,8 @@ impl threads::Handler>> for NativeA { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context, ) { - - self.peer = Some(msg.0); -reply.reply(()); + self.peer = Some(msg.0); + reply.reply(()); } } impl threads::Handler>> for NativeB { @@ -127,9 +126,8 @@ impl threads::Handler>> for NativeB { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context, ) { - - self.peer = Some(msg.0); -reply.reply(()); + self.peer = Some(msg.0); + reply.reply(()); } } @@ -367,10 +365,9 @@ mod async_leaks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.peer = Some(msg.0 .0); - self.recipient = Some(msg.0 .1); -reply.reply(()); + self.peer = Some(msg.0 .0); + self.recipient = Some(msg.0 .1); + reply.reply(()); } } impl Handler, tasks::Recipient)>> for AsyncB { @@ -380,10 +377,9 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.peer = Some(msg.0 .0); - self.recipient = Some(msg.0 .1); -reply.reply(()); + self.peer = Some(msg.0 .0); + self.recipient = Some(msg.0 .1); + reply.reply(()); } } @@ -690,9 +686,8 @@ mod sim_leaks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.peer = Some(msg.0); -reply.reply(()); + self.peer = Some(msg.0); + reply.reply(()); } } impl Handler, tasks::Recipient)>> for B { @@ -702,9 +697,8 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.peer = Some(msg.0); -reply.reply(()); + self.peer = Some(msg.0); + reply.reply(()); } } @@ -841,9 +835,7 @@ reply.reply(()); _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - panic!("injected failure for leak test"); - + panic!("injected failure for leak test"); } } @@ -956,9 +948,7 @@ reply.reply(()); _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - panic!("injected failure for leak test"); - + panic!("injected failure for leak test"); } } } diff --git a/concurrency/tests/mailbox_drop_panic.rs b/concurrency/tests/mailbox_drop_panic.rs index 63cd00bf..961804f1 100644 --- a/concurrency/tests/mailbox_drop_panic.rs +++ b/concurrency/tests/mailbox_drop_panic.rs @@ -48,11 +48,10 @@ impl Handler for Victim { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - msg.0.send(()).unwrap(); - let _ = msg.1.await; - assert!(!msg.2, "original handler failure"); -reply.reply(()); + msg.0.send(()).unwrap(); + let _ = msg.1.await; + assert!(!msg.2, "original handler failure"); + reply.reply(()); } } impl Handler for Victim { @@ -62,9 +61,8 @@ impl Handler for Victim { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0.fetch_add(1, Ordering::SeqCst); -reply.reply(()); + self.0.fetch_add(1, Ordering::SeqCst); + reply.reply(()); } } struct Ping; @@ -84,9 +82,8 @@ impl Handler for Observer { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0.send((false, down.reason)).unwrap(); -reply.reply(()); + self.0.send((false, down.reason)).unwrap(); + reply.reply(()); } } impl Handler for Observer { diff --git a/concurrency/tests/metrics.rs b/concurrency/tests/metrics.rs index ffdc3e46..10a413d3 100644 --- a/concurrency/tests/metrics.rs +++ b/concurrency/tests/metrics.rs @@ -65,9 +65,7 @@ impl Handler for Probe { _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - panic!("metric panic regression"); - + panic!("metric panic regression"); } } impl Handler for Probe { @@ -77,10 +75,9 @@ impl Handler for Probe { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - let _ = hold.entered.send(()); - let _ = hold.release.await; -reply.reply(()); + let _ = hold.entered.send(()); + let _ = hold.release.await; + reply.reply(()); } } fn runtime() -> tokio::runtime::Runtime { @@ -109,7 +106,7 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { Err(ActorError::RequestTimeout) )); assert!(matches!( - Response::from_with_timeout(actor.request_raw(Ping), Duration::ZERO).await, + Response::from(actor.request_raw_until(Ping, spawned_concurrency::Deadline::after(Duration::ZERO))).await, Err(ActorError::RequestTimeout) )); // Dropping either kind of caller wrapper does not cancel actor work. @@ -131,7 +128,7 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { .await .unwrap(); Response::from(actor.request_raw(Ping)).await.unwrap(); - Response::from_with_timeout(actor.request_raw(Ping), Duration::from_secs(2)) + Response::from(actor.request_raw_until(Ping, spawned_concurrency::Deadline::after(Duration::from_secs(2)))) .await .unwrap(); actor.context().stop(); @@ -139,7 +136,7 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { assert_eq!(actor.mailbox_depth(), 0); assert!(actor.send(Ping).is_err()); assert!(matches!( - Response::from_with_timeout(actor.request_raw(Ping), Duration::from_secs(2)).await, + Response::from(actor.request_raw_until(Ping, spawned_concurrency::Deadline::after(Duration::from_secs(2)))).await, Err(ActorError::ActorStopped) )); @@ -235,20 +232,33 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { struct Native; impl ThreadActor for Native {} impl ThreadHandler for Native { - fn handle(&mut self, _: metrics_protocol::Query, reply: spawned_concurrency::reply::PendingReply<'_, u64>, _: &ThreadContext) { + fn handle( + &mut self, + _: metrics_protocol::Query, + reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _: &ThreadContext, + ) { reply.reply(9); } } impl ThreadHandler for Native { - fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &ThreadContext) { + fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &ThreadContext, + ) { reply.reply(()); } } impl ThreadHandler for Native { - fn handle(&mut self, _: Crash, _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &ThreadContext) { - + fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &ThreadContext, + ) { panic!("native metric panic"); - } } let actor = Native.start(); diff --git a/concurrency/tests/panic_safety.rs b/concurrency/tests/panic_safety.rs index 41e5817a..c945bfe4 100644 --- a/concurrency/tests/panic_safety.rs +++ b/concurrency/tests/panic_safety.rs @@ -201,9 +201,8 @@ mod tasks_actors { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0.fault(Fault::SendHandler, "injected handler panic"); -reply.reply(()); + self.0.fault(Fault::SendHandler, "injected handler panic"); + reply.reply(()); } } impl Handler for TaskVictim { @@ -243,9 +242,8 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0.gate.wait(); -reply.reply(()); + self.0.gate.wait(); + reply.reply(()); } } impl Handler for TaskVictim { @@ -274,9 +272,8 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0.fault(Fault::DownHandler, "injected Down panic"); -reply.reply(()); + self.0.fault(Fault::DownHandler, "injected Down panic"); + reply.reply(()); } } @@ -296,11 +293,10 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0 - .events - .push(Event::Down(down.monitor_ref, down.reason)); -reply.reply(()); + self.0 + .events + .push(Event::Down(down.monitor_ref, down.reason)); + reply.reply(()); } } impl Handler for TaskObserver { @@ -360,9 +356,8 @@ mod thread_actors { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0.fault(Fault::SendHandler, "injected handler panic"); -reply.reply(()); + self.0.fault(Fault::SendHandler, "injected handler panic"); + reply.reply(()); } } impl Handler for ThreadVictim { @@ -402,9 +397,8 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0.gate.wait(); -reply.reply(()); + self.0.gate.wait(); + reply.reply(()); } } impl Handler for ThreadVictim { @@ -432,9 +426,8 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0.fault(Fault::DownHandler, "injected Down panic"); -reply.reply(()); + self.0.fault(Fault::DownHandler, "injected Down panic"); + reply.reply(()); } } @@ -454,11 +447,10 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0 - .events - .push(Event::Down(down.monitor_ref, down.reason)); -reply.reply(()); + self.0 + .events + .push(Event::Down(down.monitor_ref, down.reason)); + reply.reply(()); } } impl Handler for ThreadObserver { diff --git a/concurrency/tests/resource_cleanup.rs b/concurrency/tests/resource_cleanup.rs index 867702f2..c3be4ec3 100644 --- a/concurrency/tests/resource_cleanup.rs +++ b/concurrency/tests/resource_cleanup.rs @@ -21,9 +21,8 @@ impl tasks::Handler for Sink { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &tasks::Context, ) { - - drop(msg.0); -reply.reply(()); + drop(msg.0); + reply.reply(()); } } impl threads::Actor for Sink {} @@ -34,9 +33,8 @@ impl threads::Handler for Sink { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &threads::Context, ) { - - drop(msg.0); -reply.reply(()); + drop(msg.0); + reply.reply(()); } } fn released(weak: &Weak<()>) { diff --git a/concurrency/tests/semantics_gaps.rs b/concurrency/tests/semantics_gaps.rs index 900a86bf..c596ef75 100644 --- a/concurrency/tests/semantics_gaps.rs +++ b/concurrency/tests/semantics_gaps.rs @@ -107,11 +107,10 @@ mod native { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.trap_exit(true); - ctx.link(&watch.0); - ctx.monitor(&watch.0); -reply.reply(()); + ctx.trap_exit(true); + ctx.link(&watch.0); + ctx.monitor(&watch.0); + reply.reply(()); } } impl Handler for Observer { @@ -121,9 +120,8 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - let _ = self.0.send(Event::Ping(ping.0)); -reply.reply(()); + let _ = self.0.send(Event::Ping(ping.0)); + reply.reply(()); } } impl Handler for Observer { @@ -133,9 +131,8 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - let _ = self.0.send(Event::Down(down.reason)); -reply.reply(()); + let _ = self.0.send(Event::Down(down.reason)); + reply.reply(()); } } @@ -148,15 +145,14 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - for i in 0..burst.count { - burst.to.send(Ping(i)).unwrap(); - } - match burst.reason { - Some(reason) => ctx.stop_with_reason(reason), - None => panic!("source panicked"), - } -reply.reply(()); + for i in 0..burst.count { + burst.to.send(Ping(i)).unwrap(); + } + match burst.reason { + Some(reason) => ctx.stop_with_reason(reason), + None => panic!("source panicked"), + } + reply.reply(()); } } @@ -267,11 +263,10 @@ mod tasks { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.trap_exit(true); - ctx.link(&watch.0); - ctx.monitor(&watch.0); -reply.reply(()); + ctx.trap_exit(true); + ctx.link(&watch.0); + ctx.monitor(&watch.0); + reply.reply(()); } } impl Handler for Observer { @@ -281,9 +276,8 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - let _ = self.0.send(Event::Ping(ping.0)); -reply.reply(()); + let _ = self.0.send(Event::Ping(ping.0)); + reply.reply(()); } } impl Handler for Observer { @@ -293,9 +287,8 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - let _ = self.0.send(Event::Down(down.reason)); -reply.reply(()); + let _ = self.0.send(Event::Down(down.reason)); + reply.reply(()); } } @@ -308,15 +301,14 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - for i in 0..burst.count { - burst.to.send(Ping(i)).unwrap(); - } - match burst.reason { - Some(reason) => ctx.stop_with_reason(reason), - None => panic!("source panicked"), - } -reply.reply(()); + for i in 0..burst.count { + burst.to.send(Ping(i)).unwrap(); + } + match burst.reason { + Some(reason) => ctx.stop_with_reason(reason), + None => panic!("source panicked"), + } + reply.reply(()); } } @@ -367,9 +359,8 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - let _ = self.0.send(Instant::now()); -reply.reply(()); + let _ = self.0.send(Instant::now()); + reply.reply(()); } } @@ -458,9 +449,8 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - let _ = self.log.send((self.generation, stamp.0)); -reply.reply(()); + let _ = self.log.send((self.generation, stamp.0)); + reply.reply(()); } } impl Handler for Clocked { diff --git a/concurrency/tests/shutdown_publication.rs b/concurrency/tests/shutdown_publication.rs index 8f01bbab..8147acb4 100644 --- a/concurrency/tests/shutdown_publication.rs +++ b/concurrency/tests/shutdown_publication.rs @@ -72,15 +72,14 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - match msg { - Command::Hold { entered, release } => { - let _ = entered.send(()); - let _ = release.await; - } - Command::Work => panic!("queued work must be discarded after stop"), + match msg { + Command::Hold { entered, release } => { + let _ = entered.send(()); + let _ = release.await; } -reply.reply(()); + Command::Work => panic!("queued work must be discarded after stop"), + } + reply.reply(()); } } struct Cleanup(ChildHandle); diff --git a/concurrency/tests/sim.rs b/concurrency/tests/sim.rs index 5dc1f3f1..2cce010d 100644 --- a/concurrency/tests/sim.rs +++ b/concurrency/tests/sim.rs @@ -91,9 +91,7 @@ impl Handler for Node { _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - panic!("injected crash after readiness"); - + panic!("injected crash after readiness"); } } diff --git a/concurrency/tests/sim_explore.rs b/concurrency/tests/sim_explore.rs index 10faa202..474ea564 100644 --- a/concurrency/tests/sim_explore.rs +++ b/concurrency/tests/sim_explore.rs @@ -76,10 +76,9 @@ impl Handler for Store { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.log.push(Access::Write(value)); - self.value = value; -reply.reply(()); + self.log.push(Access::Write(value)); + self.value = value; + reply.reply(()); } } impl Handler for Store { @@ -113,13 +112,12 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - for _ in 0..self.noise { - spawned_rt::tasks::yield_now().await; - } - let value = self.store.request(Get).await.unwrap(); - self.store.request(Set(value + 1)).await.unwrap(); -reply.reply(()); + for _ in 0..self.noise { + spawned_rt::tasks::yield_now().await; + } + let value = self.store.request(Get).await.unwrap(); + self.store.request(Set(value + 1)).await.unwrap(); + reply.reply(()); } } @@ -233,9 +231,7 @@ impl Handler for Node { _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - panic!("injected crash after readiness"); - + panic!("injected crash after readiness"); } } diff --git a/concurrency/tests/stack_budget.rs b/concurrency/tests/stack_budget.rs index 06a806b3..48dedf71 100644 --- a/concurrency/tests/stack_budget.rs +++ b/concurrency/tests/stack_budget.rs @@ -229,11 +229,10 @@ mod tasks_chain { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - if self.generation == 1 { - panic!("injected stack-budget crash"); - } -reply.reply(()); + if self.generation == 1 { + panic!("injected stack-budget crash"); + } + reply.reply(()); } } impl Handler for Flaky { diff --git a/concurrency/tests/supervision_trace.rs b/concurrency/tests/supervision_trace.rs index 960fdde4..6a900fb7 100644 --- a/concurrency/tests/supervision_trace.rs +++ b/concurrency/tests/supervision_trace.rs @@ -104,10 +104,9 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - msg.0.send(()).unwrap(); - let _ = msg.1.await; -reply.reply(()); + msg.0.send(()).unwrap(); + let _ = msg.1.await; + reply.reply(()); } } struct DropBomb; @@ -126,9 +125,7 @@ impl Handler for Worker { _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - panic!("discarded message must not be handled"); - + panic!("discarded message must not be handled"); } } @@ -217,9 +214,7 @@ impl Handler for Worker { _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - panic!("injected child failure"); - + panic!("injected child failure"); } } impl Handler for Worker { @@ -229,9 +224,8 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.stop(); -reply.reply(()); + ctx.stop(); + reply.reply(()); } } impl Handler for Worker { @@ -241,9 +235,8 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.stop_with_reason(msg.0); -reply.reply(()); + ctx.stop_with_reason(msg.0); + reply.reply(()); } } @@ -539,9 +532,7 @@ impl Handler for ExitOnceDuringRestart { _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - panic!("injected child failure"); - + panic!("injected child failure"); } } diff --git a/concurrency/tests/support/failure_actor.rs b/concurrency/tests/support/failure_actor.rs index 94d69f86..63845a4d 100644 --- a/concurrency/tests/support/failure_actor.rs +++ b/concurrency/tests/support/failure_actor.rs @@ -199,11 +199,10 @@ impl spawned_concurrency::tasks::Handler for AsyncWorker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &spawned_concurrency::tasks::Context, ) { - - if let Some(sender) = &self.0.down { - let _ = sender.send(down); - } -reply.reply(()); + if let Some(sender) = &self.0.down { + let _ = sender.send(down); + } + reply.reply(()); } } pub struct NativeWorker(pub Worker); @@ -263,11 +262,10 @@ impl spawned_concurrency::threads::Handler for NativeWorker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &spawned_concurrency::threads::Context, ) { - - if let Some(sender) = &self.0.down { - let _ = sender.send(down); - } -reply.reply(()); + if let Some(sender) = &self.0.down { + let _ = sender.send(down); + } + reply.reply(()); } } #[derive(Clone)] diff --git a/concurrency/tests/tree_trace.rs b/concurrency/tests/tree_trace.rs index 1b58691a..093c7833 100644 --- a/concurrency/tests/tree_trace.rs +++ b/concurrency/tests/tree_trace.rs @@ -26,9 +26,7 @@ impl Handler for Worker { _reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - panic!("nested trace failure"); - + panic!("nested trace failure"); } } fn config(strategy: SupervisorStrategy) -> SupervisorConfig { diff --git a/examples/capabilities/src/bin/coordinated_shutdown.rs b/examples/capabilities/src/bin/coordinated_shutdown.rs index c8108c6b..b8aec835 100644 --- a/examples/capabilities/src/bin/coordinated_shutdown.rs +++ b/examples/capabilities/src/bin/coordinated_shutdown.rs @@ -42,9 +42,8 @@ impl Handler for Peers { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.peers.push(msg.0); -reply.reply(()); + self.peers.push(msg.0); + reply.reply(()); } } impl Handler for Peers { @@ -73,9 +72,8 @@ impl Handler for Importer { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.peers.send(Peer(msg.0)).unwrap(); -reply.reply(()); + self.peers.send(Peer(msg.0)).unwrap(); + reply.reply(()); } } impl Handler for Importer { diff --git a/examples/capabilities/src/bin/execution_profiles.rs b/examples/capabilities/src/bin/execution_profiles.rs index 6e6746bf..6ee9b96a 100644 --- a/examples/capabilities/src/bin/execution_profiles.rs +++ b/examples/capabilities/src/bin/execution_profiles.rs @@ -19,9 +19,9 @@ impl Handler for Worker { _: &Context, ) { reply.reply(( - std::thread::current().name().unwrap_or_default().to_owned(), - spawned_rt::tasks::affinity::current_cpu().ok(), - )); + std::thread::current().name().unwrap_or_default().to_owned(), + spawned_rt::tasks::affinity::current_cpu().ok(), + )); } } #[tokio::main(flavor = "multi_thread", worker_threads = 2)] diff --git a/examples/capabilities/src/bin/graceful_drain.rs b/examples/capabilities/src/bin/graceful_drain.rs index cc6b32de..e13fe4bf 100644 --- a/examples/capabilities/src/bin/graceful_drain.rs +++ b/examples/capabilities/src/bin/graceful_drain.rs @@ -16,9 +16,8 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0 += 1; -reply.reply(()); + self.0 += 1; + reply.reply(()); } } struct Fence; diff --git a/examples/capabilities/src/bin/native_lifecycle.rs b/examples/capabilities/src/bin/native_lifecycle.rs index e43a2ef1..f57992bb 100644 --- a/examples/capabilities/src/bin/native_lifecycle.rs +++ b/examples/capabilities/src/bin/native_lifecycle.rs @@ -25,10 +25,9 @@ impl Handler for Observer { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.counts.1 += 1; - self.downs.send(event).unwrap(); -reply.reply(()); + self.counts.1 += 1; + self.downs.send(event).unwrap(); + reply.reply(()); } } struct Counts; diff --git a/examples/capabilities/src/bin/shutdown_diagnostics.rs b/examples/capabilities/src/bin/shutdown_diagnostics.rs index 5142644c..2dabf4ca 100644 --- a/examples/capabilities/src/bin/shutdown_diagnostics.rs +++ b/examples/capabilities/src/bin/shutdown_diagnostics.rs @@ -22,10 +22,9 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - let _ = msg.0.send(()); - let _ = msg.1.await; -reply.reply(()); + let _ = msg.0.send(()); + let _ = msg.1.await; + reply.reply(()); } } struct Unhealthy; @@ -39,10 +38,9 @@ impl Handler for Worker { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - // Cooperative abnormal exit, without a panic or forced interruption. - ctx.stop_with_reason(ExitReason::Panic("reported unhealthy worker".into())); -reply.reply(()); + // Cooperative abnormal exit, without a panic or forced interruption. + ctx.stop_with_reason(ExitReason::Panic("reported unhealthy worker".into())); + reply.reply(()); } } diff --git a/examples/capabilities/src/bin/streams_and_recipients.rs b/examples/capabilities/src/bin/streams_and_recipients.rs index d28c680d..281bb3e1 100644 --- a/examples/capabilities/src/bin/streams_and_recipients.rs +++ b/examples/capabilities/src/bin/streams_and_recipients.rs @@ -25,9 +25,8 @@ mod asynchronous { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0 += N * msg.0; -reply.reply(()); + self.0 += N * msg.0; + reply.reply(()); } } impl Handler for Counter { @@ -91,9 +90,8 @@ mod synchronous { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0 += N * msg.0; -reply.reply(()); + self.0 += N * msg.0; + reply.reply(()); } } impl Handler for Counter { diff --git a/examples/exit_reason/src/main.rs b/examples/exit_reason/src/main.rs index f853a5b8..64e2d1dc 100644 --- a/examples/exit_reason/src/main.rs +++ b/examples/exit_reason/src/main.rs @@ -193,9 +193,8 @@ fn main() { reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.demonitor(msg.0); -reply.reply(()); + ctx.demonitor(msg.0); + reply.reply(()); } } impl Handler for Observer { @@ -205,14 +204,13 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, _ctx: &Context, ) { - - tracing::info!( - "[observer] received Down for {} ({})", - msg.monitor_ref, - msg.reason - ); - self.log.lock().unwrap().push(msg); -reply.reply(()); + tracing::info!( + "[observer] received Down for {} ({})", + msg.monitor_ref, + msg.reason + ); + self.log.lock().unwrap().push(msg); + reply.reply(()); } } @@ -291,9 +289,8 @@ reply.reply(()); reply: spawned_concurrency::reply::PendingReply<'_, ()>, ctx: &Context, ) { - - ctx.link(&msg.0); -reply.reply(()); + ctx.link(&msg.0); + reply.reply(()); } } diff --git a/examples/thread_only/src/main.rs b/examples/thread_only/src/main.rs index 3ebca555..54bcf3bb 100644 --- a/examples/thread_only/src/main.rs +++ b/examples/thread_only/src/main.rs @@ -32,17 +32,25 @@ impl spawned_concurrency::message::Message for Tick { struct Observer(Arc); impl Actor for Observer {} impl Handler for Observer { - fn handle(&mut self, _: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { - + fn handle( + &mut self, + _: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { self.0.fetch_add(1, Ordering::SeqCst); -reply.reply(()); + reply.reply(()); } } impl Handler for Observer { - fn handle(&mut self, _: Tick, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { - + fn handle( + &mut self, + _: Tick, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { self.0.fetch_add(1, Ordering::SeqCst); -reply.reply(()); + reply.reply(()); } } fn main() { diff --git a/testing/tests/tasks_kit.rs b/testing/tests/tasks_kit.rs index 2d64ed87..5520fff9 100644 --- a/testing/tests/tasks_kit.rs +++ b/testing/tests/tasks_kit.rs @@ -231,9 +231,8 @@ impl Handler for Counter { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0.fetch_add(1, Ordering::SeqCst); -reply.reply(()); + self.0.fetch_add(1, Ordering::SeqCst); + reply.reply(()); } } impl Handler for Counter { @@ -348,11 +347,10 @@ impl Handler for Stuck { reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context, ) { - - self.0.store(true, Ordering::SeqCst); - // Blocking code is never preempted, so shutdown cannot finish in time. - std::thread::sleep(Duration::from_millis(500)); -reply.reply(()); + self.0.store(true, Ordering::SeqCst); + // Blocking code is never preempted, so shutdown cannot finish in time. + std::thread::sleep(Duration::from_millis(500)); + reply.reply(()); } } From ef37178ddd5f4b6f8236de236d01a9bc95661848 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:54:20 -0300 Subject: [PATCH 031/208] style: rustfmt --- concurrency/tests/metrics.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/concurrency/tests/metrics.rs b/concurrency/tests/metrics.rs index 10a413d3..dd69c161 100644 --- a/concurrency/tests/metrics.rs +++ b/concurrency/tests/metrics.rs @@ -106,7 +106,10 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { Err(ActorError::RequestTimeout) )); assert!(matches!( - Response::from(actor.request_raw_until(Ping, spawned_concurrency::Deadline::after(Duration::ZERO))).await, + Response::from( + actor.request_raw_until(Ping, spawned_concurrency::Deadline::after(Duration::ZERO)) + ) + .await, Err(ActorError::RequestTimeout) )); // Dropping either kind of caller wrapper does not cancel actor work. @@ -128,15 +131,22 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { .await .unwrap(); Response::from(actor.request_raw(Ping)).await.unwrap(); - Response::from(actor.request_raw_until(Ping, spawned_concurrency::Deadline::after(Duration::from_secs(2)))) - .await - .unwrap(); + Response::from(actor.request_raw_until( + Ping, + spawned_concurrency::Deadline::after(Duration::from_secs(2)), + )) + .await + .unwrap(); actor.context().stop(); assert_eq!(actor.wait_exit().await, ExitReason::Normal); assert_eq!(actor.mailbox_depth(), 0); assert!(actor.send(Ping).is_err()); assert!(matches!( - Response::from(actor.request_raw_until(Ping, spawned_concurrency::Deadline::after(Duration::from_secs(2)))).await, + Response::from(actor.request_raw_until( + Ping, + spawned_concurrency::Deadline::after(Duration::from_secs(2)) + )) + .await, Err(ActorError::ActorStopped) )); From 8496cc46eddb79d2bae87f079276450f54e7a487 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:56:02 -0300 Subject: [PATCH 032/208] docs: deferred replies guide section, semantics R1/R3/R5-R7, migration notes; example deferred_replies --- docs/GUIDE.md | 87 ++++- docs/MIGRATION.md | 41 ++ docs/design/semantics.md | 65 +++- examples/README.md | 1 + .../capabilities/src/bin/deferred_replies.rs | 356 ++++++++++++++++++ 5 files changed, 534 insertions(+), 16 deletions(-) create mode 100644 examples/capabilities/src/bin/deferred_replies.rs diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 7c8513f8..25b6c5e6 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -20,6 +20,7 @@ or model that checks each one, are collected in the - [Type Erasure: Recipient and Receiver](#type-erasure-recipient-and-receiver) - [Registry](#registry) - [Response\](#responset) +- [Deferred replies and pipe-to-self](#deferred-replies-and-pipe-to-self) (v0.8) - [Message Trait](#message-trait) - [Error Handling](#error-handling) - [spawned-rt](#spawned-rt) @@ -508,15 +509,95 @@ before message arguments move. Send-only methods have no timeout, and a `request_timeout` attribute on one is a compile error. Manual message calls use `request_with_timeout(msg, limit)`. -Enqueue still happens when the generated method is called. In async mode the -response timer starts when its future is first polled. In native mode the -protocol call blocks immediately. Timeout/drop does not cancel accepted work or +Enqueue still happens when the generated method is called. The deadline is +fixed then, at send (send time plus the timeout, as OTP's `call`), and travels +with the request; in async mode the response future waits until that deadline +when polled. In native mode the protocol call blocks immediately. Timeout/drop does not cancel accepted work or roll back side effects; decide retry safety in application code. These settings do not propagate deadlines to downstream requests. Run the [asserted example](../examples/capabilities/src/bin/protocol_timeouts.rs) in both modes, or follow the [0.7 client migration guide](MIGRATION.md#preparing-for-07). +## Deferred replies and pipe-to-self + +A handler that would await downstream work blocks its whole mailbox. Instead, +take the request's reply obligation out of the turn and answer it later; the +caller and the protocol do not change. + +Every `Handler::handle` receives a `PendingReply<'_, R>`. `#[request_handler]` +methods answer it for you with their return value. A `#[deferred_handler]` +receives it: + +```rust +#[actor(protocol = PeerTableProtocol)] +impl PeerTable { + #[deferred_handler] + async fn wait_for_peer(&mut self, msg: WaitForPeer, reply: PendingReply<'_, Lease>, + _ctx: &Context) { + if let Some(lease) = self.take_lease(&msg.caps) { + reply.reply(lease); + return; + } + match reply.defer() { + Ok(token) => self.waiters.push_back((msg.caps, token)), // a ReplyTo + Err(DeferError::Overloaded) => {} // the caller was already answered Overloaded + Err(DeferError::Closed) => {} // the caller timed out or left + Err(DeferError::NoCaller) => {} // reached by `send` + } + } +} +``` + +Rules (the [design](design/deferred-replies.md) has the full contract): + +- `defer()` takes one of the instance's `Actor::MAX_OUTSTANDING` credits + (default 1024). At the bound the caller gets `Err(Overloaded)` at once. +- A `ReplyTo` is `Send + 'static` and answers exactly once: + `token.reply(v)` returns a `Delivery` (`Delivered`, `CallerGone`, `Expired`, + `OwnerExited`); `token.overloaded()` answers `Err(Overloaded)`. +- Dropping an unanswered token answers `ReplyDropped` (or `ActorStopped` if the + turn panicked or the actor is exiting). `stopped()` may still answer stored + tokens; at the actor's death every open token answers `ActorStopped`. +- The deadline is the caller's, fixed at send. `token.is_closed()` tells you + the caller can no longer be answered; the actor's deadline heap expires + stored tokens on time and returns their credits. +- `PendingReplies` is a bounded table of tokens by key (purges terminal + entries on insert; `InsertError::{Duplicate, Full}` hands the token back). + +**Pipe-to-self jobs** run work owned by the actor and deliver the result as a +message in a later turn (OTP's `Task.Supervisor.async_nolink` plus +`handle_info`): + +```rust +// CPU offload on an owned native thread, with a deadline: +let opts = JobOptions { deadline: Some(Instant::now() + VERDICT_DEADLINE), ..Default::default() }; +ctx.pipe_blocking_with(opts, move |_job| check(&object), move |res| Verdict { id, res })?; + +// Delegation: the obligation and its credit stay here. +let token = reply.defer()?; +let downstream = other.request_raw_until(Lookup(key), token.deadline().map_or(Deadline::Never, Deadline::At)); +ctx.pipe_reply(token, async move { downstream?.await }, |res, token| Delegated { res, token })?; +``` + +- `done(outcome)` runs in the owner's turn; `JobOutcome::Err` carries + `JobError::Panicked` or `JobError::DeadlineExceeded` (the work keeps running + and keeps its credit). +- `pipe_keyed` / `JobOptions::key` cancel the previous job with that key and + fence its queued messages; `JobHandle::cancel` fences one job. +- `JobCtx::emit_blocking` (and `emit` for async jobs) sends fenced messages, + bounded by `JobOptions::emit_credits`; the terminal message is always last. +- Each unlinked job holds one credit; native jobs also count against + `Actor::MAX_NATIVE_JOBS`. Every bound rejects at once (`PipeError`), never + waits. `pipe_reply*` errors hand the token back. +- Owner exit cancels every job; after death the jobs are joined, bounded by + `DEFAULT_STOP_DEADLINE`, and stragglers are reported (`unfinished_at_stop`). +- `threads` mode offers `pipe_blocking_with` and `pipe_reply_blocking_with` + (one owned native thread per job). + +Test a manual handler directly with `spawned_test::reply_pair()`. Run the +[deferred replies example](../examples/capabilities/src/bin/deferred_replies.rs). + ## Message Trait The `Message` trait defines a message type and its expected reply type: diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 816f791f..2dfcacd9 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -12,6 +12,47 @@ This is a breaking release. The notes below grow with the implementation: - `ActorError` is `#[non_exhaustive]` and gains `ReplyDropped` and `Overloaded`; add a wildcard arm to exhaustive matches. +**Handlers.** `#[actor]` code is unchanged. A manual `impl Handler` now +takes the request's reply obligation and returns `()`: + +```rust +// before +impl Handler for Counter { + async fn handle(&mut self, _: GetCount, _: &Context) -> u64 { self.count } +} +// after +impl Handler for Counter { + async fn handle(&mut self, _: GetCount, reply: PendingReply<'_, u64>, _: &Context) { + reply.reply(self.count); + } +} +``` + +A handler that returns without answering a request now gives the caller +`ReplyDropped` (a panic still gives `ActorStopped`). Sends reach the same +`handle` with a detached `PendingReply`. To call a manual handler directly in +a test, use `spawned_test::reply_pair()`. + +**Timeouts.** The request timeout starts at send, not at first poll. +`request_raw` gains the 5 s default; use `request_raw_until(msg, +Deadline::Never)` for the old unbounded wait. + +**Raw API.** `request_raw` returns a `ReplyReceiver` (a future in tasks +mode, `recv()` in threads mode) yielding `Result`. +`Response::from_with_timeout` is removed: build the deadline first, +`Response::from(actor.request_raw_until(msg, Deadline::after(t)))`. +`Receiver` implementors implement `request_raw_until` (`request_raw` is +provided). A `recv_timeout` on the old native receiver has no counterpart: +the deadline is fixed when the request is sent. + +**Metrics.** `metrics::Snapshot` gains `requests_reply_dropped_total` and +`requests_overloaded_total`. + +**Lifecycle.** At death the registry closes (open deferred tokens answer +`ActorStopped`), then the mailbox is disposed, then actor state is dropped: +a state destructor that sends to its own actor now fails with `ActorStopped` +instead of enqueueing into a mailbox about to be dropped. + ## 0.7.3 **Behaviour change (supervision).** A Transient child's replacement (any generation after the child was first diff --git a/docs/design/semantics.md b/docs/design/semantics.md index 85f64bf0..01be837c 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -89,12 +89,19 @@ planned (rank 17). ## 3. Requests -**R1. Three outcomes.** `request`, `request_with_timeout`, protocol calls and +**R1. One terminal outcome.** Every request has a reply slot +([deferred replies](deferred-replies.md) §1) that is discharged exactly once. +`request`, `request_with_timeout`, protocol calls and `spawned::tasks::request` end in exactly one of `Ok(reply)`, -`Err(ActorStopped)` (not accepted, or the reply sender was dropped: the actor -stopped before replying, the handler panicked, or the request was still queued -at stop) or `Err(RequestTimeout)`. A request racing shutdown gets its reply or -`ActorStopped`; a slow teardown can still produce a timeout first. +`Err(ActorStopped)` (not accepted, the handler panicked, the request was still +queued at the owner's death, or a deferred token was open at death), +`Err(RequestTimeout)` (the deadline fixed at send passed), +`Err(ReplyDropped)` (the handler dropped the reply obligation unanswered) or +`Err(Overloaded)` (`defer()` at `MAX_OUTSTANDING`, or the handler answered +`overloaded()`). A request racing shutdown gets its reply or `ActorStopped`; +a slow teardown can still produce a timeout first. `ActorStopped` from a +deferred token is reported only at the owner's actual death, never while +`stopped()` still runs. Checked by: [`actor_scenarios`][actor_scenarios], `src/tasks/actor.rs::request_timeout`, [`timeout_outcomes`][ex_timeout_outcomes] example. @@ -102,11 +109,16 @@ Checked by: [`actor_scenarios`][actor_scenarios], `src/tasks/actor.rs::request_t only the caller that sent that request. Checked by: [`actor_scenarios::concurrent_producers_preserve_order_and_reply_identity`][actor_scenarios]. -**R3. Timeouts.** Default 5 s. Precedence: `with_request_timeout` handle, -`#[request_timeout]` method attribute, `#[protocol(timeout)]`, 5 s. Send-only -methods have no timeout. tasks: `request`-family futures enqueue on first poll -and protocol methods enqueue when called; the timer starts at first poll. -threads: the call blocks and the timer starts immediately. +**R3. Timeouts.** Default 5 s, as `gen_server:call/2`; `request_raw` has the +same default and `request_raw_until(msg, Deadline::Never)` waits without one. +Precedence: `with_request_timeout` handle, `#[request_timeout]` method +attribute, `#[protocol(timeout)]`, 5 s. Send-only methods have no timeout. +The deadline is fixed when the request is sent (send time plus the timeout) +and travels in the reply slot: there is one deadline, not a caller timer plus +a server one. tasks: `request`-family futures enqueue on first poll and +protocol methods enqueue when called; either way the deadline starts at the +enqueue. threads: the call blocks. Expired requests are still dispatched (OTP); +a late reply is dropped (`Delivery::Expired`). Checked by: [`protocol_timeouts`][protocol_timeouts], [`cancellation_safety::protocol_calls_are_cancel_safe_and_send_eagerly`][cancellation_safety], [`protocol_timeouts`][ex_protocol_timeouts] example (both modes). @@ -117,12 +129,37 @@ or roll back effects. The caller cannot tell whether it ran. Checked by: [`cancellation_safety::request_family_is_cancel_safe`][cancellation_safety], [`request_histories`][request_histories] (timed-out operations are "pending"). -**R5. Handler panic.** A panic while handling a request drops its reply sender: -the caller gets an error, not a reply, and the actor exits with `Panic`. -Checked by: [`semantics_gaps`][semantics_gaps]`::{native,tasks}::panicking_request_handler_gives_the_caller_actor_stopped` +**R5. Handler panic.** A panic while handling a request drops its reply +obligation in the panicking turn: the drop is committed at the turn's end as +`ActorStopped` (a turn that returns without answering gives `ReplyDropped`), +and the actor exits with `Panic`. +Checked by: [`deferred_replies`][deferred_replies] +`::{tasks_mode::panicking_turn_answers_actor_stopped, threads_mode::panic_and_death_answer_actor_stopped}`, +[`semantics_gaps`][semantics_gaps]`::{native,tasks}::panicking_request_handler_gives_the_caller_actor_stopped` (exactly `ActorStopped`, every backend, a managed group, `Recipient` and raw receivers), [`panic_safety`][panic_safety]. +**R6. Deferred replies.** A handler may answer later: `PendingReply::defer()` +takes one of the instance's `MAX_OUTSTANDING` credits (or answers +`Overloaded` at once) and returns a storable `ReplyTo`. The token is answered +from a later turn, a job or `stopped()`; dropping it answers `ReplyDropped` +between turns, is committed at the turn's end inside a turn, and answers +`ActorStopped` while the owner exits. At the owner's death every open token +becomes `ActorStopped`, wherever it is held. The owner's deadline heap expires +deferred tokens (returning their credits) even when no caller polls. Callers +and protocols cannot tell a deferred reply from an immediate one. +Checked by: `reply_slot_loom` (every concurrent pair of the slot word), +[`deferred_replies`][deferred_replies], `reply::registry::tests`. + +**R7. Pipe-to-self jobs.** `ctx.pipe*` runs work owned by the instance and +delivers `done(outcome)` as one message in a later turn; a panic arrives as +`JobError::Panicked`, a job deadline as `JobError::DeadlineExceeded` while the +work keeps its credit. Messages of cancelled or replaced (keyed) jobs are +fenced at dispatch; a job's emits stay FIFO with the terminal message last. +Owner exit cancels every job; they are joined after death, bounded by the +cleanup deadline, and reported as `unfinished_at_stop`. +Checked by: [`deferred_jobs`][deferred_jobs]. + ## 4. Lifecycle **L1. Hook order.** `started()` runs once before any handler; `stopped()` runs @@ -556,6 +593,8 @@ ignored by non-trapping peers (X3), and the Permanent/Transient/Temporary table See the [ranked backlog](../../ROADMAP.md#ranked-backlog). +[deferred_replies]: ../../concurrency/tests/deferred_replies.rs +[deferred_jobs]: ../../concurrency/tests/deferred_jobs.rs [actor_scenarios]: ../../concurrency/tests/actor_scenarios.rs [cancellation_safety]: ../../concurrency/tests/cancellation_safety.rs [cohort_trace]: ../../concurrency/tests/cohort_trace.rs diff --git a/examples/README.md b/examples/README.md index 475d4be7..050421ce 100644 --- a/examples/README.md +++ b/examples/README.md @@ -31,6 +31,7 @@ Usage of each feature is documented in the [guide](../docs/GUIDE.md). | Nested supervisors, OneForAll / RestForOne, v0.7 | [supervision_tree](capabilities/src/bin/supervision_tree.rs) | n/a | | Tokio-free dependency graph, protocols, lifecycle, monitors and links, v0.7 | n/a | [thread_only](thread_only/README.md) (isolated workspace) | | Actors on separate runtimes | [runtime_isolation](capabilities/src/bin/runtime_isolation.rs) | n/a | +| Deferred replies and pipe-to-self (rank 11) | [deferred_replies](capabilities/src/bin/deferred_replies.rs): delegation with `pipe_reply`, CPU offload with a job deadline, requests by id in `PendingReplies`, waiters instead of polling, owner stop | tasks mode | | Request timeout outcomes | [timeout_outcomes](capabilities/src/bin/timeout_outcomes.rs), [protocol_timeouts](capabilities/src/bin/protocol_timeouts.rs) | [protocol_timeouts](capabilities/src/bin/protocol_timeouts.rs) | | Draining a finite batch before stop | [graceful_drain](capabilities/src/bin/graceful_drain.rs) | gap | | Byte-budgeted pipeline (application adapter) | [byte_budget_pipeline](capabilities/src/bin/byte_budget_pipeline.rs) | gap | diff --git a/examples/capabilities/src/bin/deferred_replies.rs b/examples/capabilities/src/bin/deferred_replies.rs new file mode 100644 index 00000000..c0f8332c --- /dev/null +++ b/examples/capabilities/src/bin/deferred_replies.rs @@ -0,0 +1,356 @@ +//! Deferred replies and pipe-to-self (rank 11), asserted: +//! +//! 1. a router delegates a slow request with `pipe_reply` and keeps serving +//! its mailbox meanwhile; +//! 2. CPU work offloaded with `pipe_blocking_with` and a job deadline; +//! 3. the ethrex `outgoing_request` pattern: request ids in a bounded +//! `PendingReplies`, answered when the "peer" responds, late responses +//! ignored; +//! 4. the `get_best_peer` waiter queue: callers wait (no polling) until +//! capacity frees, and `Overloaded` at the bound; +//! 5. owner stop with stored tokens: `stopped()` answers, death closes the +//! rest with `ActorStopped`. +use spawned_concurrency::message::Message; +use spawned_concurrency::tasks::{ + Actor, ActorRef, ActorStart, Context, Deadline, Handler, InsertError, JobError, JobOptions, + JobOutcome, PendingReplies, PendingReply, ReplyTo, +}; +use spawned_concurrency::{actor, protocol, ActorError, DeferError, Response}; +use std::collections::VecDeque; +use std::time::Duration; + +// --------------------------------------------------------------------------- +// 1. Router delegating with pipe_reply +// --------------------------------------------------------------------------- + +#[protocol] +pub trait BackendProtocol: Send + Sync { + fn lookup(&self, millis: u64) -> Response; +} + +pub struct Backend; +#[actor(protocol = BackendProtocol)] +impl Backend { + #[request_handler] + async fn lookup(&mut self, msg: backend_protocol::Lookup, _: &Context) -> u64 { + spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; + msg.millis + } +} + +#[protocol] +pub trait RouterProtocol: Send + Sync { + fn route(&self, millis: u64) -> Response; + fn ping(&self) -> Response<&'static str>; +} + +pub struct Router { + backend: ActorRef, +} +struct Delegated { + res: JobOutcome>, + token: ReplyTo, +} +impl Message for Delegated { + type Result = (); +} + +#[actor(protocol = RouterProtocol)] +impl Router { + #[deferred_handler] + async fn route(&mut self, msg: router_protocol::Route, reply: PendingReply<'_, u64>, ctx: &Context) { + let Ok(token) = reply.defer() else { return }; + // The downstream request inherits the caller's deadline. + let deadline = token.deadline().map_or(Deadline::Never, Deadline::At); + let downstream = self + .backend + .request_raw_until(backend_protocol::Lookup { millis: msg.millis }, deadline); + let work = async move { downstream?.await }; + if let Err(e) = ctx.pipe_reply(token, work, |res, token| Delegated { res, token }) { + e.into_token().overloaded(); + } + } + + #[request_handler] + async fn ping(&mut self, _: router_protocol::Ping, _: &Context) -> &'static str { + "pong" + } + + #[send_handler] + async fn delegated(&mut self, msg: Delegated, _: &Context) { + match msg.res { + JobOutcome::Ok(Ok(v)) => { + msg.token.reply(v); + } + _ => msg.token.overloaded(), + } + } +} + +// --------------------------------------------------------------------------- +// 2. CPU offload with a job deadline +// --------------------------------------------------------------------------- + +pub struct Verifier { + verdicts: Vec, +} +struct Verify { + id: u32, + spin: Duration, +} +impl Message for Verify { + type Result = (); +} +struct Verdict { + id: u32, + res: JobOutcome, +} +impl Message for Verdict { + type Result = (); +} +struct Verdicts; +impl Message for Verdicts { + type Result = Vec; +} + +#[actor] +impl Verifier { + #[send_handler] + async fn verify(&mut self, msg: Verify, ctx: &Context) { + let opts = JobOptions { + deadline: Some(spawned_rt::time::Instant::now() + Duration::from_millis(100)), + ..JobOptions::default() + }; + let (id, spin) = (msg.id, msg.spin); + let started = ctx.pipe_blocking_with( + opts, + move |_job| { + std::thread::sleep(spin); // stands in for a CPU-bound check + true + }, + move |res| Verdict { id, res }, + ); + if let Err(e) = started { + self.verdicts.push(format!("{id}: rejected ({e})")); + } + } + + #[send_handler] + async fn verdict(&mut self, msg: Verdict, _: &Context) { + self.verdicts.push(match msg.res { + JobOutcome::Ok(valid) => format!("{}: valid={valid}", msg.id), + JobOutcome::Err(JobError::DeadlineExceeded) => format!("{}: timed out", msg.id), + JobOutcome::Err(JobError::Panicked(m)) => format!("{}: panicked {m}", msg.id), + }); + } + + #[request_handler] + async fn verdicts(&mut self, _: Verdicts, _: &Context) -> Vec { + std::mem::take(&mut self.verdicts) + } +} + +// --------------------------------------------------------------------------- +// 3. Outgoing requests by id (ethrex `outgoing_request`) +// --------------------------------------------------------------------------- + +pub struct Connection { + current_requests: PendingReplies, +} +struct Outgoing { + id: u64, +} +impl Message for Outgoing { + type Result = String; +} +struct PeerResponse { + id: u64, + body: String, +} +impl Message for PeerResponse { + type Result = bool; +} + +#[actor] +impl Connection { + #[deferred_handler] + async fn outgoing(&mut self, msg: Outgoing, reply: PendingReply<'_, String>, _: &Context) { + let Ok(token) = reply.defer() else { return }; + match self.current_requests.insert(msg.id, token) { + Ok(()) => {} // the request goes out on the socket here + Err(InsertError::Duplicate(token)) => drop(token), // ReplyDropped; the first caller is kept + Err(InsertError::Full(token)) => token.overloaded(), + } + } + + /// Returns whether the response matched a pending request. + #[request_handler] + async fn peer_response(&mut self, msg: PeerResponse, _: &Context) -> bool { + match self.current_requests.take(&msg.id) { + Some(token) => { + token.reply(msg.body); + true + } + None => false, // late or unknown: expired, abandoned or purged + } + } +} + +// --------------------------------------------------------------------------- +// 4. Waiting for a free peer (ethrex `get_best_peer`) +// --------------------------------------------------------------------------- + +const MAX_WAITERS: usize = 2; + +pub struct PeerTable { + free: u32, + waiters: VecDeque>, +} +struct WaitForPeer; +impl Message for WaitForPeer { + type Result = u32; +} +struct Release(u32); +impl Message for Release { + type Result = (); +} + +#[actor] +impl PeerTable { + #[deferred_handler] + async fn wait_for_peer(&mut self, _: WaitForPeer, reply: PendingReply<'_, u32>, _: &Context) { + if self.free > 0 { + self.free -= 1; + reply.reply(self.free); + return; + } + self.waiters.retain(|t| !t.is_closed()); + if self.waiters.len() >= MAX_WAITERS { + return reply.overloaded(); + } + match reply.defer() { + Ok(token) => self.waiters.push_back(token), + Err(DeferError::Overloaded | DeferError::Closed | DeferError::NoCaller) => {} + } + } + + #[send_handler] + async fn release(&mut self, msg: Release, _: &Context) { + self.free += msg.0; + while self.free > 0 { + let Some(token) = self.waiters.pop_front() else { break }; + if token.is_closed() { + continue; + } + self.free -= 1; + token.reply(self.free); + } + } +} + +// --------------------------------------------------------------------------- +// 5. Owner stop with stored tokens +// --------------------------------------------------------------------------- + +pub struct Parker { + parked: Vec>, + answer_first: usize, +} +struct Park; +impl Message for Park { + type Result = &'static str; +} +struct StopAnswering(usize); +impl Message for StopAnswering { + type Result = (); +} + +#[actor] +impl Parker { + #[deferred_handler] + async fn park(&mut self, _: Park, reply: PendingReply<'_, &'static str>, _: &Context) { + if let Ok(token) = reply.defer() { + self.parked.push(token); + } + } + + /// Stop; `stopped()` answers the first `n` parked callers. + #[request_handler] + async fn stop_answering(&mut self, msg: StopAnswering, ctx: &Context) { + self.answer_first = msg.0; + ctx.stop(); + } + + #[stopped] + async fn on_stop(&mut self, _: &Context) { + let n = self.answer_first.min(self.parked.len()); + for token in self.parked.drain(..n) { + token.reply("answered in stopped()"); + } + } +} + +#[tokio::main(flavor = "multi_thread", worker_threads = 2)] +async fn main() { + // 1. Delegation: the slow request does not block the router. + let backend = Backend.start(); + let router = Router { backend: backend.clone() }.start(); + let slow = router.route(300); + assert_eq!(router.ping().await.unwrap(), "pong"); + assert_eq!(router.route(1).await.unwrap(), 1); + assert_eq!(slow.await.unwrap(), 300); + println!("1. router served other requests while a delegated one was pending"); + + // 2. CPU offload: a fast check, a check past its job deadline. + let verifier = Verifier { verdicts: Vec::new() }.start(); + verifier.send(Verify { id: 1, spin: Duration::from_millis(1) }).unwrap(); + verifier.send(Verify { id: 2, spin: Duration::from_millis(400) }).unwrap(); + let mut verdicts = Vec::new(); + while verdicts.len() < 2 { + verdicts.extend(verifier.request(Verdicts).await.unwrap()); + tokio::time::sleep(Duration::from_millis(10)).await; + } + verdicts.sort(); + assert_eq!(verdicts, ["1: valid=true", "2: timed out"]); + println!("2. {verdicts:?} (the hung check's thread keeps its slot until it returns)"); + + // 3. Outgoing requests by id. + let conn = Connection { + current_requests: PendingReplies::with_capacity(2), + } + .start(); + let a = conn.request_raw(Outgoing { id: 1 }).unwrap(); + let dup = conn.request_raw(Outgoing { id: 1 }).unwrap(); + let b = conn.request_raw(Outgoing { id: 2 }).unwrap(); + let full = conn.request_raw(Outgoing { id: 3 }).unwrap(); + assert!(matches!(dup.await, Err(ActorError::ReplyDropped))); + assert!(matches!(full.await, Err(ActorError::Overloaded))); + drop(b); // the caller gives up: its entry is purged on the next insert + assert!(conn.request(PeerResponse { id: 1, body: "block".into() }).await.unwrap()); + assert_eq!(a.await.unwrap(), "block"); + assert!(!conn.request(PeerResponse { id: 2, body: "late".into() }).await.unwrap()); + println!("3. responses matched by id; duplicate, full and late responses handled"); + + // 4. Waiting for a free peer instead of polling. + let table = PeerTable { free: 0, waiters: VecDeque::new() }.start(); + let first = table.request_raw(WaitForPeer).unwrap(); + let second = table.request_raw(WaitForPeer).unwrap(); + assert!(matches!(table.request(WaitForPeer).await, Err(ActorError::Overloaded))); + table.send(Release(2)).unwrap(); + assert_eq!((first.await.unwrap(), second.await.unwrap()), (1, 0)); + println!("4. waiters answered when capacity freed; Overloaded at the queue bound"); + + // 5. Owner stop: stopped() answers one caller, death answers the other. + let parker = Parker { parked: Vec::new(), answer_first: 0 }.start(); + let one = parker.request_raw_until(Park, Deadline::Never).unwrap(); + let two = parker.request_raw_until(Park, Deadline::Never).unwrap(); + parker.request(StopAnswering(1)).await.unwrap(); + assert_eq!(one.await.unwrap(), "answered in stopped()"); + assert!(matches!(two.await, Err(ActorError::ActorStopped))); + parker.join().await; + println!("5. stopped() answered a stored token; death answered the rest ActorStopped"); + + for actor in [router.child_handle(), backend.child_handle(), verifier.child_handle(), conn.child_handle(), table.child_handle()] { + actor.stop(); + } +} From 4524afd4b13fa46ca776410a53eba7798912a202 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 20:56:30 -0300 Subject: [PATCH 033/208] docs: changelog for rank 11; example index --- CHANGELOG.md | 23 ++++- examples/README.md | 2 +- .../capabilities/src/bin/deferred_replies.rs | 90 ++++++++++++++++--- 3 files changed, 98 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a487a6d..518d93aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,27 @@ (ROADMAP rank 11), implementing [the design](docs/design/deferred-replies.md). Every request now has a reply slot: one packed atomic word (state, token, registry and running-job count) - whose transitions are checked with Loom (`reply_slot_loom`). See - [migration](docs/MIGRATION.md#unreleased). + whose transitions are checked with Loom (`reply_slot_loom`, 15 concurrent + pairs, 5 mutation controls). `Handler::handle` takes a `PendingReply<'_, R>` + and returns `()`; `#[actor]` generates it, and `#[deferred_handler]` methods + receive the obligation and may `defer()` it into a storable `ReplyTo` + (bounded by `Actor::MAX_OUTSTANDING`, `Overloaded` at the bound). Dropped + obligations answer `ReplyDropped` (`ActorStopped` in a panicking turn or + while exiting); at death every open token answers `ActorStopped`. Request + deadlines are fixed at send (`Deadline`, `request_raw_until`; `request_raw` + gains the 5 s default); a per-actor deadline heap and service expire stored + tokens without a polling caller. `PendingReplies` is a bounded token + table. Pipe-to-self jobs: `ctx.pipe`, `pipe_keyed`, `pipe_with`, + `pipe_blocking_with`, `pipe_reply`, `pipe_reply_with`, + `pipe_reply_blocking_with` (threads mode: the blocking forms on owned native + threads), with `JobOutcome`/`JobError::{Panicked, DeadlineExceeded}`, keyed + replacement and cancellation fenced at dispatch, `JobCtx::emit`/ + `emit_blocking` with emit credits and the terminal message last, + `MAX_NATIVE_JOBS`, owner-exit cancellation and a bounded join after death + (`unfinished_at_stop`). `ActorError` is `#[non_exhaustive]` with + `ReplyDropped` and `Overloaded`; `metrics::Snapshot` gains + `requests_reply_dropped_total` and `requests_overloaded_total`. Example: + `deferred_replies`. See [migration](docs/MIGRATION.md#unreleased). - Design (not implementation) for rank 11, deferred typed replies and pipe-to-self, revised over four passes after independent reviews: `docs/design/deferred-replies.md`, client mapping for ethrex and ethlambda, diff --git a/examples/README.md b/examples/README.md index 050421ce..7734f841 100644 --- a/examples/README.md +++ b/examples/README.md @@ -31,7 +31,7 @@ Usage of each feature is documented in the [guide](../docs/GUIDE.md). | Nested supervisors, OneForAll / RestForOne, v0.7 | [supervision_tree](capabilities/src/bin/supervision_tree.rs) | n/a | | Tokio-free dependency graph, protocols, lifecycle, monitors and links, v0.7 | n/a | [thread_only](thread_only/README.md) (isolated workspace) | | Actors on separate runtimes | [runtime_isolation](capabilities/src/bin/runtime_isolation.rs) | n/a | -| Deferred replies and pipe-to-self (rank 11) | [deferred_replies](capabilities/src/bin/deferred_replies.rs): delegation with `pipe_reply`, CPU offload with a job deadline, requests by id in `PendingReplies`, waiters instead of polling, owner stop | tasks mode | +| Deferred replies and pipe-to-self (rank 11) | [deferred_replies](capabilities/src/bin/deferred_replies.rs): delegation with `pipe_reply`, CPU offload with a job deadline, requests by id in `PendingReplies`, waiters instead of polling, owner stop | gap (native jobs are tested in `deferred_jobs`) | | Request timeout outcomes | [timeout_outcomes](capabilities/src/bin/timeout_outcomes.rs), [protocol_timeouts](capabilities/src/bin/protocol_timeouts.rs) | [protocol_timeouts](capabilities/src/bin/protocol_timeouts.rs) | | Draining a finite batch before stop | [graceful_drain](capabilities/src/bin/graceful_drain.rs) | gap | | Byte-budgeted pipeline (application adapter) | [byte_budget_pipeline](capabilities/src/bin/byte_budget_pipeline.rs) | gap | diff --git a/examples/capabilities/src/bin/deferred_replies.rs b/examples/capabilities/src/bin/deferred_replies.rs index c0f8332c..1c172360 100644 --- a/examples/capabilities/src/bin/deferred_replies.rs +++ b/examples/capabilities/src/bin/deferred_replies.rs @@ -58,7 +58,12 @@ impl Message for Delegated { #[actor(protocol = RouterProtocol)] impl Router { #[deferred_handler] - async fn route(&mut self, msg: router_protocol::Route, reply: PendingReply<'_, u64>, ctx: &Context) { + async fn route( + &mut self, + msg: router_protocol::Route, + reply: PendingReply<'_, u64>, + ctx: &Context, + ) { let Ok(token) = reply.defer() else { return }; // The downstream request inherits the caller's deadline. let deadline = token.deadline().map_or(Deadline::Never, Deadline::At); @@ -174,7 +179,12 @@ impl Message for PeerResponse { #[actor] impl Connection { #[deferred_handler] - async fn outgoing(&mut self, msg: Outgoing, reply: PendingReply<'_, String>, _: &Context) { + async fn outgoing( + &mut self, + msg: Outgoing, + reply: PendingReply<'_, String>, + _: &Context, + ) { let Ok(token) = reply.defer() else { return }; match self.current_requests.insert(msg.id, token) { Ok(()) => {} // the request goes out on the socket here @@ -218,7 +228,12 @@ impl Message for Release { #[actor] impl PeerTable { #[deferred_handler] - async fn wait_for_peer(&mut self, _: WaitForPeer, reply: PendingReply<'_, u32>, _: &Context) { + async fn wait_for_peer( + &mut self, + _: WaitForPeer, + reply: PendingReply<'_, u32>, + _: &Context, + ) { if self.free > 0 { self.free -= 1; reply.reply(self.free); @@ -238,7 +253,9 @@ impl PeerTable { async fn release(&mut self, msg: Release, _: &Context) { self.free += msg.0; while self.free > 0 { - let Some(token) = self.waiters.pop_front() else { break }; + let Some(token) = self.waiters.pop_front() else { + break; + }; if token.is_closed() { continue; } @@ -294,7 +311,10 @@ impl Parker { async fn main() { // 1. Delegation: the slow request does not block the router. let backend = Backend.start(); - let router = Router { backend: backend.clone() }.start(); + let router = Router { + backend: backend.clone(), + } + .start(); let slow = router.route(300); assert_eq!(router.ping().await.unwrap(), "pong"); assert_eq!(router.route(1).await.unwrap(), 1); @@ -302,9 +322,22 @@ async fn main() { println!("1. router served other requests while a delegated one was pending"); // 2. CPU offload: a fast check, a check past its job deadline. - let verifier = Verifier { verdicts: Vec::new() }.start(); - verifier.send(Verify { id: 1, spin: Duration::from_millis(1) }).unwrap(); - verifier.send(Verify { id: 2, spin: Duration::from_millis(400) }).unwrap(); + let verifier = Verifier { + verdicts: Vec::new(), + } + .start(); + verifier + .send(Verify { + id: 1, + spin: Duration::from_millis(1), + }) + .unwrap(); + verifier + .send(Verify { + id: 2, + spin: Duration::from_millis(400), + }) + .unwrap(); let mut verdicts = Vec::new(); while verdicts.len() < 2 { verdicts.extend(verifier.request(Verdicts).await.unwrap()); @@ -326,22 +359,45 @@ async fn main() { assert!(matches!(dup.await, Err(ActorError::ReplyDropped))); assert!(matches!(full.await, Err(ActorError::Overloaded))); drop(b); // the caller gives up: its entry is purged on the next insert - assert!(conn.request(PeerResponse { id: 1, body: "block".into() }).await.unwrap()); + assert!(conn + .request(PeerResponse { + id: 1, + body: "block".into() + }) + .await + .unwrap()); assert_eq!(a.await.unwrap(), "block"); - assert!(!conn.request(PeerResponse { id: 2, body: "late".into() }).await.unwrap()); + assert!(!conn + .request(PeerResponse { + id: 2, + body: "late".into() + }) + .await + .unwrap()); println!("3. responses matched by id; duplicate, full and late responses handled"); // 4. Waiting for a free peer instead of polling. - let table = PeerTable { free: 0, waiters: VecDeque::new() }.start(); + let table = PeerTable { + free: 0, + waiters: VecDeque::new(), + } + .start(); let first = table.request_raw(WaitForPeer).unwrap(); let second = table.request_raw(WaitForPeer).unwrap(); - assert!(matches!(table.request(WaitForPeer).await, Err(ActorError::Overloaded))); + assert!(matches!( + table.request(WaitForPeer).await, + Err(ActorError::Overloaded) + )); table.send(Release(2)).unwrap(); assert_eq!((first.await.unwrap(), second.await.unwrap()), (1, 0)); println!("4. waiters answered when capacity freed; Overloaded at the queue bound"); // 5. Owner stop: stopped() answers one caller, death answers the other. - let parker = Parker { parked: Vec::new(), answer_first: 0 }.start(); + let parker = Parker { + parked: Vec::new(), + answer_first: 0, + } + .start(); let one = parker.request_raw_until(Park, Deadline::Never).unwrap(); let two = parker.request_raw_until(Park, Deadline::Never).unwrap(); parker.request(StopAnswering(1)).await.unwrap(); @@ -350,7 +406,13 @@ async fn main() { parker.join().await; println!("5. stopped() answered a stored token; death answered the rest ActorStopped"); - for actor in [router.child_handle(), backend.child_handle(), verifier.child_handle(), conn.child_handle(), table.child_handle()] { + for actor in [ + router.child_handle(), + backend.child_handle(), + verifier.child_handle(), + conn.child_handle(), + table.child_handle(), + ] { actor.stop(); } } From edd572c16b3ad4a81df9e0a0f534d6755fb37143 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 21:10:45 -0300 Subject: [PATCH 034/208] perf(reply): one-pointer ReplyReceiver; the deadline timer lives in the awaiting ReplyFuture (IntoFuture) --- concurrency/src/reply/mod.rs | 112 +++++++++++++++++++++-------------- concurrency/src/response.rs | 8 ++- concurrency/src/tasks/mod.rs | 4 +- 3 files changed, 73 insertions(+), 51 deletions(-) diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index f65742eb..24abe7af 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -391,33 +391,24 @@ impl Drop for ReplyTo { // ReplyReceiver // --------------------------------------------------------------------------- -#[cfg(feature = "async")] -type DeadlineSleep = spawned_rt::tasks::Sleep; -#[cfg(not(feature = "async"))] -type DeadlineSleep = std::future::Pending<()>; +/// The caller's side of a request: `.await` it in `tasks` mode (it is +/// [`IntoFuture`](std::future::IntoFuture)), or call [`recv`](Self::recv) in +/// `threads` mode. Yields `Ok(reply)`, or `RequestTimeout` at the deadline +/// fixed at send, `ReplyDropped`, `Overloaded`, or `ActorStopped` when the +/// owner died. +/// +/// One pointer wide: the deadline timer lives in the awaiting future +/// ([`ReplyFuture`]), not in the receiver, so receivers are cheap to store +/// and move. Dropping it abandons the request: a later reply is dropped and +/// any deferred credit returns at once. +pub struct ReplyReceiver { + slot: Option>>, +} -pin_project_lite::pin_project! { - /// The caller's side of a request: a future in `tasks` mode, and - /// [`recv`](Self::recv) in `threads` mode. Yields `Ok(reply)`, or - /// `RequestTimeout` at the deadline fixed at send, `ReplyDropped`, - /// `Overloaded`, or `ActorStopped` when the owner died. - /// - /// While polled it sleeps until the slot's deadline and then expires the - /// slot. Dropping it abandons the request: a later reply is dropped and - /// any deferred credit returns at once. - pub struct ReplyReceiver { - slot: Arc>, - done: bool, - armed: u64, - #[pin] - sleep: Option, - } - impl PinnedDrop for ReplyReceiver { - fn drop(this: Pin<&mut Self>) { - let this = this.project(); - if !*this.done { - this.slot.abandon(); - } +impl Drop for ReplyReceiver { + fn drop(&mut self) { + if let Some(slot) = self.slot.take() { + slot.abandon(); } } } @@ -425,45 +416,74 @@ pin_project_lite::pin_project! { impl std::fmt::Debug for ReplyReceiver { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("ReplyReceiver") - .field("done", &self.done) + .field("done", &self.slot.is_none()) .finish_non_exhaustive() } } impl ReplyReceiver { fn new(slot: Arc>) -> Self { - Self { - slot, - done: false, - armed: NEVER, - sleep: None, - } + Self { slot: Some(slot) } + } + + fn slot(&self) -> &Arc> { + self.slot.as_ref().expect("ReplyReceiver used after completion") } /// The deadline fixed at send, or `None` for [`Deadline::Never`]. pub fn deadline(&self) -> Option { - Deadline::from_ticks(self.slot.deadline()) + Deadline::from_ticks(self.slot().deadline()) } /// Block the calling thread until the reply, the deadline or the owner's /// death. Do not call it on an async executor thread. pub fn recv(mut self) -> Result { - let result = self.slot.recv_blocking(&now_ticks).map_err(terminal_error); - self.done = true; - result + let slot = self.slot.take().expect("ReplyReceiver used after completion"); + slot.recv_blocking(&now_ticks).map_err(terminal_error) } /// Non-blocking check: `None` while the request is still open. pub fn try_recv(&mut self) -> Option> { - assert!(!self.done, "ReplyReceiver used after completion"); - let result = self.slot.try_take(&now_ticks)?.map_err(terminal_error); - self.done = true; + let result = self.slot().try_take(&now_ticks)?.map_err(terminal_error); + self.slot = None; Some(result) } } #[cfg(feature = "async")] -impl std::future::Future for ReplyReceiver { +impl std::future::IntoFuture for ReplyReceiver { + type Output = Result; + type IntoFuture = ReplyFuture; + fn into_future(self) -> ReplyFuture { + ReplyFuture { + rx: self, + armed: NEVER, + sleep: None, + } + } +} + +#[cfg(feature = "async")] +pin_project_lite::pin_project! { + /// Awaiting a [`ReplyReceiver`]: polls the slot and, while it is open, + /// sleeps until its deadline and then expires it. + pub struct ReplyFuture { + rx: ReplyReceiver, + armed: u64, + #[pin] + sleep: Option, + } +} + +#[cfg(feature = "async")] +impl std::fmt::Debug for ReplyFuture { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ReplyFuture").finish_non_exhaustive() + } +} + +#[cfg(feature = "async")] +impl std::future::Future for ReplyFuture { type Output = Result; fn poll( @@ -472,13 +492,13 @@ impl std::future::Future for ReplyReceiver { ) -> std::task::Poll { use std::task::Poll; let mut this = self.project(); - assert!(!*this.done, "ReplyReceiver polled after completion"); loop { - if let Poll::Ready(result) = this.slot.poll(cx.waker()) { - *this.done = true; + let slot = this.rx.slot(); + if let Poll::Ready(result) = slot.poll(cx.waker()) { + this.rx.slot = None; return Poll::Ready(result.map_err(terminal_error)); } - let deadline = this.slot.deadline(); + let deadline = slot.deadline(); if deadline == NEVER { return Poll::Pending; } @@ -496,7 +516,7 @@ impl std::future::Future for ReplyReceiver { // keep working. Poll::Ready(()) => { this.sleep.set(None); - this.slot.terminate(Terminal::TimedOut); + this.rx.slot().terminate(Terminal::TimedOut); } } } diff --git a/concurrency/src/response.rs b/concurrency/src/response.rs index 2d6f9d56..f27ce410 100644 --- a/concurrency/src/response.rs +++ b/concurrency/src/response.rs @@ -1,7 +1,9 @@ use crate::error::ActorError; #[cfg(feature = "async")] -use crate::reply::ReplyReceiver; +use crate::reply::{ReplyFuture, ReplyReceiver}; use std::future::Future; +#[cfg(feature = "async")] +use std::future::IntoFuture; use std::pin::Pin; use std::time::Duration; @@ -40,7 +42,7 @@ impl RequestOptions { enum ResponseState { /// The deadline is already in the receiver's slot, fixed at send. #[cfg(feature = "async")] - Receiver(Pin>>), + Receiver(Pin>>), Ready(Result), Done, } @@ -152,7 +154,7 @@ impl From, ActorError>> for Response { let observation = crate::instrumentation::RequestObservation::new(); match result { Ok(rx) => Self( - ResponseState::Receiver(Box::pin(rx)), + ResponseState::Receiver(Box::pin(rx.into_future())), #[cfg(feature = "metrics")] Some(observation), ), diff --git a/concurrency/src/tasks/mod.rs b/concurrency/src/tasks/mod.rs index ae802182..c1880fdf 100644 --- a/concurrency/src/tasks/mod.rs +++ b/concurrency/src/tasks/mod.rs @@ -12,8 +12,8 @@ pub use crate::reply::jobs::{ PipeReplyError, }; pub use crate::reply::{ - Deadline, DeferError, Delivery, InsertError, PendingReplies, PendingReply, ReplyReceiver, - ReplyTo, DEFAULT_MAX_OUTSTANDING, + Deadline, DeferError, Delivery, InsertError, PendingReplies, PendingReply, ReplyFuture, + ReplyReceiver, ReplyTo, DEFAULT_MAX_OUTSTANDING, }; pub use crate::response::Response; pub use actor::{ From 5088047dc48acd8896c2614eb98e4ceb8130c451 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 21:11:05 -0300 Subject: [PATCH 035/208] bench: await the receiver via IntoFuture --- concurrency/benches/baseline.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/concurrency/benches/baseline.rs b/concurrency/benches/baseline.rs index 265b363a..c841c0d9 100644 --- a/concurrency/benches/baseline.rs +++ b/concurrency/benches/baseline.rs @@ -3,6 +3,7 @@ mod cli; use spawned_concurrency::{message::Message, tasks, threads, ChildHandle, Deadline, ReplyReceiver}; use std::{ + future::IntoFuture, alloc::{GlobalAlloc, Layout, System}, hint::black_box, sync::{ @@ -154,7 +155,7 @@ impl Target { impl Reply { fn wait(self, rt: &tokio::runtime::Handle) -> Ack { match self { - Self::Tasks(rx) => rt.block_on(rx).expect("reply timeout or dropped"), + Self::Tasks(rx) => rt.block_on(rx.into_future()).expect("reply timeout or dropped"), Self::Tokio(rx) => rt.block_on(async { tokio::time::timeout(Duration::from_secs(10), rx) .await From 414482c8cc21f257d32362e2bb75002987d3829b Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 21:13:02 -0300 Subject: [PATCH 036/208] feat: cleanup deadline from supervisor shutdown_timeout / system grace, unfinished jobs in ShutdownReport, join() waits for cleanup, async JobCtx via pipe_with_ctx --- concurrency/src/reply/jobs.rs | 27 +++++++++++++++++++ concurrency/src/reply/registry.rs | 13 ++++++++++ concurrency/src/supervision.rs | 2 ++ concurrency/src/system.rs | 41 +++++++++++++++++++++++++++++ concurrency/src/tasks/actor.rs | 43 ++++++++++++++++++++++++++----- concurrency/src/tasks/jobs.rs | 37 ++++++++++++++++++++++++++ concurrency/src/threads/actor.rs | 21 +++++++++++++-- 7 files changed, 176 insertions(+), 8 deletions(-) diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index e49dfdba..bbf13100 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -571,3 +571,30 @@ impl Expirable for JobDeadline { } } } + +/// Threads-mode cleanup completion (death plus the bounded job join), for +/// `join()`. Set when the guard drops, so a panicking cleanup also releases +/// joiners. +#[derive(Default)] +pub(crate) struct Cleaned { + done: Mutex, + changed: Condvar, +} + +impl Cleaned { + pub(crate) fn wait(&self) { + let mut done = lock(&self.done); + while !*done { + done = self.changed.wait(done).unwrap_or_else(|p| p.into_inner()); + } + } +} + +pub(crate) struct CleanedGuard(pub(crate) Arc); + +impl Drop for CleanedGuard { + fn drop(&mut self) { + *lock(&self.0.done) = true; + self.0.changed.notify_all(); + } +} diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index bea0469e..d226b20e 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -74,6 +74,8 @@ pub(crate) struct Registry { pool: Arc, phase: AtomicU8, lists: Mutex, + /// When exit began: the cleanup deadline is armed from here. + exit_began: Mutex>, #[cfg(all(spawned_sim, feature = "async"))] sim: deadline::SimService, } @@ -95,6 +97,7 @@ impl Registry { prune_at: 16, ..Lists::default() }), + exit_began: Mutex::new(None), #[cfg(all(spawned_sim, feature = "async"))] sim: deadline::SimService::default(), }) @@ -273,11 +276,21 @@ impl Registry { if previous == DEAD { return; } + self.exit_began + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get_or_insert_with(spawned_rt::time::Instant::now); if self.phase.swap(EXITING, Ordering::AcqRel) == IN_TURN_PENDING { self.commit(Terminal::Stopped); } } + /// What remains of `timeout`, measured from the start of exit. + pub(crate) fn cleanup_remaining(&self, timeout: std::time::Duration) -> std::time::Duration { + let began = *self.exit_began.lock().unwrap_or_else(|p| p.into_inner()); + began.map_or(timeout, |began| timeout.saturating_sub(began.elapsed())) + } + pub(crate) fn is_exiting(&self) -> bool { self.phase.load(Ordering::Acquire) >= EXITING } diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs index 328ed91b..618316e7 100644 --- a/concurrency/src/supervision.rs +++ b/concurrency/src/supervision.rs @@ -2045,6 +2045,8 @@ async fn run_child(spec: ChildSpec, view: Arc>, core: Arc { diff --git a/concurrency/src/system.rs b/concurrency/src/system.rs index 0de9b0f5..84cfb231 100644 --- a/concurrency/src/system.rs +++ b/concurrency/src/system.rs @@ -425,6 +425,7 @@ impl ActorSystem { supervisor_failures: Vec::new(), group_failures: Vec::new(), coordinator_failures: Vec::new(), + unfinished_jobs: Vec::new(), } .wait(deadline) .await @@ -523,6 +524,7 @@ impl SystemInner { } } ShutdownTarget::Scope(scope) => { + scope.set_cleanup_timeout(remaining); scope.stop(ExitReason::Shutdown); let _ = rt::timeout(remaining, scope.joined()).await; } @@ -663,6 +665,9 @@ pub struct ShutdownReport { pub group_failures: Vec, /// Coordinator panics recovered through emergency stop-and-join. pub coordinator_failures: Vec, + /// Dead actors whose job cleanup is incomplete: `(actor, running jobs)` + /// still executing after the cleanup deadline (deferred-replies §5). + pub unfinished_jobs: Vec<(crate::ActorId, usize)>, } #[derive(Debug, Clone)] pub struct ActorExit { @@ -705,6 +710,14 @@ impl ShutdownReport { .filter_map(|s| s.diagnostic()) .collect(); self.unfinished_actors = self.unfinished_actor_details.iter().map(|d| d.id).collect(); + self.unfinished_jobs = state + .shutdown_scopes + .iter() + .filter_map(|s| { + let running = s.unfinished_jobs(); + (running > 0).then(|| s.child().map(|c| (c.id(), running)))? + }) + .collect(); self.exits = state .shutdown_scopes .iter() @@ -853,6 +866,10 @@ impl ExecutionGroup { active_turn: Mutex::new(None), ready, done, + cleanup_timeout: std::sync::atomic::AtomicU64::new( + crate::tasks::DEFAULT_STOP_DEADLINE.as_nanos() as u64, + ), + jobs: Mutex::new(None), }); state.scopes.insert(id, scope.clone()); Ok((scope.clone(), Lease { scope, root: true })) @@ -890,8 +907,32 @@ pub(crate) struct Scope { active_turn: Mutex>, ready: watch::Sender, done: watch::Sender, + /// The actor's cleanup deadline (deferred-replies §5): the supervisor's + /// `shutdown_timeout` or the system's remaining grace, in nanoseconds. + cleanup_timeout: std::sync::atomic::AtomicU64, + /// The actor's job set, for unfinished-job reporting. + jobs: Mutex>>, } impl Scope { + /// Shorten the cleanup deadline that bounds the actor's job join. + pub(crate) fn set_cleanup_timeout(&self, timeout: Duration) { + let nanos = u64::try_from(timeout.as_nanos()).unwrap_or(u64::MAX); + self.cleanup_timeout + .fetch_min(nanos, std::sync::atomic::Ordering::AcqRel); + } + pub(crate) fn cleanup_timeout(&self) -> Duration { + Duration::from_nanos(self.cleanup_timeout.load(std::sync::atomic::Ordering::Acquire)) + } + pub(crate) fn set_jobs(&self, jobs: std::sync::Weak) { + *lock(&self.jobs) = Some(jobs); + } + /// Jobs still running after the actor's death (job cleanup incomplete). + fn unfinished_jobs(&self) -> usize { + lock(&self.jobs) + .as_ref() + .and_then(std::sync::Weak::upgrade) + .map_or(0, |jobs| jobs.running()) + } pub(crate) fn group(&self) -> ExecutionGroup { ExecutionGroup { inner: self.group.clone(), diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 8ee3051b..1cd259f9 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -303,7 +303,10 @@ struct ActorShared { /// This instance's reply registry: deferred tokens, credits, turn phase. replies: Arc, /// This instance's pipe-to-self jobs. - jobs: crate::reply::jobs::JobSet, + jobs: Arc, + /// Becomes `true` when cleanup completes (death plus the bounded job + /// join); closed if the actor task is dropped. `join()` waits for it. + cleaned: watch::Receiver, } impl Clone for Context { @@ -922,8 +925,23 @@ impl ActorRef { } /// Wait until the actor has fully stopped (including `stopped()` callback). + /// + /// Returns when cleanup completes: the actor died and its cancelled jobs + /// were joined (bounded by the cleanup deadline; jobs still running then + /// are reported, not awaited). A `stopped()` or state destructor that + /// never returns keeps it waiting; bound it with a timeout, or use + /// [`wait_exit`](Self::wait_exit) to observe death alone. pub async fn join(&self) { let _ = self.wait_exit().await; + let mut cleaned = self.shared.cleaned.clone(); + loop { + if *cleaned.borrow_and_update() { + return; + } + if cleaned.changed().await.is_err() { + return; + } + } } /// Poll the exit reason. Returns `None` if the actor is still running. @@ -1064,14 +1082,16 @@ impl ActorRef { .unwrap_or_else(new_linked_exit_reason); let id = ActorId::next(); + let (cleaned_tx, cleaned_rx) = watch::channel(false); let actor_ref = ActorRef { shared: Arc::new(ActorShared { id, replies: Registry::new(id, A::MAX_OUTSTANDING), - jobs: crate::reply::jobs::JobSet::new( + jobs: Arc::new(crate::reply::jobs::JobSet::new( id, A::MAX_NATIVE_JOBS.unwrap_or_else(crate::reply::jobs::default_max_native_jobs), - ), + )), + cleaned: cleaned_rx, sender: tx, cancellation_token: cancellation_token.clone(), completion_rx, @@ -1087,6 +1107,7 @@ impl ActorRef { if let Some(scope) = &managed { scope.attach::(actor_ref.child_handle()); + scope.set_jobs(Arc::downgrade(&actor_ref.shared.jobs)); } if let Some(parent) = parent { @@ -1139,6 +1160,7 @@ impl ActorRef { let startup = crate::actor_cleanup::ActorStartup::new((actor, ctx, rx, cancellation_token), cleanup); let job_owner = actor_ref.shared.clone(); + let cleaned_tx = cleaned_tx; let inner_future = async move { let ((actor, ctx, rx, cancellation_token), mut cleanup) = startup.into_parts(); #[cfg(feature = "backend-experiments")] @@ -1172,9 +1194,16 @@ impl ActorRef { // cancelled jobs, bounded by the cleanup deadline; a job still // running then does not undo death and is reported unfinished. if job_owner.jobs.running() > 0 { - let unfinished = job_owner.jobs.join(DEFAULT_STOP_DEADLINE).await; + let timeout = job_owner + .managed + .as_ref() + .map_or(DEFAULT_STOP_DEADLINE, |scope| scope.cleanup_timeout()); + let remaining = job_owner.replies.cleanup_remaining(timeout); + let unfinished = job_owner.jobs.join(remaining).await; crate::reply::jobs::report_unfinished(job_owner.id, unfinished); } + drop(job_owner); + let _ = cleaned_tx.send(true); }; if let Some(lease) = lease { @@ -3344,7 +3373,8 @@ mod turn_regressions { mailbox: crate::instrumentation::Mailbox::default(), managed: None, replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), - jobs: crate::reply::jobs::JobSet::new(ActorId::next(), 1), + jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), + cleaned: watch::channel(false).1, }), }; let count = Arc::new(AtomicUsize::new(0)); @@ -3420,7 +3450,8 @@ mod idle_stop_regression { mailbox: crate::instrumentation::Mailbox::default(), managed: None, replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), - jobs: crate::reply::jobs::JobSet::new(ActorId::next(), 1), + jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), + cleaned: watch::channel(false).1, }), }; let count = Arc::new(AtomicUsize::new(0)); diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index 5300a70a..82130adb 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -460,6 +460,43 @@ impl Context { )) } + /// [`pipe_with`](Self::pipe_with) for async work that needs its + /// [`JobCtx`]: `work` receives it to check cancellation and `emit` fenced + /// messages (bounded by `JobOptions::emit_credits`; the terminal message + /// is always last). + pub fn pipe_with_ctx( + &self, + opts: JobOptions, + work: F, + done: D, + ) -> Result + where + A: Handler, + M: Message, + T: Send + 'static, + F: FnOnce(JobCtx) -> W, + W: Future + Send + 'static, + D: FnOnce(JobOutcome) -> M + Send + 'static, + { + let state = self.admit(&opts).map_err(|kind| PipeError { kind })?; + let credit = JobCredit::reserve(&self.shared.replies).ok_or(PipeError { + kind: PipeErrorKind::Overloaded, + })?; + let job = JobCtx { + state: state.clone(), + ctx: self.clone(), + }; + let work = work(job); + Ok(self.launch_async::( + opts, + state, + Hold::Credit(credit), + work, + Box::new(done), + None, + )) + } + /// Run blocking `work` on an owned native thread, under the actor's /// `MAX_NATIVE_JOBS` budget. `work` gets a [`JobCtx`] to check /// cancellation and `emit_blocking` fenced messages. diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index f3bdcd95..fefdea5f 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -199,6 +199,7 @@ pub struct Context { mailbox: crate::instrumentation::Mailbox, replies: Arc, jobs: Arc, + cleaned: Arc, } impl Clone for Context { @@ -215,6 +216,7 @@ impl Clone for Context { mailbox: self.mailbox.clone(), replies: self.replies.clone(), jobs: self.jobs.clone(), + cleaned: self.cleaned.clone(), } } } @@ -241,6 +243,7 @@ impl Context { mailbox: actor_ref.mailbox.clone(), replies: actor_ref.replies.clone(), jobs: actor_ref.jobs.clone(), + cleaned: actor_ref.cleaned.clone(), } } @@ -368,6 +371,7 @@ impl Context { mailbox: self.mailbox.clone(), replies: self.replies.clone(), jobs: self.jobs.clone(), + cleaned: self.cleaned.clone(), } } @@ -630,6 +634,7 @@ pub struct ActorRef { mailbox: crate::instrumentation::Mailbox, replies: Arc, jobs: Arc, + cleaned: Arc, } impl Debug for ActorRef { @@ -652,6 +657,7 @@ impl Clone for ActorRef { mailbox: self.mailbox.clone(), replies: self.replies.clone(), jobs: self.jobs.clone(), + cleaned: self.cleaned.clone(), } } } @@ -742,8 +748,13 @@ impl ActorRef { } /// Block until the actor has fully stopped (including `stopped()` callback). + /// + /// Returns when cleanup completes: the actor died and its cancelled jobs + /// were joined (bounded by the cleanup deadline). Use + /// [`wait_exit`](Self::wait_exit) to observe death alone. pub fn join(&self) { let _ = self.wait_exit(); + self.cleaned.wait(); } /// Poll the exit reason. Returns `None` if the actor is still running. @@ -850,6 +861,7 @@ impl ActorRef { id, A::MAX_NATIVE_JOBS.unwrap_or_else(crate::reply::jobs::default_max_native_jobs), )), + cleaned: Arc::default(), }; let ctx = Context { @@ -864,6 +876,7 @@ impl ActorRef { mailbox: actor_ref.mailbox.clone(), replies: actor_ref.replies.clone(), jobs: actor_ref.jobs.clone(), + cleaned: actor_ref.cleaned.clone(), }; if let Some(parent) = parent { @@ -885,7 +898,9 @@ impl ActorRef { crate::actor_cleanup::ActorStartup::new((actor, ctx, rx, cancellation_token), cleanup); let replies = actor_ref.replies.clone(); let jobs = actor_ref.jobs.clone(); + let cleaned = crate::reply::jobs::CleanedGuard(actor_ref.cleaned.clone()); let _thread_handle = rt::spawn(move || { + let _cleaned = cleaned; let ((mut actor, ctx, mut rx, cancellation_token), mut cleanup) = startup.into_parts(); let primary = match catch_unwind(AssertUnwindSafe(|| { run_actor(&mut actor, ctx, &mut rx, cancellation_token) @@ -914,8 +929,8 @@ impl ActorRef { // Job cleanup (§5 step 4), after death: join the cancelled jobs // that signal completion, bounded by the cleanup deadline. if jobs.running() > 0 { - let unfinished = - jobs.join_blocking(std::time::Instant::now() + DEFAULT_STOP_DEADLINE); + let remaining = replies.cleanup_remaining(DEFAULT_STOP_DEADLINE); + let unfinished = jobs.join_blocking(std::time::Instant::now() + remaining); crate::reply::jobs::report_unfinished(jobs.owner(), unfinished); } }); @@ -1946,6 +1961,7 @@ mod idle_stop_regression { mailbox: crate::instrumentation::Mailbox::default(), replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), + cleaned: Arc::default(), }; let count = Arc::new(AtomicUsize::new(0)); let seen = count.clone(); @@ -2069,6 +2085,7 @@ mod stop_wakeup_regression { mailbox: crate::instrumentation::Mailbox::default(), replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), + cleaned: Arc::default(), }; let (entered_tx, entered) = mpsc::channel(); let (stopped_tx, stopped) = mpsc::channel(); From 565b24d5444ea64a0c642b6f15328fda453424ad Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 21:18:53 -0300 Subject: [PATCH 037/208] fix(review): credit guard, keyed fencing after completion, destruction-safe close, done panics in the turn, no user code under locks, bounded deadline storage, multiple watchers, shortening no-op after WRITING, DropFailures for fenced payloads, Mock::defer --- concurrency/src/reply/deadline.rs | 50 +++++++-- concurrency/src/reply/jobs.rs | 10 +- concurrency/src/reply/mod.rs | 11 +- concurrency/src/reply/registry.rs | 61 +++++++++-- concurrency/src/reply/slot.rs | 150 +++++++++++++++++---------- concurrency/src/tasks/actor.rs | 15 +++ concurrency/src/tasks/jobs.rs | 23 ++-- concurrency/src/threads/actor.rs | 4 + concurrency/src/threads/jobs.rs | 4 +- concurrency/tests/reply_slot_loom.rs | 21 +++- testing/src/inbox.rs | 24 +++++ testing/src/rules.rs | 10 ++ testing/src/shared.rs | 15 +++ testing/src/tasks.rs | 6 ++ testing/src/threads.rs | 5 + 15 files changed, 312 insertions(+), 97 deletions(-) diff --git a/concurrency/src/reply/deadline.rs b/concurrency/src/reply/deadline.rs index 330b60da..1a78273f 100644 --- a/concurrency/src/reply/deadline.rs +++ b/concurrency/src/reply/deadline.rs @@ -52,10 +52,16 @@ impl Ord for Entry { } impl Entry { - fn is_live(&self) -> bool { - self.item - .upgrade() - .is_some_and(|item| item.pending_deadline() == Some(self.at)) + /// Live: the item exists, is pending, and this is its current deadline. + /// The upgraded item is handed to `upgraded`, to be dropped by the caller + /// outside every lock (it may be the last strong reference). + fn is_live(&self, upgraded: &mut Vec>) -> bool { + let Some(item) = self.item.upgrade() else { + return false; + }; + let live = item.pending_deadline() == Some(self.at); + upgraded.push(item); + live } } @@ -70,10 +76,17 @@ pub(crate) struct DeadlineHeap { } impl DeadlineHeap { - /// Add an entry. Returns whether it is the new earliest deadline. - pub(crate) fn push(&mut self, at: u64, item: Weak) -> bool { + /// Add an entry. Returns whether it is the new earliest deadline, and + /// the items compaction upgraded (drop them outside the lock). + #[must_use] + pub(crate) fn push( + &mut self, + at: u64, + item: Weak, + ) -> (bool, Vec>) { + let mut upgraded = Vec::new(); if self.heap.len() >= 2 * self.live_at_rebuild + 16 { - self.compact(); + self.compact(&mut upgraded); } self.seq += 1; let earliest = self.earliest().is_none_or(|e| at < e); @@ -82,7 +95,7 @@ impl DeadlineHeap { seq: self.seq, item, })); - earliest + (earliest, upgraded) } pub(crate) fn earliest(&self) -> Option { @@ -95,11 +108,11 @@ impl DeadlineHeap { } /// Rebuild without stale entries. - fn compact(&mut self) { + fn compact(&mut self, upgraded: &mut Vec>) { let entries = std::mem::take(&mut self.heap).into_vec(); self.heap = entries .into_iter() - .filter(|Reverse(e)| e.is_live()) + .filter(|Reverse(e)| e.is_live(upgraded)) .collect(); self.live_at_rebuild = self.heap.len(); } @@ -129,6 +142,12 @@ impl DeadlineHeap { // Service // --------------------------------------------------------------------------- +/// The owner died: the service forgets it (its heap is gone). +pub(crate) fn forget(_registry: &Registry) { + #[cfg(not(all(spawned_sim, feature = "async")))] + thread_service::forget(_registry.key()); +} + /// Ask the service to visit `registry` at `at`. pub(crate) fn schedule(registry: &Arc, at: u64) { #[cfg(all(spawned_sim, feature = "async"))] @@ -175,8 +194,19 @@ mod thread_service { }) } + pub(super) fn forget(key: u64) { + let Some(service) = SERVICE_REF.get() else { + return; + }; + let forgotten = lock(service).actors.remove(&key); + drop(forgotten); + } + + static SERVICE_REF: OnceLock<&'static Service> = OnceLock::new(); + pub(super) fn schedule(registry: &Arc, at: u64) { let service = service(); + let _ = SERVICE_REF.set(service); let key = registry.key(); let mut state = lock(service); match state.actors.get(&key) { diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index bbf13100..85c0ccdd 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -448,10 +448,9 @@ impl JobSet { /// A job's execution ended. pub(crate) fn ended(&self, state: &Arc) { state.finished(); - { - let mut keys = lock(&self.keys); - keys.retain(|_, current| !Arc::ptr_eq(current, state)); - } + // The key keeps its finished job until replaced: a replacement must + // still cancel it, so its queued terminal message and emits are + // fenced (`NoStaleKeyed`). The map holds one job per key. let idle = { let mut running = lock(&self.running); *running -= 1; @@ -537,7 +536,8 @@ pub(crate) struct JobCredit(Option>); impl JobCredit { pub(crate) fn reserve(registry: &Registry) -> Option { let pool = registry.pool().clone(); - pool.reserve(1).then_some(Self(Some(pool))) + // `then`, not `then_some`: the guard must exist only when reserved. + pool.reserve(1).then(|| Self(Some(pool))) } } diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 24abe7af..50ec3a22 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -336,9 +336,14 @@ impl ReplyTo { /// Shorten the deadline to `at` (never extends it). pub fn with_deadline(&self, at: Instant) { - self.slot() - .shorten(spawned_rt::time::to_ticks(at), &now_ticks); - self.registry.track_deadline(self.slot()); + // Only a lowered deadline gets a heap entry, so repeated + // `with_deadline` calls with the same or a later instant store nothing. + if self + .slot() + .shorten(spawned_rt::time::to_ticks(at), &now_ticks) + { + self.registry.track_deadline(self.slot()); + } } /// Whether the caller can no longer be answered. Checks the deadline diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index d226b20e..5e4957a6 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -128,13 +128,32 @@ impl Registry { pub(crate) fn register(&self, slot: &Arc>) { let weak: Weak = Arc::downgrade(slot) as Weak; let mut lists = lock(&self.lists); - if lists.tokens.len() >= lists.prune_at { - lists - .tokens - .retain(|t| t.upgrade().is_some_and(|s| !s.is_terminal())); - lists.prune_at = (lists.tokens.len() * 2).max(16); + if lists.tokens.len() < lists.prune_at { + lists.tokens.push(weak); + return; + } + // Prune outside the lock: a temporary upgrade can become a slot's + // last strong reference, and dropping it drops an unread reply (user + // code, which may itself drop a token of this owner). + let listed = std::mem::take(&mut lists.tokens); + drop(lists); + let mut keep = Vec::with_capacity(listed.len()); + let mut upgraded = Vec::new(); + for token in listed { + if let Some(slot) = token.upgrade() { + if !slot.is_terminal() { + keep.push(token); + } + upgraded.push(slot); + } } + let mut lists = lock(&self.lists); + keep.append(&mut lists.tokens); + lists.prune_at = (keep.len() * 2).max(16); + lists.tokens = keep; lists.tokens.push(weak); + drop(lists); + drop(upgraded); } #[cfg_attr(all(spawned_sim, feature = "async"), allow(dead_code))] @@ -164,13 +183,15 @@ impl Registry { /// Add a heap entry; reschedule the service on a new earliest deadline. pub(crate) fn track(self: &Arc, at: u64, item: Weak) { - let earliest = { + let (earliest, upgraded) = { let mut lists = lock(&self.lists); if self.is_dead() { return; } lists.deadlines.push(at, item) }; + // Compaction's temporary upgrades are dropped outside the lock. + drop(upgraded); if earliest { deadline::schedule(self, at); } @@ -307,8 +328,10 @@ impl Registry { .filter_map(|t| t.upgrade()) .filter(|s| s.close()) .count(); - // The heap is dropped after the close (§5 step 3.4). + // The heap is dropped after the close (§5 step 3.4), and the service + // forgets this owner, so dead actors leave nothing behind. lock(&self.lists).deadlines.clear(); + deadline::forget(self); #[cfg(all(spawned_sim, feature = "async"))] self.sim.wake(); closed @@ -409,3 +432,27 @@ mod tests { assert_eq!(registry.outstanding(), 0); } } + +/// Closes the registry and cancels the jobs if the owner's lifecycle is +/// dropped without completing (for example, its runtime was destroyed). +/// Idempotent after a normal death. +pub(crate) struct CloseOnDrop { + replies: Arc, + jobs: Arc, +} + +impl CloseOnDrop { + pub(crate) fn new(replies: Arc, jobs: Arc) -> Self { + Self { replies, jobs } + } +} + +impl Drop for CloseOnDrop { + fn drop(&mut self) { + if !self.replies.is_dead() { + self.replies.exiting(); + self.jobs.cancel_all(); + self.replies.close(); + } + } +} diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 4e83366b..9f3ea52d 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -168,8 +168,9 @@ struct Inner { value: Option, /// The caller's receiver. waker: Option, - /// The token holder waiting in `ReplyTo::closed()`. - watcher: Option, + /// Token-side waiters: `ReplyTo::closed()` and caller-gone job + /// cancellation. Several may wait at once. + watchers: Vec, /// The owner's pool, set by `defer()` before its CAS installs `TOKEN`. pool: Option>, /// Threads parked in `recv_blocking`. @@ -192,7 +193,7 @@ struct Wake { pool: Option>, release: u64, waker: Option, - watcher: Option, + watchers: Vec, notify: bool, } @@ -204,7 +205,7 @@ impl Wake { if let Some(waker) = self.waker { waker.wake(); } - if let Some(watcher) = self.watcher { + for watcher in self.watchers { watcher.wake(); } if self.notify { @@ -222,7 +223,7 @@ impl Slot { inner: Mutex::new(Inner { value: None, waker: None, - watcher: None, + watchers: Vec::new(), pool: None, parked: 0, }), @@ -287,11 +288,16 @@ impl Slot { pool: None, release: 0, waker: None, - watcher: None, + watchers: Vec::new(), notify: false, }; } let mut inner = lock(&self.inner); + Self::collect(&mut inner, release, terminal) + } + + /// The wake-up work of a transition, taken under the slot lock. + fn collect(inner: &mut Inner, release: u64, terminal: bool) -> Wake { Wake { pool: if release > 0 { inner.pool.clone() @@ -300,7 +306,11 @@ impl Slot { }, release, waker: if terminal { inner.waker.take() } else { None }, - watcher: if terminal { inner.watcher.take() } else { None }, + watchers: if terminal { + std::mem::take(&mut inner.watchers) + } else { + Vec::new() + }, notify: terminal && inner.parked > 0, } } @@ -358,13 +368,24 @@ impl Slot { } /// `with_deadline(at)`: shorten only, then the `[deadline]` check. - pub(crate) fn shorten(&self, at: u64, now: &impl Fn() -> u64) { - let _ = self - .deadline - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |d| { - (at < d).then_some(at) - }); + /// Returns whether the deadline was lowered. + /// + /// The reply's `WRITING` CAS is the linearization point: the reply reads + /// the deadline under the slot lock, and a shortening changes it only + /// under that lock and only while the word is `OPEN`. A shortening that + /// finds `WRITING` (or later) is a no-op; one that lowers the deadline is + /// seen by any reply that has not read it yet. + pub(crate) fn shorten(&self, at: u64, now: &impl Fn() -> u64) -> bool { + let lowered = { + let _inner = lock(&self.inner); + let lower = state(self.word()) == OPEN && at < self.deadline(); + if lower { + self.deadline.store(at, Ordering::Release); + } + lower + }; self.check_deadline(now); + lowered } /// `defer()`: `OPEN, TOKEN=0 -> OPEN, TOKEN=1, REG=1`, reserving the @@ -460,31 +481,22 @@ impl Slot { }; return (delivery, Some(value)); } + // The deadline is read under the lock that shortening takes, so a + // shortening either precedes this read or finds `WRITING` and does + // nothing. + let mut inner = lock(&self.inner); let deadline = self.deadline(); - if deadline != NEVER && now() >= deadline { - let (old, new) = self.resolve_writing(TIMED_OUT); - self.after(old, new).run(&self.parked); - return (Delivery::Expired, Some(value)); - } - let wake = { - let mut inner = lock(&self.inner); + let (delivery, unsent, to) = if deadline != NEVER && now() >= deadline { + (Delivery::Expired, Some(value), TIMED_OUT) + } else { inner.value = Some(value); - let (old, new) = self.resolve_writing(REPLIED); - let release = credits(old).saturating_sub(credits(new)); - Wake { - pool: if release > 0 { - inner.pool.clone() - } else { - None - }, - release, - waker: inner.waker.take(), - watcher: inner.watcher.take(), - notify: inner.parked > 0, - } + (Delivery::Delivered, None, REPLIED) }; + let (old, new) = self.resolve_writing(to); + let wake = Self::collect(&mut inner, credits(old).saturating_sub(credits(new)), true); + drop(inner); wake.run(&self.parked); - (Delivery::Delivered, None) + (delivery, unsent) } /// Receiver side: take the outcome of a terminal word. `Replied` with no @@ -506,24 +518,30 @@ impl Slot { /// reads no clock. #[cfg_attr(not(feature = "async"), allow(dead_code))] pub(crate) fn poll(&self, waker: &Waker) -> Poll> { + // `Waker::clone` and dropping a waker run executor code, so neither + // happens under the slot lock. + let mut fresh: Option = None; loop { if let Some(result) = self.take(state(self.word())) { return Poll::Ready(result); } let mut inner = lock(&self.inner); - let w = self.word(); - if !live(state(w)) { - drop(inner); + if !live(state(self.word())) { continue; } - match &inner.waker { - Some(old) if old.will_wake(waker) => {} - _ => { - let old = inner.waker.replace(waker.clone()); - drop(inner); - drop(old); - } + if inner.waker.as_ref().is_some_and(|w| w.will_wake(waker)) { + drop(inner); + drop(fresh); + return Poll::Pending; } + let Some(new) = fresh.take() else { + drop(inner); + fresh = Some(waker.clone()); + continue; + }; + let old = inner.waker.replace(new); + drop(inner); + drop(old); return Poll::Pending; } } @@ -532,20 +550,40 @@ impl Slot { /// which expires it). Registers `waker` as the watcher otherwise. #[cfg_attr(not(feature = "async"), allow(dead_code))] pub(crate) fn poll_closed(&self, waker: &Waker, now: &impl Fn() -> u64) -> Poll<()> { - if !live(state(self.word())) || self.check_deadline(now) { - return Poll::Ready(()); - } - let mut inner = lock(&self.inner); - if !live(state(self.word())) { - return Poll::Ready(()); + let mut fresh: Option = None; + loop { + if !live(state(self.word())) || self.check_deadline(now) { + return Poll::Ready(()); + } + let mut inner = lock(&self.inner); + if !live(state(self.word())) { + continue; + } + if inner.watchers.iter().any(|w| w.will_wake(waker)) { + drop(inner); + drop(fresh); + return Poll::Pending; + } + let Some(new) = fresh.take() else { + drop(inner); + fresh = Some(waker.clone()); + continue; + }; + inner.watchers.push(new); + return Poll::Pending; } - let old = match &inner.watcher { - Some(old) if old.will_wake(waker) => None, - _ => inner.watcher.replace(waker.clone()), + } + + /// Stop watching with `waker`, so finished watchers do not accumulate on + /// a long-lived token. + #[cfg_attr(not(feature = "async"), allow(dead_code))] + pub(crate) fn unwatch(&self, waker: &Waker) { + let removed = { + let mut inner = lock(&self.inner); + let index = inner.watchers.iter().position(|w| w.will_wake(waker)); + index.map(|i| inner.watchers.swap_remove(i)) }; - drop(inner); - drop(old); - Poll::Pending + drop(removed); } /// Non-blocking check from the caller's receiver. diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 1cd259f9..56fdc35b 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -1161,7 +1161,15 @@ impl ActorRef { crate::actor_cleanup::ActorStartup::new((actor, ctx, rx, cancellation_token), cleanup); let job_owner = actor_ref.shared.clone(); let cleaned_tx = cleaned_tx; + // Destruction-safe close: if the runtime drops this future before it + // completes, the guard still cancels the jobs and closes the registry, + // so no deferred caller waits for an instance that can never answer. + let close_guard = crate::reply::registry::CloseOnDrop::new( + actor_ref.shared.replies.clone(), + actor_ref.shared.jobs.clone(), + ); let inner_future = async move { + let _close_guard = close_guard; let ((actor, ctx, rx, cancellation_token), mut cleanup) = startup.into_parts(); #[cfg(feature = "backend-experiments")] let _borrowed_runtime = borrowed_runtime; @@ -1190,6 +1198,8 @@ impl ActorRef { let actual = crate::system::lock(&scope.cause).clone().unwrap_or(reason); scope.exited(actual); } + // Publish the exit (monitors and links fire) before the job join. + drop(cleanup); // Job cleanup (deferred-replies §5 step 4), after death: join the // cancelled jobs, bounded by the cleanup deadline; a job still // running then does not undo death and is reported unfinished. @@ -1345,6 +1355,7 @@ async fn run_actor( #[cfg(feature = "backend-experiments")] turn: Option, ) -> Result { let replies = ctx.shared.replies.clone(); + let jobs = ctx.shared.jobs.clone(); let primary = match AssertUnwindSafe(run_actor_lifecycle( &mut actor, ctx, @@ -1368,6 +1379,10 @@ async fn run_actor( // Death (deferred-replies §5 step 3), only now that `stopped()` returned: // close the registry (every open token answers ActorStopped), dispose // the mailbox (queued requests answer ActorStopped), drop the state. + // Early returns (startup panic, closed scope) and an unwound lifecycle + // also begin exit here, so jobs never outlive their owner. + replies.exiting(); + jobs.cancel_all(); replies.close(); drop(rx); crate::actor_cleanup::finish_actor_state(actor, primary) diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index 82130adb..792b84f6 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -58,7 +58,7 @@ where // The credit returns at dispatch (or here, when fenced). drop(credit); if fenced { - drop(msg); + ctx.shared.sender.failures().discard(msg); return Box::pin(async {}); } Box::pin(actor.handle(msg, PendingReply::detached(), ctx)) @@ -91,14 +91,19 @@ where ctx: &'a Context, ) -> Pin + Send + 'a>> { let TerminalEnvelope { job, done, outcome } = *self; - if job.is_cancelled() || job.owner() != ctx.id() { - // Fenced: `done` and its captures (a linked token) drop in the - // turn, by the drop rules. - drop((done, outcome)); - return Box::pin(async {}); - } - let msg = done(outcome); - Box::pin(actor.handle(msg, PendingReply::detached(), ctx)) + // Everything runs inside the returned future, so a panicking `done` + // is a handler panic caught by the run loop (`stopped()`, turn end + // and job cancellation still run). + Box::pin(async move { + if job.is_cancelled() || job.owner() != ctx.id() { + // Fenced: `done` and its captures (a linked token) are + // discarded in the turn, through the mailbox drop path. + ctx.shared.sender.failures().discard((done, outcome)); + return; + } + let msg = done(outcome); + actor.handle(msg, PendingReply::detached(), ctx).await + }) } } diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index fefdea5f..39597196 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -919,6 +919,8 @@ impl ActorRef { // ActorStopped), dispose the mailbox (queued requests answer // ActorStopped; later sends from state destructors fail), then drop // the state, all before the exit is published. + replies.exiting(); + jobs.cancel_all(); replies.close(); drop(rx); let outcome = crate::actor_cleanup::finish_actor_state(actor, primary); @@ -926,6 +928,8 @@ impl ActorRef { Ok(reason) => cleanup.complete(reason), Err(reason) => cleanup.complete_secondary(reason), } + // Publish the exit (monitors and links fire) before the job join. + drop(cleanup); // Job cleanup (§5 step 4), after death: join the cancelled jobs // that signal completion, bounded by the cleanup deadline. if jobs.running() > 0 { diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index 425ba807..df284414 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -49,7 +49,7 @@ where // The credit returns at dispatch (or here, when fenced). drop(credit); if fenced { - drop(msg); + ctx.sender.failures().discard(msg); return; } actor.handle(msg, PendingReply::detached(), ctx); @@ -77,7 +77,7 @@ where if job.is_cancelled() || job.owner() != ctx.id() { // Fenced: `done` and its captures (a linked token) drop in the // turn, by the drop rules. - drop((done, outcome)); + ctx.sender.failures().discard((done, outcome)); return; } let msg = done(outcome); diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index 50454dfa..fa0b75c1 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -126,12 +126,21 @@ fn reply_vs_shortening() { let (slot, clock) = (slot.clone(), clock.clone()); loom::thread::spawn(move || slot.reply(1, &clock.now())) }; - slot.shorten(3, &clock.now()); + let lowered = slot.shorten(3, &clock.now()); let (delivery, _) = replier.join().unwrap(); - assert_eq!(slot.deadline(), 3); + // Linearization at the reply's WRITING CAS: a shortening after it is + // a no-op (the deadline stays), one before it is seen by the reply. match delivery { - Delivery::Delivered => assert_eq!(state(slot.word()), slot::REPLIED), - Delivery::Expired => assert_eq!(state(slot.word()), slot::TIMED_OUT), + Delivery::Delivered => { + assert_eq!(state(slot.word()), slot::REPLIED); + assert!(!lowered, "a shortening was applied but not seen by the reply"); + assert_eq!(slot.deadline(), 100); + } + Delivery::Expired => { + assert_eq!(state(slot.word()), slot::TIMED_OUT); + assert!(lowered); + assert_eq!(slot.deadline(), 3); + } other => panic!("unexpected {other:?}"), } assert_balanced(&pool, &[&slot]); @@ -416,7 +425,9 @@ fn abandon_vs_shorten_vs_expiry() { let slot = deferred(&pool, 100); let shortener = { let (slot, clock) = (slot.clone(), clock.clone()); - loom::thread::spawn(move || slot.shorten(5, &clock.now())) + loom::thread::spawn(move || { + slot.shorten(5, &clock.now()); + }) }; let abandoned = slot.abandon(); shortener.join().unwrap(); diff --git a/testing/src/inbox.rs b/testing/src/inbox.rs index dcc33829..007233cd 100644 --- a/testing/src/inbox.rs +++ b/testing/src/inbox.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; +use spawned_concurrency::message::Message; //! Shared message log behind mocks and probes: blocking and async waiters. use std::any::{Any, TypeId}; use std::panic::Location; @@ -114,6 +116,8 @@ impl Drop for Pending { #[derive(Default)] pub(crate) struct Inbox { + /// Deferred reply tokens by message type (`Mock::defer`). + deferred: Mutex>>>, entries: Mutex>, arrived: Condvar, #[cfg(feature = "async")] @@ -130,6 +134,26 @@ impl Default for AsyncSignal { } impl Inbox { + pub(crate) fn store_deferred(&self, token: spawned_concurrency::reply::ReplyTo) { + self.deferred + .lock() + .unwrap_or_else(|p| p.into_inner()) + .entry(std::any::TypeId::of::()) + .or_default() + .push(Box::new(token)); + } + pub(crate) fn take_deferred(&self) -> Vec> { + let tokens = self + .deferred + .lock() + .unwrap_or_else(|p| p.into_inner()) + .remove(&std::any::TypeId::of::()) + .unwrap_or_default(); + tokens + .into_iter() + .filter_map(|t| t.downcast().ok().map(|t| *t)) + .collect() + } fn lock(&self) -> MutexGuard<'_, Vec> { self.entries.lock().unwrap_or_else(|p| p.into_inner()) } diff --git a/testing/src/rules.rs b/testing/src/rules.rs index d7d46a84..550a6dd6 100644 --- a/testing/src/rules.rs +++ b/testing/src/rules.rs @@ -42,12 +42,15 @@ pub(crate) enum Plan { record: Option, }, Crash(String), + /// `Mock::defer`: take the reply obligation and store it for the test. + Defer, } #[derive(Default)] pub(crate) struct Rules { by_type: HashMap>, allow_sends: bool, + deferred: std::collections::HashSet, } /// `Some(())` as `M::Result` when that type is `()`: lets one generic handler @@ -81,6 +84,9 @@ impl Rules { pub(crate) fn crash_on(&mut self, reason: String) { self.rule::().crash = Some(reason); } + pub(crate) fn defer(&mut self) { + self.deferred.insert(TypeId::of::()); + } pub(crate) fn allow_sends(&mut self) { self.allow_sends = true; } @@ -101,6 +107,10 @@ impl Rules { record: None, }; } + if self.deferred.contains(&TypeId::of::()) { + inbox.push(msg, Outcome::Handled); + return Plan::Defer; + } let name = std::any::type_name::(); let allow_sends = self.allow_sends; let rule = self diff --git a/testing/src/shared.rs b/testing/src/shared.rs index 4572e510..e91ef62e 100644 --- a/testing/src/shared.rs +++ b/testing/src/shared.rs @@ -8,6 +8,14 @@ /// Builder methods of `Mock`, except `delay` and `start`. macro_rules! mock_builder { () => { + /// Defer every `M` request: the mock takes its reply obligation + /// (`PendingReply::defer`) and stores the token; take it with + /// `MockRef::deferred` to reply later, drop it, or never reply. + pub fn defer(mut self) -> Self { + self.rules.defer::(); + self + } + /// A strict mock: every message type needs a rule. /// /// The receive log keeps one entry per message for the mock's @@ -117,6 +125,13 @@ macro_rules! mock_ref_common { self.inbox.count::() } + /// Take the reply tokens of the `M` requests the mock deferred + /// (`Mock::defer`), oldest first: reply later, drop them + /// (`ReplyDropped`), or keep them unanswered. + pub fn deferred(&self) -> Vec> { + self.inbox.take_deferred::() + } + /// Take every not-yet-expected `M`, oldest first. pub fn take_all(&self) -> Vec { self.inbox.take_all::() diff --git a/testing/src/tasks.rs b/testing/src/tasks.rs index c57a6d63..ab383e5d 100644 --- a/testing/src/tasks.rs +++ b/testing/src/tasks.rs @@ -101,6 +101,7 @@ impl Handler for Mock { _: &Context, ) -> impl Future + Send { let plan = self.rules.plan(&self.inbox, msg); + let inbox = self.inbox.clone(); async move { match plan { Plan::Reply { @@ -116,6 +117,11 @@ impl Handler for Mock { } reply.reply(value); } + Plan::Defer => { + if let Ok(token) = reply.defer() { + inbox.store_deferred::(token); + } + } Plan::Crash(reason) => panic!("{reason}"), } } diff --git a/testing/src/threads.rs b/testing/src/threads.rs index 45189e13..4a46d4ba 100644 --- a/testing/src/threads.rs +++ b/testing/src/threads.rs @@ -67,6 +67,11 @@ impl Handler for Mock { } reply.reply(value); } + Plan::Defer => { + if let Ok(token) = reply.defer() { + self.inbox.store_deferred::(token); + } + } Plan::Crash(reason) => panic!("{reason}"), } } From b1d9b98d2893cfe5b2846f74bc7723db2bfee6e8 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 21:19:11 -0300 Subject: [PATCH 038/208] fix: caller-give-up cancellation for blocking linked jobs; inbox imports --- concurrency/src/reply/jobs.rs | 16 ++++++++++++++++ concurrency/src/reply/slot.rs | 12 ------------ concurrency/src/tasks/jobs.rs | 3 +++ concurrency/src/threads/jobs.rs | 3 +++ testing/src/inbox.rs | 4 ++-- 5 files changed, 24 insertions(+), 14 deletions(-) diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 85c0ccdd..75684337 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -598,3 +598,19 @@ impl Drop for CleanedGuard { self.0.changed.notify_all(); } } + +/// Opt-in caller-give-up cancellation for jobs that cannot await +/// (`pipe_reply_blocking*` with `cancel_on_caller_gone`): a watcher waker on +/// the linked slot cancels the job when the slot closes. +pub(crate) fn cancel_when_closed(slot: &super::slot::Slot, state: &Arc) { + struct CancelJob(Arc); + impl std::task::Wake for CancelJob { + fn wake(self: Arc) { + self.0.cancel(); + } + } + let waker = std::task::Waker::from(Arc::new(CancelJob(state.clone()))); + if slot.poll_closed(&waker, &super::now_ticks).is_ready() { + state.cancel(); + } +} diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 9f3ea52d..bb99f455 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -574,18 +574,6 @@ impl Slot { } } - /// Stop watching with `waker`, so finished watchers do not accumulate on - /// a long-lived token. - #[cfg_attr(not(feature = "async"), allow(dead_code))] - pub(crate) fn unwatch(&self, waker: &Waker) { - let removed = { - let mut inner = lock(&self.inner); - let index = inner.watchers.iter().position(|w| w.will_wake(waker)); - index.map(|i| inner.watchers.swap_remove(i)) - }; - drop(removed); - } - /// Non-blocking check from the caller's receiver. pub(crate) fn try_take(&self, now: &impl Fn() -> u64) -> Option> { if let Some(result) = self.take(state(self.word())) { diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index 792b84f6..d3873e5f 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -588,6 +588,9 @@ impl Context { D: FnOnce(JobOutcome, ReplyTo) -> M + Send + 'static, { let (state, slot, token) = self.link_reply(&opts, token)?; + if opts.cancel_on_caller_gone { + crate::reply::jobs::cancel_when_closed(&slot, &state); + } let hold = Hold::Linked(LinkedJob(slot)); let cell = Arc::new(Mutex::new(Some(token))); let shared = cell.clone(); diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index df284414..82be81d1 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -360,6 +360,9 @@ impl Context { D: FnOnce(JobOutcome, ReplyTo) -> M + Send + 'static, { let (state, slot, token) = self.link_reply(&opts, token)?; + if opts.cancel_on_caller_gone { + crate::reply::jobs::cancel_when_closed(&slot, &state); + } let hold = Hold::Linked(LinkedJob(slot)); let cell = Arc::new(Mutex::new(Some(token))); let shared = cell.clone(); diff --git a/testing/src/inbox.rs b/testing/src/inbox.rs index 007233cd..db03a772 100644 --- a/testing/src/inbox.rs +++ b/testing/src/inbox.rs @@ -1,6 +1,6 @@ -use std::collections::HashMap; -use spawned_concurrency::message::Message; //! Shared message log behind mocks and probes: blocking and async waiters. +use spawned_concurrency::message::Message; +use std::collections::HashMap; use std::any::{Any, TypeId}; use std::panic::Location; use std::sync::{Arc, Condvar, Mutex, MutexGuard}; From 0c0d116fc7503d3d6a2077e128d8e39155e83731 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 21:20:22 -0300 Subject: [PATCH 039/208] test(review): regression tests for credit guard, finished keyed fencing, runtime destruction close, panicking done; exit before state drop on lifecycle destruction --- concurrency/src/reply/jobs.rs | 32 ++++++++++++++ concurrency/src/tasks/actor.rs | 27 +++++++++++- concurrency/tests/deferred_jobs.rs | 61 +++++++++++++++++++++++++++ concurrency/tests/deferred_replies.rs | 28 ++++++++++++ 4 files changed, 146 insertions(+), 2 deletions(-) diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 75684337..db6693a9 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -614,3 +614,35 @@ pub(crate) fn cancel_when_closed(slot: &super::slot::Slot, state: &Arc { + actor: Option, + replies: Arc, + jobs: Arc, +} +impl Drop for ExitOnDrop { + fn drop(&mut self) { + if self.actor.is_some() { + self.replies.exiting(); + self.jobs.cancel_all(); + } + } +} + async fn run_actor( - mut actor: A, + actor: A, ctx: Context, mut rx: mpsc::Receiver + Send>>, cancellation_token: CancellationToken, @@ -1356,8 +1373,13 @@ async fn run_actor( ) -> Result { let replies = ctx.shared.replies.clone(); let jobs = ctx.shared.jobs.clone(); + let mut owned = ExitOnDrop { + actor: Some(actor), + replies: replies.clone(), + jobs: jobs.clone(), + }; let primary = match AssertUnwindSafe(run_actor_lifecycle( - &mut actor, + owned.actor.as_mut().expect("the state lives until death"), ctx, &mut rx, cancellation_token, @@ -1385,6 +1407,7 @@ async fn run_actor( jobs.cancel_all(); replies.close(); drop(rx); + let actor = owned.actor.take().expect("the state lives until death"); crate::actor_cleanup::finish_actor_state(actor, primary) } diff --git a/concurrency/tests/deferred_jobs.rs b/concurrency/tests/deferred_jobs.rs index f4abfb84..1ecf3351 100644 --- a/concurrency/tests/deferred_jobs.rs +++ b/concurrency/tests/deferred_jobs.rs @@ -34,6 +34,9 @@ mod tasks_mode { impl Actor for Owner { const MAX_OUTSTANDING: usize = 4; const MAX_NATIVE_JOBS: Option = Some(1); + async fn stopped(&mut self, _: &Context) { + self.log.push("stopped"); + } } struct Done(String); @@ -254,6 +257,64 @@ mod tasks_mode { stop(actor).await; } + struct ReplaceAfterFinish; + impl Message for ReplaceAfterFinish { + type Result = (); + } + impl Handler for Owner { + async fn handle(&mut self, _: ReplaceAfterFinish, reply: PendingReply<'_, ()>, ctx: &Context) { + // The old job finishes (its terminal message queued behind this + // turn) before the replacement takes the key. + let old = ctx + .pipe_keyed("r", async { 1 }, |o| describe("old", o)) + .unwrap(); + while old.is_running() { + std::thread::sleep(Duration::from_millis(1)); + } + ctx.pipe_keyed("r", async { 2 }, |o| describe("new", o)) + .unwrap(); + reply.reply(()); + } + } + + /// Review P1 (`NoStaleKeyed`): a finished keyed job's queued terminal + /// message is fenced by its replacement. + #[cfg(not(spawned_sim))] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn replacement_fences_a_finished_keyed_job() { + let (actor, log) = start(); + actor.request(ReplaceAfterFinish).await.unwrap(); + assert_eq!(entries(&actor, &log, 1).await, ["new:ok:2"]); + sleep(Duration::from_millis(20)).await; + assert!(log.take().is_empty()); + stop(actor).await; + } + + struct Bomb; + impl Message for Bomb { + type Result = (); + } + impl Handler for Owner { + async fn handle(&mut self, _: Bomb, reply: PendingReply<'_, ()>, ctx: &Context) { + ctx.pipe(async {}, |_: JobOutcome<()>| -> Done { panic!("done boom") }) + .unwrap(); + reply.reply(()); + } + } + + /// Review P1: a panicking `done` is a handler panic: `stopped()` runs + /// and the owner's other jobs are cancelled. + #[spawned_macros::sim_test] + async fn a_panicking_done_runs_stopped_and_cancels_jobs() { + let (actor, log) = start(); + actor.request(Start::Gated("survivor", None)).await.unwrap().unwrap(); + actor.request(Bomb).await.unwrap(); + let reason = actor.wait_exit().await; + assert!(matches!(reason, spawned_concurrency::ExitReason::Panic(_)), "{reason:?}"); + assert!(log.take().contains(&"stopped".to_string()), "stopped() did not run"); + actor.join().await; + } + #[spawned_macros::sim_test] async fn job_deadline_notifies_while_the_work_keeps_its_credit() { let (actor, log) = start(); diff --git a/concurrency/tests/deferred_replies.rs b/concurrency/tests/deferred_replies.rs index f68eec6c..caa27509 100644 --- a/concurrency/tests/deferred_replies.rs +++ b/concurrency/tests/deferred_replies.rs @@ -302,6 +302,34 @@ mod tasks_mode { actor.join().await; } + /// Review P1: destroying the runtime under a live actor still closes its + /// registry; a `Deadline::Never` caller is answered `ActorStopped`. + #[cfg(not(spawned_sim))] + #[test] + fn runtime_destruction_closes_the_registry() { + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .unwrap(); + let (actor, rx) = rt.block_on(async { + let (actor, _) = start(); + let rx = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); + assert_eq!(settle(&actor).await, 1); + (actor, rx) + }); + drop(rt); + let caller = std::thread::spawn(move || rx.recv()); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !caller.is_finished() { + assert!(std::time::Instant::now() < deadline, "the caller hangs"); + std::thread::sleep(Duration::from_millis(5)); + } + let result = caller.join().unwrap(); + assert!(matches!(result, Err(ActorError::ActorStopped)), "{result:?}"); + drop(actor); + } + #[spawned_macros::sim_test] async fn stopped_may_answer_stored_tokens() { let (actor, _) = start(); From 2657a41cec816476a9309db08e11804cedf77150 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 21:21:35 -0300 Subject: [PATCH 040/208] test: keyed-fencing regression without blocking the job's worker --- concurrency/tests/deferred_jobs.rs | 49 +++++++++++++++++++----------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/concurrency/tests/deferred_jobs.rs b/concurrency/tests/deferred_jobs.rs index 1ecf3351..a6b575d7 100644 --- a/concurrency/tests/deferred_jobs.rs +++ b/concurrency/tests/deferred_jobs.rs @@ -257,36 +257,51 @@ mod tasks_mode { stop(actor).await; } - struct ReplaceAfterFinish; - impl Message for ReplaceAfterFinish { + /// Starts keyed job `old`, publishing its handle. + struct StartOld(Arc>>); + impl Message for StartOld { type Result = (); } - impl Handler for Owner { - async fn handle(&mut self, _: ReplaceAfterFinish, reply: PendingReply<'_, ()>, ctx: &Context) { - // The old job finishes (its terminal message queued behind this - // turn) before the replacement takes the key. - let old = ctx - .pipe_keyed("r", async { 1 }, |o| describe("old", o)) - .unwrap(); - while old.is_running() { + impl Handler for Owner { + async fn handle(&mut self, msg: StartOld, reply: PendingReply<'_, ()>, ctx: &Context) { + let job = ctx.pipe_keyed("r", async { 1 }, |o| describe("old", o)).unwrap(); + *msg.0.lock().unwrap() = Some(job); + reply.reply(()); + } + } + /// Busy until `go`, then replaces the key. + struct ReplaceWhen(Arc); + impl Message for ReplaceWhen { + type Result = (); + } + impl Handler for Owner { + async fn handle(&mut self, msg: ReplaceWhen, _: PendingReply<'_, ()>, ctx: &Context) { + while !msg.0.load(std::sync::atomic::Ordering::SeqCst) { std::thread::sleep(Duration::from_millis(1)); } - ctx.pipe_keyed("r", async { 2 }, |o| describe("new", o)) - .unwrap(); - reply.reply(()); + ctx.pipe_keyed("r", async { 2 }, |o| describe("new", o)).unwrap(); } } - /// Review P1 (`NoStaleKeyed`): a finished keyed job's queued terminal - /// message is fenced by its replacement. + /// Review P1 (`NoStaleKeyed`): the old keyed job finishes and queues its + /// terminal message while the owner is busy; the replacement made in that + /// busy turn fences it. #[cfg(not(spawned_sim))] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn replacement_fences_a_finished_keyed_job() { let (actor, log) = start(); - actor.request(ReplaceAfterFinish).await.unwrap(); + let handle = Arc::new(Mutex::new(None)); + actor.request(StartOld(handle.clone())).await.unwrap(); + let go = Arc::new(std::sync::atomic::AtomicBool::new(false)); + actor.send(ReplaceWhen(go.clone())).unwrap(); + let old = handle.lock().unwrap().take().unwrap(); + while old.is_running() { + sleep(Duration::from_millis(1)).await; + } + go.store(true, std::sync::atomic::Ordering::SeqCst); assert_eq!(entries(&actor, &log, 1).await, ["new:ok:2"]); sleep(Duration::from_millis(20)).await; - assert!(log.take().is_empty()); + assert!(log.take().is_empty(), "the replaced job's terminal message was dispatched"); stop(actor).await; } From 7945679f4e23aa33aa9c3a09c81c1aa1f5ef30d3 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 21:24:53 -0300 Subject: [PATCH 041/208] test: gate the old keyed job so its terminal message queues behind the replacing turn --- concurrency/tests/deferred_jobs.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/concurrency/tests/deferred_jobs.rs b/concurrency/tests/deferred_jobs.rs index a6b575d7..7c4646f0 100644 --- a/concurrency/tests/deferred_jobs.rs +++ b/concurrency/tests/deferred_jobs.rs @@ -258,14 +258,20 @@ mod tasks_mode { } /// Starts keyed job `old`, publishing its handle. - struct StartOld(Arc>>); + type OldJob = Arc)>>>; + struct StartOld(OldJob); impl Message for StartOld { type Result = (); } impl Handler for Owner { async fn handle(&mut self, msg: StartOld, reply: PendingReply<'_, ()>, ctx: &Context) { - let job = ctx.pipe_keyed("r", async { 1 }, |o| describe("old", o)).unwrap(); - *msg.0.lock().unwrap() = Some(job); + let (open, gate) = oneshot::channel::<()>(); + let job = ctx + .pipe_keyed("r", async move { gate.await.map(|()| 1).unwrap_or(0) }, |o| { + describe("old", o) + }) + .unwrap(); + *msg.0.lock().unwrap() = Some((job, open)); reply.reply(()); } } @@ -294,7 +300,9 @@ mod tasks_mode { actor.request(StartOld(handle.clone())).await.unwrap(); let go = Arc::new(std::sync::atomic::AtomicBool::new(false)); actor.send(ReplaceWhen(go.clone())).unwrap(); - let old = handle.lock().unwrap().take().unwrap(); + // `ReplaceWhen` is queued before the old job can finish. + let (old, open) = handle.lock().unwrap().take().unwrap(); + open.send(()).unwrap(); while old.is_running() { sleep(Duration::from_millis(1)).await; } From 8f80adb27cef3bb2fe264b41879f7d67186cbc6e Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 21:35:47 -0300 Subject: [PATCH 042/208] fix(review agy): cas re-evaluates a changed word before reporting the bound, credits released before the terminal message, sim service starts at no deadline, closed() follows shortening, Mock::discard drops the obligation; regression tests --- concurrency/src/reply/deadline.rs | 14 +++++++++-- concurrency/src/reply/mod.rs | 35 +++++++++++++++++++++------ concurrency/src/reply/slot.rs | 22 ++++++++++++++--- concurrency/src/tasks/jobs.rs | 10 +++++++- concurrency/src/threads/jobs.rs | 2 ++ concurrency/tests/deferred_replies.rs | 20 +++++++++++++++ concurrency/tests/reply_slot_loom.rs | 26 ++++++++++++++++++++ testing/src/rules.rs | 14 +++++------ testing/src/shared.rs | 10 ++++---- testing/src/tasks.rs | 6 +++++ testing/src/threads.rs | 6 +++++ 11 files changed, 138 insertions(+), 27 deletions(-) diff --git a/concurrency/src/reply/deadline.rs b/concurrency/src/reply/deadline.rs index 1a78273f..eccd84e9 100644 --- a/concurrency/src/reply/deadline.rs +++ b/concurrency/src/reply/deadline.rs @@ -275,18 +275,28 @@ mod sim { use std::time::Duration; /// Per-heap service task state, kept by the registry. - #[derive(Default)] pub(crate) struct SimService { started: AtomicBool, earliest: AtomicU64, waker: AtomicWaker, } + impl Default for SimService { + fn default() -> Self { + Self { + started: AtomicBool::new(false), + // Nothing scheduled yet: a first schedule only lowers it, so a + // concurrent earlier deadline is never overwritten. + earliest: AtomicU64::new(u64::MAX), + waker: AtomicWaker::new(), + } + } + } + pub(super) fn schedule(registry: &Arc, at: u64) { let service = registry.sim_service(); service.earliest.fetch_min(at, Ordering::AcqRel); if !service.started.swap(true, Ordering::AcqRel) { - service.earliest.store(at, Ordering::Release); let weak = Arc::downgrade(registry); spawned_rt::tasks::spawn(run(weak)); } diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 50ec3a22..116886c7 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -357,14 +357,33 @@ impl ReplyTo { #[cfg(feature = "async")] pub async fn closed(&self) { let slot = self.slot(); - let waiting = std::future::poll_fn(|cx| slot.poll_closed(cx.waker(), &now_ticks)); - let deadline = slot.deadline(); - if deadline == NEVER { - return waiting.await; - } - let wait = Duration::from_nanos(deadline.saturating_sub(now_ticks())); - if spawned_rt::tasks::timeout(wait, waiting).await.is_err() { - slot.terminate(Terminal::TimedOut); + loop { + // One wait per deadline value: a shortening wakes the watchers, + // and the loop re-arms for the earlier deadline. + let deadline = slot.deadline(); + let mut woken = false; + let waiting = std::future::poll_fn(|cx| { + if woken && slot.deadline() != deadline { + return std::task::Poll::Ready(false); + } + woken = true; + slot.poll_closed(cx.waker(), &now_ticks).map(|()| true) + }); + let closed = if deadline == NEVER { + waiting.await + } else { + let wait = Duration::from_nanos(deadline.saturating_sub(now_ticks())); + match spawned_rt::tasks::timeout(wait, waiting).await { + Ok(closed) => closed, + Err(_) => { + slot.check_deadline(&now_ticks); + slot.is_terminal() + } + } + }; + if closed { + return; + } } } diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index bb99f455..00cd0cca 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -259,7 +259,18 @@ impl Slot { if up > 0 { match pool { Some(pool) if pool.reserve(up) => {} - _ => return Err(None), + _ => { + // Report the bound only for the word it was computed + // from: a word that changed meanwhile (closed, + // abandoned) is re-evaluated, so a closed slot reports + // `Closed`, not `Overloaded`. + let actual = self.word.load(Ordering::Acquire); + if actual != current { + current = actual; + continue; + } + return Err(None); + } } } match self @@ -377,13 +388,18 @@ impl Slot { /// seen by any reply that has not read it yet. pub(crate) fn shorten(&self, at: u64, now: &impl Fn() -> u64) -> bool { let lowered = { - let _inner = lock(&self.inner); + let mut _inner = lock(&self.inner); let lower = state(self.word()) == OPEN && at < self.deadline(); if lower { self.deadline.store(at, Ordering::Release); } - lower + (lower, if lower { std::mem::take(&mut _inner.watchers) } else { Vec::new() }) }; + let (lowered, watchers) = lowered; + // Token-side waiters re-arm their timers for the earlier deadline. + for watcher in watchers { + watcher.wake(); + } self.check_deadline(now); lowered } diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index d3873e5f..e3a47705 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -330,7 +330,13 @@ impl Context { let work = pin!(AssertUnwindSafe(work).catch_unwind()); let stop = future::select(cancelled, gone); match future::select(work, stop).await { - future::Either::Left((result, _)) => run.notify(caught(result)), + future::Either::Left((result, _)) => { + // Execution ended: return its credit (and linked-job + // count) before the terminal message can be dispatched, + // so `done`'s turn can start the next job at the bound. + drop(_hold); + run.notify(caught(result)) + } // Cancelled (or the opt-in caller give-up): the work future is // dropped here, and with it any downstream receiver. future::Either::Right(_) => state.cancel(), @@ -383,6 +389,8 @@ impl Context { let _ended = ended; let _deadline = deadline; let result = std::panic::catch_unwind(AssertUnwindSafe(|| work(&job))); + // Execution ended: release the native slot and credit first. + drop((_native, _hold)); thread_run.notify(caught(result)); }); match spawned { diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index 82be81d1..bd28c86e 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -296,6 +296,8 @@ impl Context { let _ended = ended; let _deadline = deadline; let result = std::panic::catch_unwind(AssertUnwindSafe(|| work(&job))); + // Execution ended: release the native slot and credit first. + drop((_native, _hold)); thread_run.notify(caught(result)); }); match spawned { diff --git a/concurrency/tests/deferred_replies.rs b/concurrency/tests/deferred_replies.rs index caa27509..ed2757c5 100644 --- a/concurrency/tests/deferred_replies.rs +++ b/concurrency/tests/deferred_replies.rs @@ -61,6 +61,7 @@ mod tasks_mode { use super::*; use spawned_concurrency::tasks::{Actor, ActorStart, Context, Handler}; use spawned_concurrency::PendingReply; + use spawned_rt::tasks::sleep; pub struct Parker { pub parked: Vec<(u64, ReplyTo)>, @@ -402,6 +403,25 @@ mod tasks_mode { actor.join().await; } + /// agy #11: `closed()` re-arms when the deadline is shortened while it + /// waits. + #[spawned_macros::sim_test] + async fn closed_follows_a_shortened_deadline() { + let (reply, _rx) = spawned_concurrency::reply::reply_pair::(Deadline::Never); + let token = std::sync::Arc::new(reply.defer().unwrap()); + let watcher = { + let token = token.clone(); + spawned_rt::tasks::spawn(async move { token.closed().await }) + }; + sleep(Duration::from_millis(5)).await; + token.with_deadline(spawned_rt::time::Instant::now() + Duration::from_millis(20)); + spawned_rt::tasks::timeout(Duration::from_secs(5), watcher) + .await + .expect("closed() missed the shortened deadline") + .unwrap(); + assert!(token.is_closed()); + } + #[spawned_macros::sim_test] async fn request_timeout_starts_at_send() { let (actor, _) = start(); diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index fa0b75c1..f996ba0b 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -438,3 +438,29 @@ fn abandon_vs_shorten_vs_expiry() { assert_balanced(&pool, &[&slot]); }); } + +/// agy #6: a deferral that finds the pool full while the caller abandons +/// reports `Overloaded` only if the slot really became `OVERLOADED`; a slot +/// that closed first reports `Closed`. +#[test] +fn defer_at_the_bound_vs_abandon() { + model(|| { + let pool = Arc::new(CreditPool::new(0)); + let slot: Arc> = Arc::new(Slot::new(NEVER)); + let caller = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.abandon()) + }; + let deferred = slot.defer(&pool, &|| 0); + let abandoned = caller.join().unwrap(); + match deferred { + Err(Refused::Overloaded) => { + assert_eq!(state(slot.word()), slot::OVERLOADED); + assert!(!abandoned); + } + Err(Refused::Closed) => assert!(abandoned), + Ok(()) => panic!("a full pool granted a credit"), + } + assert_eq!(pool.used(), 0); + }); +} diff --git a/testing/src/rules.rs b/testing/src/rules.rs index 550a6dd6..667e29b6 100644 --- a/testing/src/rules.rs +++ b/testing/src/rules.rs @@ -44,6 +44,8 @@ pub(crate) enum Plan { Crash(String), /// `Mock::defer`: take the reply obligation and store it for the test. Defer, + /// `Mock::discard`: wait `delay`, then drop the reply obligation. + Discard { delay: Option }, } #[derive(Default)] @@ -149,14 +151,10 @@ impl Rules { return Plan::Crash(reason); } if rule.discard { - let value = unit_result::().expect("discard requires Result = ()"); - let mut record = inbox.arrive(msg); - record.replied(Outcome::Discarded); - return Plan::Reply { - delay, - value, - record: Some(record), - }; + // Discarded: the reply obligation is dropped unanswered, so a + // request gets `ReplyDropped`, as a lost message would. + inbox.push(msg, Outcome::Discarded); + return Plan::Discard { delay }; } if rule.reply.is_none() && unit_result::().is_none() { inbox.push(msg, Outcome::Unexpected); diff --git a/testing/src/shared.rs b/testing/src/shared.rs index e91ef62e..6f936921 100644 --- a/testing/src/shared.rs +++ b/testing/src/shared.rs @@ -62,11 +62,11 @@ macro_rules! mock_builder { /// Fault: log each `M` as [`crate::Outcome::Discarded`] and do nothing /// else, as if the message were lost after a successful send. /// - /// Meant for fire-and-forget messages. A handler cannot tell a send - /// from a request, so a *request* `M` (a protocol method returning - /// `Response<()>`) still gets its `()` reply and the caller sees - /// success, whereas a genuinely lost request would time out; use - /// [`delay`](Self::delay) beyond the caller's timeout for that. A + /// Meant for fire-and-forget messages. A *request* `M` (a protocol + /// method returning `Response<()>`) has its reply obligation dropped + /// unanswered, so the caller gets `ReplyDropped`; use + /// [`delay`](Self::delay) beyond the caller's timeout to simulate a + /// request that is never answered. A /// discarded message stays `Discarded` even if the mock dies during /// a `delay` on it. pub fn discard>(mut self) -> Self { diff --git a/testing/src/tasks.rs b/testing/src/tasks.rs index ab383e5d..a7c84ba2 100644 --- a/testing/src/tasks.rs +++ b/testing/src/tasks.rs @@ -117,6 +117,12 @@ impl Handler for Mock { } reply.reply(value); } + Plan::Discard { delay } => { + if let Some(delay) = delay { + sleep(delay).await; + } + drop(reply); + } Plan::Defer => { if let Ok(token) = reply.defer() { inbox.store_deferred::(token); diff --git a/testing/src/threads.rs b/testing/src/threads.rs index 4a46d4ba..51569498 100644 --- a/testing/src/threads.rs +++ b/testing/src/threads.rs @@ -67,6 +67,12 @@ impl Handler for Mock { } reply.reply(value); } + Plan::Discard { delay } => { + if let Some(delay) = delay { + std::thread::sleep(delay); + } + drop(reply); + } Plan::Defer => { if let Ok(token) = reply.defer() { self.inbox.store_deferred::(token); From 529fd229bcb44e63d4f14639462184d43f313d22 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 21:36:42 -0300 Subject: [PATCH 043/208] fix(slot): defer at the bound reports Closed when the caller left first (found by Loom) --- concurrency/src/reply/slot.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 00cd0cca..51881def 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -425,10 +425,10 @@ impl Slot { }) { Ok(_) => Ok(()), Err(Some(())) => Err(Refused::Closed), - Err(None) => { - self.terminate(Terminal::Overloaded); - Err(Refused::Overloaded) - } + // At the bound the caller is answered `Overloaded` in the same + // step; if it closed first (abandoned, expired), this is `Closed`. + Err(None) if self.terminate(Terminal::Overloaded) => Err(Refused::Overloaded), + Err(None) => Err(Refused::Closed), } } From 581e7de36fce6719790cbac6aee4d299009b01ea Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 21:37:26 -0300 Subject: [PATCH 044/208] docs(design): implementation notes for #239 --- docs/design/deferred-replies.md | 34 ++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index 65b6a23d..7dc9c290 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -1,6 +1,6 @@ # Deferred replies and pipe-to-self: design contract -**Status: design in progress, not implemented.** +**Status: implementation in review (PR #239); see [Implementation notes](#implementation-notes).** - Covers ROADMAP rank 11 and the rank 11a decisions. The rank 11b compute pool is deferred to 11b. @@ -922,6 +922,38 @@ ordering and `EmitCreditsBounded` checks in-flight emit credits. **Trace validation:** against both models. +## Implementation notes + +Mechanism choices made by the implementation (PR #239). None changes the +semantics above. + +- **Value cell.** The crates are `#![forbid(unsafe_code)]`, so the slot's value + and the caller's waker live under one uncontended `Mutex` instead of an + `UnsafeCell` plus `AtomicWaker`. The packed word, its CAS rules and every + credit delta are as specified; the reply reads the deadline under the same + lock that a shortening takes, which makes a shortening after the `WRITING` + CAS a no-op (Loom: `reply_vs_shortening`). +- **Receiver and timer.** `ReplyReceiver` is one pointer and implements + `IntoFuture`; the deadline timer lives in the awaiting `ReplyFuture`. When + the runtime's timer fires the receiver expires the slot, so paused Tokio + clocks keep working. +- **Deadline service.** Unmanaged and managed actors share one process-wide + service thread (not a task on the `ActorSystem` control runtime); a dead + owner is forgotten at its registry close. Under `spawned_sim` each owner's + heap is served by a task of the simulated runtime, so clock domains never + mix. +- **Cleanup deadline.** Armed when exit begins: the supervisor's + `shutdown_timeout` for a supervised child, the remaining system grace + during `ActorSystem::shutdown`, otherwise `DEFAULT_STOP_DEADLINE` (5 s). + Jobs still running then appear in `ShutdownReport::unfinished_jobs` and as + an `unfinished_at_stop` event. +- **Async emits.** `ctx.pipe_with_ctx(opts, |job| async move { .. }, done)` + hands async work its `JobCtx` (`emit`, `cancelled`); `pipe(work, done)` + keeps taking a plain future, as in the delegation example. +- **Destruction.** If an owner's lifecycle future is dropped (its runtime + destroyed), exit begins before its state is dropped and the registry + closes, so open tokens answer `ActorStopped`. + ## Open questions 1. Exact names (`PendingReply`, `DeferError`, `PipeReplyError`) and whether From c134b6f2069e5f652ddab30bc3d0e5ff3edbc3ca Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 21:38:57 -0300 Subject: [PATCH 045/208] perf(reply): raw receivers put their deadline in the owner's heap instead of arming a runtime timer; request/Response keep the runtime timer --- concurrency/src/reply/mod.rs | 48 +++++++++++++++++++++++++++++--- concurrency/src/response.rs | 4 +-- concurrency/src/tasks/actor.rs | 12 ++++---- concurrency/src/threads/actor.rs | 6 ++-- 4 files changed, 54 insertions(+), 16 deletions(-) diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 116886c7..99e3c813 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -111,9 +111,17 @@ fn terminal_error(terminal: Terminal) -> ActorError { } /// A new reply slot for a request with `deadline`, and the caller's receiver. -pub(crate) fn reply_slot(deadline: Deadline) -> (SlotHandle, ReplyReceiver) { +/// +/// `owner` is the receiving instance's registry: an awaited raw receiver +/// puts its deadline in that owner's heap instead of arming its own timer. +pub(crate) fn reply_slot( + deadline: Deadline, + owner: Option<&Arc>, +) -> (SlotHandle, ReplyReceiver) { let slot = Arc::new(Slot::new(deadline.ticks())); - (SlotHandle(Some(slot.clone())), ReplyReceiver::new(slot)) + let mut receiver = ReplyReceiver::new(slot.clone()); + receiver.owner = owner.cloned(); + (SlotHandle(Some(slot)), receiver) } // --------------------------------------------------------------------------- @@ -427,6 +435,7 @@ impl Drop for ReplyTo { /// any deferred credit returns at once. pub struct ReplyReceiver { slot: Option>>, + owner: Option>, } impl Drop for ReplyReceiver { @@ -447,7 +456,20 @@ impl std::fmt::Debug for ReplyReceiver { impl ReplyReceiver { fn new(slot: Arc>) -> Self { - Self { slot: Some(slot) } + Self { + slot: Some(slot), + owner: None, + } + } + + /// Await with the caller's own runtime timer (the `request` and + /// `Response` paths, which follow the runtime's clock, paused test clocks + /// included). + #[cfg(feature = "async")] + pub(crate) fn into_timed_future(self) -> ReplyFuture { + let mut future = std::future::IntoFuture::into_future(self); + future.timer = true; + future } fn slot(&self) -> &Arc> { @@ -483,6 +505,8 @@ impl std::future::IntoFuture for ReplyReceiver { rx: self, armed: NEVER, sleep: None, + timer: false, + tracked: false, } } } @@ -494,6 +518,10 @@ pin_project_lite::pin_project! { pub struct ReplyFuture { rx: ReplyReceiver, armed: u64, + // Arm a runtime timer (`request`/`Response`), rather than putting the + // deadline in the owner's heap (raw receivers). + timer: bool, + tracked: bool, #[pin] sleep: Option, } @@ -526,6 +554,18 @@ impl std::future::Future for ReplyFuture { if deadline == NEVER { return Poll::Pending; } + // A raw receiver registers its deadline once in the owner's + // deadline heap; the service expires the slot (waking this + // future) without a per-request runtime timer. + if !*this.timer { + if let Some(owner) = &this.rx.owner { + if !*this.tracked { + owner.track_deadline(slot); + *this.tracked = true; + } + return Poll::Pending; + } + } if *this.armed != deadline || this.sleep.is_none() { let wait = Duration::from_nanos(deadline.saturating_sub(now_ticks())); this.sleep.set(Some(spawned_rt::tasks::sleep(wait))); @@ -558,7 +598,7 @@ impl std::future::Future for ReplyFuture { pub fn reply_pair( deadline: Deadline, ) -> (PendingReply<'static, R>, ReplyReceiver) { - let (mut handle, receiver) = reply_slot(deadline); + let (mut handle, receiver) = reply_slot(deadline, None); let reply = PendingReply::new(handle.take(), Registry::detached()); (reply, receiver) } diff --git a/concurrency/src/response.rs b/concurrency/src/response.rs index f27ce410..01979325 100644 --- a/concurrency/src/response.rs +++ b/concurrency/src/response.rs @@ -2,8 +2,6 @@ use crate::error::ActorError; #[cfg(feature = "async")] use crate::reply::{ReplyFuture, ReplyReceiver}; use std::future::Future; -#[cfg(feature = "async")] -use std::future::IntoFuture; use std::pin::Pin; use std::time::Duration; @@ -154,7 +152,7 @@ impl From, ActorError>> for Response { let observation = crate::instrumentation::RequestObservation::new(); match result { Ok(rx) => Self( - ResponseState::Receiver(Box::pin(rx.into_future())), + ResponseState::Receiver(Box::pin(rx.into_timed_future())), #[cfg(feature = "metrics")] Some(observation), ), diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 3c9ad919..eed8d055 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -405,7 +405,7 @@ impl Context { A: Handler, M: Message, { - let (slot, rx) = reply_slot(deadline); + let (slot, rx) = reply_slot(deadline, Some(&self.shared.replies)); enqueue(&self.shared.sender, &self.shared.mailbox, msg, slot)?; Ok(rx) } @@ -442,7 +442,7 @@ impl Context { return result; } }; - let result = rx.await; + let result = rx.into_timed_future().await; observation.finish(&result); result } @@ -779,7 +779,7 @@ pub async fn request( return result; } }; - let result = rx.await; + let result = rx.into_timed_future().await; observation.finish(&result); result } @@ -868,7 +868,7 @@ impl ActorRef { A: Handler, M: Message, { - let (slot, rx) = reply_slot(deadline); + let (slot, rx) = reply_slot(deadline, Some(&self.shared.replies)); enqueue(&self.shared.sender, &self.shared.mailbox, msg, slot)?; Ok(rx) } @@ -905,7 +905,7 @@ impl ActorRef { return result; } }; - let result = rx.await; + let result = rx.into_timed_future().await; observation.finish(&result); result } @@ -3504,7 +3504,7 @@ mod idle_stop_regression { // Start and park the actual actor loop, without allowing another // executor task to run it between admission and stop below. assert!(futures::poll!(actor.as_mut()).is_pending()); - let (slot, response) = reply_slot(crate::reply::Deadline::Never); + let (slot, response) = reply_slot(crate::reply::Deadline::Never, None); tx.send(Box::new(MessageEnvelope { msg: Increment, slot, diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 39597196..801b5bae 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -314,7 +314,7 @@ impl Context { A: Handler, M: Message, { - let (slot, rx) = reply_slot(deadline); + let (slot, rx) = reply_slot(deadline, None); enqueue(&self.sender, &self.mailbox, msg, slot)?; Ok(rx) } @@ -700,7 +700,7 @@ impl ActorRef { A: Handler, M: Message, { - let (slot, rx) = reply_slot(deadline); + let (slot, rx) = reply_slot(deadline, None); enqueue(&self.sender, &self.mailbox, msg, slot)?; Ok(rx) } @@ -1993,7 +1993,7 @@ mod idle_stop_regression { ) }); entered.wait(); - let (slot, response) = reply_slot(crate::reply::Deadline::Never); + let (slot, response) = reply_slot(crate::reply::Deadline::Never, None); tx.send(MailboxItem::Message(Box::new(MessageEnvelope { msg: Increment(events.clone(), monitors), slot, From f47ddb26dfd1eefe77b7f5ad6584d6222913cf13 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 21:39:16 -0300 Subject: [PATCH 046/208] fix: bounds for the owner-heap receiver path --- concurrency/src/reply/mod.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 99e3c813..fca14482 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -467,9 +467,13 @@ impl ReplyReceiver { /// included). #[cfg(feature = "async")] pub(crate) fn into_timed_future(self) -> ReplyFuture { - let mut future = std::future::IntoFuture::into_future(self); - future.timer = true; - future + ReplyFuture { + rx: self, + armed: NEVER, + sleep: None, + timer: true, + tracked: false, + } } fn slot(&self) -> &Arc> { @@ -497,7 +501,7 @@ impl ReplyReceiver { } #[cfg(feature = "async")] -impl std::future::IntoFuture for ReplyReceiver { +impl std::future::IntoFuture for ReplyReceiver { type Output = Result; type IntoFuture = ReplyFuture; fn into_future(self) -> ReplyFuture { @@ -535,7 +539,7 @@ impl std::fmt::Debug for ReplyFuture { } #[cfg(feature = "async")] -impl std::future::Future for ReplyFuture { +impl std::future::Future for ReplyFuture { type Output = Result; fn poll( From 85ff997af95897194d7e4f061ec4118aeb336fcf Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 21:39:55 -0300 Subject: [PATCH 047/208] perf: the slot carries its owner; ReplyReceiver stays one pointer --- concurrency/src/reply/mod.rs | 15 +++++---------- concurrency/src/reply/slot.rs | 8 ++++++++ concurrency/tests/reply_slot_loom.rs | 1 + 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index fca14482..f78376d9 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -31,6 +31,7 @@ mod sync { pub(crate) use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; pub(crate) use std::sync::{Arc, Condvar, Mutex, MutexGuard}; pub(crate) use std::thread::yield_now; + pub(crate) type SlotOwner = Option>; /// Wait on `condvar` for at most `ticks` clock ticks (nanoseconds), or /// until notified when `None`. @@ -118,10 +119,8 @@ pub(crate) fn reply_slot( deadline: Deadline, owner: Option<&Arc>, ) -> (SlotHandle, ReplyReceiver) { - let slot = Arc::new(Slot::new(deadline.ticks())); - let mut receiver = ReplyReceiver::new(slot.clone()); - receiver.owner = owner.cloned(); - (SlotHandle(Some(slot)), receiver) + let slot = Arc::new(Slot::with_owner(deadline.ticks(), owner.cloned())); + (SlotHandle(Some(slot.clone())), ReplyReceiver::new(slot)) } // --------------------------------------------------------------------------- @@ -435,7 +434,6 @@ impl Drop for ReplyTo { /// any deferred credit returns at once. pub struct ReplyReceiver { slot: Option>>, - owner: Option>, } impl Drop for ReplyReceiver { @@ -456,10 +454,7 @@ impl std::fmt::Debug for ReplyReceiver { impl ReplyReceiver { fn new(slot: Arc>) -> Self { - Self { - slot: Some(slot), - owner: None, - } + Self { slot: Some(slot) } } /// Await with the caller's own runtime timer (the `request` and @@ -562,7 +557,7 @@ impl std::future::Future for ReplyFuture { // deadline heap; the service expires the slot (waking this // future) without a per-request runtime timer. if !*this.timer { - if let Some(owner) = &this.rx.owner { + if let Some(owner) = &slot.owner { if !*this.tracked { owner.track_deadline(slot); *this.tracked = true; diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 51881def..ffd8518a 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -178,6 +178,9 @@ struct Inner { } pub(crate) struct Slot { + /// The receiving instance (production: its registry), whose deadline + /// heap expires an awaited raw request. + pub(crate) owner: super::sync::SlotOwner, word: AtomicU64, deadline: AtomicU64, inner: Mutex>, @@ -217,7 +220,12 @@ impl Wake { impl Slot { /// A fresh `OPEN` slot whose deadline is `deadline` ticks (`NEVER` for none). pub(crate) fn new(deadline: u64) -> Self { + Self::with_owner(deadline, Default::default()) + } + + pub(crate) fn with_owner(deadline: u64, owner: super::sync::SlotOwner) -> Self { Self { + owner, word: AtomicU64::new(OPEN), deadline: AtomicU64::new(deadline), inner: Mutex::new(Inner { diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index f996ba0b..11dd2115 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -11,6 +11,7 @@ mod sync { pub(crate) use loom::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; pub(crate) use loom::sync::{Arc, Condvar, Mutex, MutexGuard}; pub(crate) use loom::thread::yield_now; + pub(crate) type SlotOwner = Option<()>; /// Loom has no timed wait; tests park only on slots without a deadline. pub(crate) fn wait_ticks<'a, T>( From 9ab004ce13e1a67e7df5ffc603f9a0c7e4ed85ec Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 21:40:15 -0300 Subject: [PATCH 048/208] chore: allow the ownerless slot constructor --- concurrency/src/reply/slot.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index ffd8518a..c9e93a03 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -219,6 +219,7 @@ impl Wake { impl Slot { /// A fresh `OPEN` slot whose deadline is `deadline` ticks (`NEVER` for none). + #[allow(dead_code)] // the Loom model and unit tests build ownerless slots pub(crate) fn new(deadline: u64) -> Self { Self::with_owner(deadline, Default::default()) } From ddff7f16300b6ec9a54ceb47d14280c0d9a5c395 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 21:41:59 -0300 Subject: [PATCH 049/208] perf(registry): keep the turn phase off the Arc-count cache line --- concurrency/src/reply/registry.rs | 33 ++++++++++++++++++------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index 5e4957a6..1bbf71c7 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -66,13 +66,18 @@ struct Lists { deadlines: DeadlineHeap, } +#[repr(align(64))] +struct Phase(AtomicU8); + pub(crate) struct Registry { owner: ActorId, /// Service key, unique per registry. #[cfg_attr(all(spawned_sim, feature = "async"), allow(dead_code))] key: u64, pool: Arc, - phase: AtomicU8, + /// On its own cache line: the run loop writes it every turn, while + /// callers touch the `Arc` counts and the lists from other threads. + phase: Phase, lists: Mutex, /// When exit began: the cleanup deadline is armed from here. exit_began: Mutex>, @@ -92,7 +97,7 @@ impl Registry { owner, key: NEXT_KEY.fetch_add(1, Ordering::Relaxed), pool: Arc::new(CreditPool::new(max)), - phase: AtomicU8::new(RUNNING), + phase: Phase(AtomicU8::new(RUNNING)), lists: Mutex::new(Lists { prune_at: 16, ..Lists::default() @@ -167,7 +172,7 @@ impl Registry { } pub(crate) fn is_dead(&self) -> bool { - self.phase.load(Ordering::Acquire) == DEAD + self.phase.0.load(Ordering::Acquire) == DEAD } /// Put a deferred token's deadline in the owner's heap, so it expires @@ -222,7 +227,7 @@ impl Registry { } let slot: Arc = slot; loop { - match self.phase.load(Ordering::Acquire) { + match self.phase.0.load(Ordering::Acquire) { RUNNING => { slot.terminate(Terminal::Dropped); return; @@ -238,12 +243,12 @@ impl Registry { _ => { let mut lists = lock(&self.lists); lists.turn_drops.push(slot.clone()); - let mut current = self.phase.load(Ordering::Acquire); + let mut current = self.phase.0.load(Ordering::Acquire); let recorded = loop { if current != IN_TURN && current != IN_TURN_PENDING { break false; } - match self.phase.compare_exchange( + match self.phase.0.compare_exchange( current, IN_TURN_PENDING, Ordering::AcqRel, @@ -275,13 +280,13 @@ impl Registry { /// Only the run loop moves the phase out of `RUNNING`/`IN_TURN` (droppers /// only mark `IN_TURN_PENDING`), so a plain store suffices here. pub(crate) fn begin_turn(&self) { - debug_assert_eq!(self.phase.load(Ordering::Relaxed), RUNNING); - self.phase.store(IN_TURN, Ordering::Release); + debug_assert_eq!(self.phase.0.load(Ordering::Relaxed), RUNNING); + self.phase.0.store(IN_TURN, Ordering::Release); } /// The handler returned (`panicked == false`) or unwound. pub(crate) fn end_turn(&self, panicked: bool) { - if self.phase.swap(RUNNING, Ordering::AcqRel) == IN_TURN_PENDING { + if self.phase.0.swap(RUNNING, Ordering::AcqRel) == IN_TURN_PENDING { self.commit(if panicked { Terminal::Stopped } else { @@ -293,7 +298,7 @@ impl Registry { /// Exit begins: later drops answer `Stopped`, and the current turn's /// recorded drops commit as `Stopped`. pub(crate) fn exiting(&self) { - let previous = self.phase.load(Ordering::Acquire); + let previous = self.phase.0.load(Ordering::Acquire); if previous == DEAD { return; } @@ -301,7 +306,7 @@ impl Registry { .lock() .unwrap_or_else(|p| p.into_inner()) .get_or_insert_with(spawned_rt::time::Instant::now); - if self.phase.swap(EXITING, Ordering::AcqRel) == IN_TURN_PENDING { + if self.phase.0.swap(EXITING, Ordering::AcqRel) == IN_TURN_PENDING { self.commit(Terminal::Stopped); } } @@ -313,13 +318,13 @@ impl Registry { } pub(crate) fn is_exiting(&self) -> bool { - self.phase.load(Ordering::Acquire) >= EXITING + self.phase.0.load(Ordering::Acquire) >= EXITING } /// Death: every open token listed becomes `STOPPED`, wherever it is. /// Returns how many were closed. pub(crate) fn close(&self) -> usize { - if self.phase.swap(DEAD, Ordering::AcqRel) == IN_TURN_PENDING { + if self.phase.0.swap(DEAD, Ordering::AcqRel) == IN_TURN_PENDING { self.commit(Terminal::Stopped); } let tokens = std::mem::take(&mut lock(&self.lists).tokens); @@ -341,7 +346,7 @@ impl Registry { impl Drop for Registry { fn drop(&mut self) { // Every handle to the instance is gone; nothing can still answer. - if self.phase.load(Ordering::Acquire) != DEAD { + if self.phase.0.load(Ordering::Acquire) != DEAD { self.close(); } } From f299b8236622fa840fae91f912947c5fd89d277b Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 21:50:08 -0300 Subject: [PATCH 050/208] fix: test races on the handler's record of Overloaded; clippy (test module last, cfg-gated items) --- concurrency/src/reply/registry.rs | 49 ++++++++++++++++--------------- concurrency/src/reply/slot.rs | 1 + 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index 1bbf71c7..47652b7a 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -352,6 +352,32 @@ impl Drop for Registry { } } +/// Closes the registry and cancels the jobs if the owner's lifecycle is +/// dropped without completing (for example, its runtime was destroyed). +/// Idempotent after a normal death. +#[cfg_attr(not(feature = "async"), allow(dead_code))] +pub(crate) struct CloseOnDrop { + replies: Arc, + jobs: Arc, +} + +#[cfg_attr(not(feature = "async"), allow(dead_code))] +impl CloseOnDrop { + pub(crate) fn new(replies: Arc, jobs: Arc) -> Self { + Self { replies, jobs } + } +} + +impl Drop for CloseOnDrop { + fn drop(&mut self) { + if !self.replies.is_dead() { + self.replies.exiting(); + self.jobs.cancel_all(); + self.replies.close(); + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -438,26 +464,3 @@ mod tests { } } -/// Closes the registry and cancels the jobs if the owner's lifecycle is -/// dropped without completing (for example, its runtime was destroyed). -/// Idempotent after a normal death. -pub(crate) struct CloseOnDrop { - replies: Arc, - jobs: Arc, -} - -impl CloseOnDrop { - pub(crate) fn new(replies: Arc, jobs: Arc) -> Self { - Self { replies, jobs } - } -} - -impl Drop for CloseOnDrop { - fn drop(&mut self) { - if !self.replies.is_dead() { - self.replies.exiting(); - self.jobs.cancel_all(); - self.replies.close(); - } - } -} diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index c9e93a03..5579afce 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -180,6 +180,7 @@ struct Inner { pub(crate) struct Slot { /// The receiving instance (production: its registry), whose deadline /// heap expires an awaited raw request. + #[cfg_attr(not(feature = "async"), allow(dead_code))] pub(crate) owner: super::sync::SlotOwner, word: AtomicU64, deadline: AtomicU64, From 65d57f3c0de6bff6abb2567251e7290cfe3f3db0 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 21:50:51 -0300 Subject: [PATCH 051/208] test: read the handler's Overloaded record after a later turn --- concurrency/tests/deferred_replies.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/concurrency/tests/deferred_replies.rs b/concurrency/tests/deferred_replies.rs index ed2757c5..368b7c87 100644 --- a/concurrency/tests/deferred_replies.rs +++ b/concurrency/tests/deferred_replies.rs @@ -226,6 +226,9 @@ mod tasks_mode { actor.request(Park(3)).await, Err(ActorError::Overloaded) )); + // The caller is answered in the deferring step; the handler's record + // of it is visible once a later turn ran (FIFO). + settle(&actor).await; assert_eq!(*seen.defer_errors.lock().unwrap(), [DeferError::Overloaded]); assert_eq!(actor.request(Release(0)).await.unwrap(), 2); assert_eq!((a.await.unwrap(), b.await.unwrap()), (1, 2)); @@ -560,10 +563,11 @@ mod threads_mode { actor.request(Park(3)), Err(ActorError::Overloaded) )); - assert_eq!(*seen.defer_errors.lock().unwrap(), [DeferError::Overloaded]); // Abandonment returns the credit. drop(c); assert_eq!(actor.request(Outstanding).unwrap(), 1); + // The handler's record is visible after a later turn (FIFO). + assert_eq!(*seen.defer_errors.lock().unwrap(), [DeferError::Overloaded]); // Expired deadline: the receiver expires the slot; the late reply is dropped. let e = actor .request_raw_until(Park(4), Deadline::after(Duration::from_millis(20))) From 200cff64137100da6e19630e2816e67f35750081 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Tue, 10 Nov 2026 21:52:48 -0300 Subject: [PATCH 052/208] refactor(registry): turn phase in its own module; Loom model of drop vs turn end vs exit vs death --- .github/workflows/verification.yml | 2 +- concurrency/src/reply/mod.rs | 3 +- concurrency/src/reply/registry.rs | 104 +++++------------- concurrency/src/reply/turn.rs | 140 ++++++++++++++++++++++++ concurrency/tests/registry_turn_loom.rs | 120 ++++++++++++++++++++ 5 files changed, 288 insertions(+), 81 deletions(-) create mode 100644 concurrency/src/reply/turn.rs create mode 100644 concurrency/tests/registry_turn_loom.rs diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml index 977bdba3..3d13c8bd 100644 --- a/.github/workflows/verification.yml +++ b/.github/workflows/verification.yml @@ -103,7 +103,7 @@ jobs: - name: Model-check the reply slot's packed-word protocol env: RUSTFLAGS: --cfg spawned_scheduler_loom - run: cargo test --locked --release -p spawned-concurrency --no-default-features --test reply_slot_loom + run: cargo test --locked --release -p spawned-concurrency --no-default-features --test reply_slot_loom --test registry_turn_loom # Data races in the runtime and its dependencies. The crates forbid # unsafe code, so ThreadSanitizer is the sanitizer that matters here. diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index f78376d9..541b1341 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -11,6 +11,7 @@ pub mod jobs; mod pending; pub(crate) mod registry; pub(crate) mod slot; +pub(crate) mod turn; pub use pending::{InsertError, PendingReplies}; pub use registry::DEFAULT_MAX_OUTSTANDING; @@ -28,7 +29,7 @@ use std::time::Duration; /// Production synchronization types for [`slot`]; the Loom model supplies its /// own (`tests/reply_slot_loom.rs`). mod sync { - pub(crate) use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + pub(crate) use std::sync::atomic::{AtomicU64, AtomicU8, AtomicUsize, Ordering}; pub(crate) use std::sync::{Arc, Condvar, Mutex, MutexGuard}; pub(crate) use std::thread::yield_now; pub(crate) type SlotOwner = Option>; diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index 47652b7a..49452385 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -17,17 +17,20 @@ use super::deadline::{self, DeadlineHeap, Expirable}; use super::slot::{CreditPool, Slot, Terminal, NEVER}; use crate::ActorId; -use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, OnceLock, Weak}; /// Default `Actor::MAX_OUTSTANDING`: deferred credits per actor instance. pub const DEFAULT_MAX_OUTSTANDING: usize = 1024; -const RUNNING: u8 = 0; -const IN_TURN: u8 = 1; -const IN_TURN_PENDING: u8 = 2; -const EXITING: u8 = 3; -const DEAD: u8 = 4; +use super::turn::{Outcome, TurnPhase, DEAD, EXITING}; + +fn terminal(outcome: Outcome) -> Terminal { + match outcome { + Outcome::Dropped => Terminal::Dropped, + Outcome::Stopped => Terminal::Stopped, + } +} /// A slot seen without its reply type. pub(crate) trait Closeable: Send + Sync { @@ -62,22 +65,17 @@ impl Closeable for Slot { struct Lists { tokens: Vec>, prune_at: usize, - turn_drops: Vec>, deadlines: DeadlineHeap, } -#[repr(align(64))] -struct Phase(AtomicU8); - pub(crate) struct Registry { owner: ActorId, /// Service key, unique per registry. #[cfg_attr(all(spawned_sim, feature = "async"), allow(dead_code))] key: u64, pool: Arc, - /// On its own cache line: the run loop writes it every turn, while - /// callers touch the `Arc` counts and the lists from other threads. - phase: Phase, + /// Turn phase and in-turn drops (its phase sits on its own cache line). + turn: TurnPhase>, lists: Mutex, /// When exit began: the cleanup deadline is armed from here. exit_began: Mutex>, @@ -97,7 +95,7 @@ impl Registry { owner, key: NEXT_KEY.fetch_add(1, Ordering::Relaxed), pool: Arc::new(CreditPool::new(max)), - phase: Phase(AtomicU8::new(RUNNING)), + turn: TurnPhase::default(), lists: Mutex::new(Lists { prune_at: 16, ..Lists::default() @@ -172,7 +170,7 @@ impl Registry { } pub(crate) fn is_dead(&self) -> bool { - self.phase.0.load(Ordering::Acquire) == DEAD + self.turn.load() == DEAD } /// Put a deferred token's deadline in the owner's heap, so it expires @@ -226,89 +224,39 @@ impl Registry { return; } let slot: Arc = slot; - loop { - match self.phase.0.load(Ordering::Acquire) { - RUNNING => { - slot.terminate(Terminal::Dropped); - return; - } - EXITING => { - slot.terminate(Terminal::Stopped); - return; - } - DEAD => { - slot.terminate(Terminal::Stopped); - return; - } - _ => { - let mut lists = lock(&self.lists); - lists.turn_drops.push(slot.clone()); - let mut current = self.phase.0.load(Ordering::Acquire); - let recorded = loop { - if current != IN_TURN && current != IN_TURN_PENDING { - break false; - } - match self.phase.0.compare_exchange( - current, - IN_TURN_PENDING, - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => break true, - Err(actual) => current = actual, - } - }; - if recorded { - return; - } - // The turn ended first: classify by the new phase. - lists.turn_drops.pop(); - } - } + if let Some((slot, outcome)) = self.turn.dropped(slot) { + slot.terminate(terminal(outcome)); } } - fn commit(&self, outcome: Terminal) { - let drops = std::mem::take(&mut lock(&self.lists).turn_drops); + fn commit(drops: Vec>, outcome: Outcome) { for slot in drops { - slot.terminate(outcome); + slot.terminate(terminal(outcome)); } } /// The run loop is about to dispatch a handler. - /// - /// Only the run loop moves the phase out of `RUNNING`/`IN_TURN` (droppers - /// only mark `IN_TURN_PENDING`), so a plain store suffices here. pub(crate) fn begin_turn(&self) { - debug_assert_eq!(self.phase.0.load(Ordering::Relaxed), RUNNING); - self.phase.0.store(IN_TURN, Ordering::Release); + self.turn.begin_turn(); } /// The handler returned (`panicked == false`) or unwound. pub(crate) fn end_turn(&self, panicked: bool) { - if self.phase.0.swap(RUNNING, Ordering::AcqRel) == IN_TURN_PENDING { - self.commit(if panicked { - Terminal::Stopped - } else { - Terminal::Dropped - }); - } + let (drops, outcome) = self.turn.end_turn(panicked); + Self::commit(drops, outcome); } /// Exit begins: later drops answer `Stopped`, and the current turn's /// recorded drops commit as `Stopped`. pub(crate) fn exiting(&self) { - let previous = self.phase.0.load(Ordering::Acquire); - if previous == DEAD { + if self.turn.load() == DEAD { return; } self.exit_began .lock() .unwrap_or_else(|p| p.into_inner()) .get_or_insert_with(spawned_rt::time::Instant::now); - if self.phase.0.swap(EXITING, Ordering::AcqRel) == IN_TURN_PENDING { - self.commit(Terminal::Stopped); - } + Self::commit(self.turn.exiting(), Outcome::Stopped); } /// What remains of `timeout`, measured from the start of exit. @@ -318,15 +266,13 @@ impl Registry { } pub(crate) fn is_exiting(&self) -> bool { - self.phase.0.load(Ordering::Acquire) >= EXITING + self.turn.load() >= EXITING } /// Death: every open token listed becomes `STOPPED`, wherever it is. /// Returns how many were closed. pub(crate) fn close(&self) -> usize { - if self.phase.0.swap(DEAD, Ordering::AcqRel) == IN_TURN_PENDING { - self.commit(Terminal::Stopped); - } + Self::commit(self.turn.close(), Outcome::Stopped); let tokens = std::mem::take(&mut lock(&self.lists).tokens); let closed = tokens .into_iter() @@ -346,7 +292,7 @@ impl Registry { impl Drop for Registry { fn drop(&mut self) { // Every handle to the instance is gone; nothing can still answer. - if self.phase.0.load(Ordering::Acquire) != DEAD { + if self.turn.load() != DEAD { self.close(); } } diff --git a/concurrency/src/reply/turn.rs b/concurrency/src/reply/turn.rs new file mode 100644 index 00000000..6b5509a9 --- /dev/null +++ b/concurrency/src/reply/turn.rs @@ -0,0 +1,140 @@ +//! The owner's turn phase and the commit of reply obligations dropped in a +//! turn (docs/design/deferred-replies.md §4, "Drop classification"). +//! +//! | phase | a dropped obligation | +//! |---|---| +//! | `InTurn` | recorded; committed at turn end: `Dropped` if the handler returned, `Stopped` if it panicked | +//! | `Running` | `Dropped` at once | +//! | `Exiting` / `Dead` | `Stopped` at once | +//! +//! Shared with the Loom model (`tests/registry_turn_loom.rs`), which supplies +//! the synchronization types through `super::sync`. The protocol never runs +//! user code: it returns the items and their outcome, and the caller applies +//! them after every lock is released. + +use super::sync::{AtomicU8, Mutex, Ordering}; + +pub(crate) const RUNNING: u8 = 0; +pub(crate) const IN_TURN: u8 = 1; +pub(crate) const IN_TURN_PENDING: u8 = 2; +pub(crate) const EXITING: u8 = 3; +pub(crate) const DEAD: u8 = 4; + +/// How a dropped obligation is answered. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Outcome { + Dropped, + Stopped, +} + +/// On its own cache line: the run loop writes it every turn. +#[repr(align(64))] +struct Phase(AtomicU8); + +pub(crate) struct TurnPhase { + phase: Phase, + drops: Mutex>, +} + +impl Default for TurnPhase { + fn default() -> Self { + Self { + phase: Phase(AtomicU8::new(RUNNING)), + drops: Mutex::new(Vec::new()), + } + } +} + +fn take(drops: &Mutex>) -> Vec { + std::mem::take(&mut *drops.lock().unwrap_or_else(|p| p.into_inner())) +} + +impl TurnPhase { + pub(crate) fn load(&self) -> u8 { + self.phase.0.load(Ordering::Acquire) + } + + /// An obligation was dropped: `None` when it was recorded on the current + /// turn, otherwise the item and how to answer it now. + pub(crate) fn dropped(&self, item: T) -> Option<(T, Outcome)> { + loop { + match self.load() { + RUNNING => return Some((item, Outcome::Dropped)), + EXITING | DEAD => return Some((item, Outcome::Stopped)), + _ => { + let mut drops = self.drops.lock().unwrap_or_else(|p| p.into_inner()); + let mut current = self.load(); + let recorded = loop { + if current != IN_TURN && current != IN_TURN_PENDING { + break false; + } + match self.phase.0.compare_exchange( + current, + IN_TURN_PENDING, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => break true, + Err(actual) => current = actual, + } + }; + if recorded { + drops.push(item); + return None; + } + // The turn ended first: classify by the new phase. + } + } + } + } + + /// The run loop is about to dispatch a handler. Only the run loop moves + /// the phase out of `RUNNING`, so a plain store suffices. + pub(crate) fn begin_turn(&self) { + self.phase.0.store(IN_TURN, Ordering::Release); + } + + /// The handler returned (`panicked == false`) or unwound: the recorded + /// drops and their outcome. + pub(crate) fn end_turn(&self, panicked: bool) -> (Vec, Outcome) { + let outcome = if panicked { + Outcome::Stopped + } else { + Outcome::Dropped + }; + if self.phase.0.swap(RUNNING, Ordering::AcqRel) == IN_TURN_PENDING { + (take(&self.drops), outcome) + } else { + (Vec::new(), outcome) + } + } + + /// Exit begins (idempotent; not after death): recorded drops commit as + /// `Stopped`. + pub(crate) fn exiting(&self) -> Vec { + let mut current = self.load(); + loop { + if current >= EXITING { + return Vec::new(); + } + match self + .phase + .0 + .compare_exchange(current, EXITING, Ordering::AcqRel, Ordering::Acquire) + { + Ok(previous) if previous == IN_TURN_PENDING => return take(&self.drops), + Ok(_) => return Vec::new(), + Err(actual) => current = actual, + } + } + } + + /// Death: recorded drops commit as `Stopped`. + pub(crate) fn close(&self) -> Vec { + if self.phase.0.swap(DEAD, Ordering::AcqRel) == IN_TURN_PENDING { + take(&self.drops) + } else { + Vec::new() + } + } +} diff --git a/concurrency/tests/registry_turn_loom.rs b/concurrency/tests/registry_turn_loom.rs new file mode 100644 index 00000000..6d363753 --- /dev/null +++ b/concurrency/tests/registry_turn_loom.rs @@ -0,0 +1,120 @@ +//! Loom model of the owner's turn phase and in-turn drop commit +//! (`src/reply/turn.rs`, docs/design/deferred-replies.md §4): drops from +//! other threads racing turn end, exit and death. Every dropped obligation +//! gets exactly one outcome, `Stopped` exactly when the turn it landed on +//! panicked or the owner was exiting or dead, and none is lost. +#![cfg(spawned_scheduler_loom)] + +mod sync { + pub(crate) use loom::sync::atomic::{AtomicU8, Ordering}; + pub(crate) use loom::sync::Mutex; +} + +#[allow(dead_code)] +#[path = "../src/reply/turn.rs"] +mod turn; + +use loom::sync::Arc; +use std::sync::Mutex as StdMutex; +use turn::{Outcome, TurnPhase}; + +fn model(f: impl Fn() + Sync + Send + 'static) { + let mut builder = loom::model::Builder::new(); + builder.preemption_bound = Some(3); + builder.check(f); +} + +/// Records each item's outcome; panics on a second one. +#[derive(Default)] +struct Outcomes(StdMutex>); +impl Outcomes { + fn apply(&self, items: Vec, outcome: Outcome) { + for item in items { + self.one(item, outcome); + } + } + fn one(&self, item: u32, outcome: Outcome) { + let mut seen = self.0.lock().unwrap(); + assert!(seen.iter().all(|(i, _)| *i != item), "item {item} answered twice"); + seen.push((item, outcome)); + } + fn of(&self, item: u32) -> Option { + self.0.lock().unwrap().iter().find(|(i, _)| *i == item).map(|(_, o)| *o) + } +} + +fn drop_on(turn: &TurnPhase, outcomes: &Outcomes, item: u32) { + if let Some((item, outcome)) = turn.dropped(item) { + outcomes.one(item, outcome); + } +} + +/// A drop from another thread racing the end of a turn that returned or +/// panicked: it commits with that turn, or (if after it) answers `Dropped`. +#[test] +fn drop_vs_turn_end() { + for panicked in [false, true] { + model(move || { + let turn = Arc::new(TurnPhase::::default()); + let outcomes = Arc::new(Outcomes::default()); + turn.begin_turn(); + let dropper = { + let (turn, outcomes) = (turn.clone(), outcomes.clone()); + loom::thread::spawn(move || drop_on(&turn, &outcomes, 1)) + }; + let (drops, outcome) = turn.end_turn(panicked); + let committed = !drops.is_empty(); + outcomes.apply(drops, outcome); + dropper.join().unwrap(); + let got = outcomes.of(1).expect("a dropped obligation was lost"); + if committed { + let expected = if panicked { Outcome::Stopped } else { Outcome::Dropped }; + assert_eq!(got, expected); + } else { + assert_eq!(got, Outcome::Dropped, "a drop after the turn answers at once"); + } + }); + } +} + +/// Drops racing exit and death: nothing is lost, and anything recorded +/// on the interrupted turn commits as `Stopped`. +#[test] +fn drops_vs_exit_vs_death() { + model(|| { + let turn = Arc::new(TurnPhase::::default()); + let outcomes = Arc::new(Outcomes::default()); + turn.begin_turn(); + let droppers: Vec<_> = (1..=2) + .map(|item| { + let (turn, outcomes) = (turn.clone(), outcomes.clone()); + loom::thread::spawn(move || drop_on(&turn, &outcomes, item)) + }) + .collect(); + outcomes.apply(turn.exiting(), Outcome::Stopped); + outcomes.apply(turn.close(), Outcome::Stopped); + for dropper in droppers { + dropper.join().unwrap(); + } + for item in 1..=2 { + assert_eq!(outcomes.of(item), Some(Outcome::Stopped), "item {item}"); + } + assert_eq!(turn.load(), turn::DEAD); + }); +} + +/// Exit is idempotent and never revives a dead owner. +#[test] +fn exiting_after_death_is_a_no_op() { + model(|| { + let turn = Arc::new(TurnPhase::::default()); + let late = { + let turn = turn.clone(); + loom::thread::spawn(move || turn.exiting()) + }; + let closed = turn.close(); + assert!(closed.is_empty()); + assert!(late.join().unwrap().is_empty()); + assert_eq!(turn.load(), turn::DEAD); + }); +} From f93688a65c17926bd9847c9e3926ec9637441bce Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:54:44 +0000 Subject: [PATCH 053/208] style: cargo fmt --- concurrency/benches/baseline.rs | 6 ++-- concurrency/src/reply/jobs.rs | 6 +++- concurrency/src/reply/mod.rs | 9 +++-- concurrency/src/reply/registry.rs | 1 - concurrency/src/reply/slot.rs | 9 ++++- concurrency/src/system.rs | 5 ++- concurrency/tests/deferred_jobs.rs | 45 +++++++++++++++++++------ concurrency/tests/deferred_replies.rs | 5 ++- concurrency/tests/registry_turn_loom.rs | 24 ++++++++++--- concurrency/tests/reply_slot_loom.rs | 5 ++- testing/src/inbox.rs | 11 ++++-- testing/src/rules.rs | 4 ++- testing/src/shared.rs | 4 ++- 13 files changed, 104 insertions(+), 30 deletions(-) diff --git a/concurrency/benches/baseline.rs b/concurrency/benches/baseline.rs index c841c0d9..a2adff51 100644 --- a/concurrency/benches/baseline.rs +++ b/concurrency/benches/baseline.rs @@ -3,8 +3,8 @@ mod cli; use spawned_concurrency::{message::Message, tasks, threads, ChildHandle, Deadline, ReplyReceiver}; use std::{ - future::IntoFuture, alloc::{GlobalAlloc, Layout, System}, + future::IntoFuture, hint::black_box, sync::{ atomic::{AtomicBool, AtomicU64, Ordering}, @@ -155,7 +155,9 @@ impl Target { impl Reply { fn wait(self, rt: &tokio::runtime::Handle) -> Ack { match self { - Self::Tasks(rx) => rt.block_on(rx.into_future()).expect("reply timeout or dropped"), + Self::Tasks(rx) => rt + .block_on(rx.into_future()) + .expect("reply timeout or dropped"), Self::Tokio(rx) => rt.block_on(async { tokio::time::timeout(Duration::from_secs(10), rx) .await diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index db6693a9..338202a3 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -627,7 +627,11 @@ mod tests { let registry = Registry::new(ActorId::next(), 1); let held = JobCredit::reserve(®istry).expect("first credit"); assert!(JobCredit::reserve(®istry).is_none()); - assert_eq!(registry.outstanding(), 1, "a rejected job released a credit"); + assert_eq!( + registry.outstanding(), + 1, + "a rejected job released a credit" + ); drop(held); assert_eq!(registry.outstanding(), 0); } diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 541b1341..0ede7a20 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -473,7 +473,9 @@ impl ReplyReceiver { } fn slot(&self) -> &Arc> { - self.slot.as_ref().expect("ReplyReceiver used after completion") + self.slot + .as_ref() + .expect("ReplyReceiver used after completion") } /// The deadline fixed at send, or `None` for [`Deadline::Never`]. @@ -484,7 +486,10 @@ impl ReplyReceiver { /// Block the calling thread until the reply, the deadline or the owner's /// death. Do not call it on an async executor thread. pub fn recv(mut self) -> Result { - let slot = self.slot.take().expect("ReplyReceiver used after completion"); + let slot = self + .slot + .take() + .expect("ReplyReceiver used after completion"); slot.recv_blocking(&now_ticks).map_err(terminal_error) } diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index 49452385..530c0815 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -409,4 +409,3 @@ mod tests { assert_eq!(registry.outstanding(), 0); } } - diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 5579afce..0549a2c0 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -403,7 +403,14 @@ impl Slot { if lower { self.deadline.store(at, Ordering::Release); } - (lower, if lower { std::mem::take(&mut _inner.watchers) } else { Vec::new() }) + ( + lower, + if lower { + std::mem::take(&mut _inner.watchers) + } else { + Vec::new() + }, + ) }; let (lowered, watchers) = lowered; // Token-side waiters re-arm their timers for the earlier deadline. diff --git a/concurrency/src/system.rs b/concurrency/src/system.rs index 84cfb231..3e337bb2 100644 --- a/concurrency/src/system.rs +++ b/concurrency/src/system.rs @@ -921,7 +921,10 @@ impl Scope { .fetch_min(nanos, std::sync::atomic::Ordering::AcqRel); } pub(crate) fn cleanup_timeout(&self) -> Duration { - Duration::from_nanos(self.cleanup_timeout.load(std::sync::atomic::Ordering::Acquire)) + Duration::from_nanos( + self.cleanup_timeout + .load(std::sync::atomic::Ordering::Acquire), + ) } pub(crate) fn set_jobs(&self, jobs: std::sync::Weak) { *lock(&self.jobs) = Some(jobs); diff --git a/concurrency/tests/deferred_jobs.rs b/concurrency/tests/deferred_jobs.rs index 7c4646f0..092e5929 100644 --- a/concurrency/tests/deferred_jobs.rs +++ b/concurrency/tests/deferred_jobs.rs @@ -264,12 +264,19 @@ mod tasks_mode { type Result = (); } impl Handler for Owner { - async fn handle(&mut self, msg: StartOld, reply: PendingReply<'_, ()>, ctx: &Context) { + async fn handle( + &mut self, + msg: StartOld, + reply: PendingReply<'_, ()>, + ctx: &Context, + ) { let (open, gate) = oneshot::channel::<()>(); let job = ctx - .pipe_keyed("r", async move { gate.await.map(|()| 1).unwrap_or(0) }, |o| { - describe("old", o) - }) + .pipe_keyed( + "r", + async move { gate.await.map(|()| 1).unwrap_or(0) }, + |o| describe("old", o), + ) .unwrap(); *msg.0.lock().unwrap() = Some((job, open)); reply.reply(()); @@ -285,7 +292,8 @@ mod tasks_mode { while !msg.0.load(std::sync::atomic::Ordering::SeqCst) { std::thread::sleep(Duration::from_millis(1)); } - ctx.pipe_keyed("r", async { 2 }, |o| describe("new", o)).unwrap(); + ctx.pipe_keyed("r", async { 2 }, |o| describe("new", o)) + .unwrap(); } } @@ -309,7 +317,10 @@ mod tasks_mode { go.store(true, std::sync::atomic::Ordering::SeqCst); assert_eq!(entries(&actor, &log, 1).await, ["new:ok:2"]); sleep(Duration::from_millis(20)).await; - assert!(log.take().is_empty(), "the replaced job's terminal message was dispatched"); + assert!( + log.take().is_empty(), + "the replaced job's terminal message was dispatched" + ); stop(actor).await; } @@ -319,8 +330,10 @@ mod tasks_mode { } impl Handler for Owner { async fn handle(&mut self, _: Bomb, reply: PendingReply<'_, ()>, ctx: &Context) { - ctx.pipe(async {}, |_: JobOutcome<()>| -> Done { panic!("done boom") }) - .unwrap(); + ctx.pipe(async {}, |_: JobOutcome<()>| -> Done { + panic!("done boom") + }) + .unwrap(); reply.reply(()); } } @@ -330,11 +343,21 @@ mod tasks_mode { #[spawned_macros::sim_test] async fn a_panicking_done_runs_stopped_and_cancels_jobs() { let (actor, log) = start(); - actor.request(Start::Gated("survivor", None)).await.unwrap().unwrap(); + actor + .request(Start::Gated("survivor", None)) + .await + .unwrap() + .unwrap(); actor.request(Bomb).await.unwrap(); let reason = actor.wait_exit().await; - assert!(matches!(reason, spawned_concurrency::ExitReason::Panic(_)), "{reason:?}"); - assert!(log.take().contains(&"stopped".to_string()), "stopped() did not run"); + assert!( + matches!(reason, spawned_concurrency::ExitReason::Panic(_)), + "{reason:?}" + ); + assert!( + log.take().contains(&"stopped".to_string()), + "stopped() did not run" + ); actor.join().await; } diff --git a/concurrency/tests/deferred_replies.rs b/concurrency/tests/deferred_replies.rs index 368b7c87..258f293e 100644 --- a/concurrency/tests/deferred_replies.rs +++ b/concurrency/tests/deferred_replies.rs @@ -330,7 +330,10 @@ mod tasks_mode { std::thread::sleep(Duration::from_millis(5)); } let result = caller.join().unwrap(); - assert!(matches!(result, Err(ActorError::ActorStopped)), "{result:?}"); + assert!( + matches!(result, Err(ActorError::ActorStopped)), + "{result:?}" + ); drop(actor); } diff --git a/concurrency/tests/registry_turn_loom.rs b/concurrency/tests/registry_turn_loom.rs index 6d363753..74687a9a 100644 --- a/concurrency/tests/registry_turn_loom.rs +++ b/concurrency/tests/registry_turn_loom.rs @@ -35,11 +35,19 @@ impl Outcomes { } fn one(&self, item: u32, outcome: Outcome) { let mut seen = self.0.lock().unwrap(); - assert!(seen.iter().all(|(i, _)| *i != item), "item {item} answered twice"); + assert!( + seen.iter().all(|(i, _)| *i != item), + "item {item} answered twice" + ); seen.push((item, outcome)); } fn of(&self, item: u32) -> Option { - self.0.lock().unwrap().iter().find(|(i, _)| *i == item).map(|(_, o)| *o) + self.0 + .lock() + .unwrap() + .iter() + .find(|(i, _)| *i == item) + .map(|(_, o)| *o) } } @@ -68,10 +76,18 @@ fn drop_vs_turn_end() { dropper.join().unwrap(); let got = outcomes.of(1).expect("a dropped obligation was lost"); if committed { - let expected = if panicked { Outcome::Stopped } else { Outcome::Dropped }; + let expected = if panicked { + Outcome::Stopped + } else { + Outcome::Dropped + }; assert_eq!(got, expected); } else { - assert_eq!(got, Outcome::Dropped, "a drop after the turn answers at once"); + assert_eq!( + got, + Outcome::Dropped, + "a drop after the turn answers at once" + ); } }); } diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index 11dd2115..c099c7cd 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -134,7 +134,10 @@ fn reply_vs_shortening() { match delivery { Delivery::Delivered => { assert_eq!(state(slot.word()), slot::REPLIED); - assert!(!lowered, "a shortening was applied but not seen by the reply"); + assert!( + !lowered, + "a shortening was applied but not seen by the reply" + ); assert_eq!(slot.deadline(), 100); } Delivery::Expired => { diff --git a/testing/src/inbox.rs b/testing/src/inbox.rs index db03a772..3d08ab7f 100644 --- a/testing/src/inbox.rs +++ b/testing/src/inbox.rs @@ -1,7 +1,7 @@ //! Shared message log behind mocks and probes: blocking and async waiters. use spawned_concurrency::message::Message; -use std::collections::HashMap; use std::any::{Any, TypeId}; +use std::collections::HashMap; use std::panic::Location; use std::sync::{Arc, Condvar, Mutex, MutexGuard}; use std::time::{Duration, Instant}; @@ -134,7 +134,10 @@ impl Default for AsyncSignal { } impl Inbox { - pub(crate) fn store_deferred(&self, token: spawned_concurrency::reply::ReplyTo) { + pub(crate) fn store_deferred( + &self, + token: spawned_concurrency::reply::ReplyTo, + ) { self.deferred .lock() .unwrap_or_else(|p| p.into_inner()) @@ -142,7 +145,9 @@ impl Inbox { .or_default() .push(Box::new(token)); } - pub(crate) fn take_deferred(&self) -> Vec> { + pub(crate) fn take_deferred( + &self, + ) -> Vec> { let tokens = self .deferred .lock() diff --git a/testing/src/rules.rs b/testing/src/rules.rs index 667e29b6..4bd1d934 100644 --- a/testing/src/rules.rs +++ b/testing/src/rules.rs @@ -45,7 +45,9 @@ pub(crate) enum Plan { /// `Mock::defer`: take the reply obligation and store it for the test. Defer, /// `Mock::discard`: wait `delay`, then drop the reply obligation. - Discard { delay: Option }, + Discard { + delay: Option, + }, } #[derive(Default)] diff --git a/testing/src/shared.rs b/testing/src/shared.rs index 6f936921..0f548a3d 100644 --- a/testing/src/shared.rs +++ b/testing/src/shared.rs @@ -128,7 +128,9 @@ macro_rules! mock_ref_common { /// Take the reply tokens of the `M` requests the mock deferred /// (`Mock::defer`), oldest first: reply later, drop them /// (`ReplyDropped`), or keep them unanswered. - pub fn deferred(&self) -> Vec> { + pub fn deferred( + &self, + ) -> Vec> { self.inbox.take_deferred::() } From a1ed0fc710905a7a476caa514083075cbd39b399 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 01:04:37 +0000 Subject: [PATCH 054/208] test(jobs): job core extracted for Loom; model admission rollback, completed-key replacement, emit vs terminal/cancel, native completion vs cleanup, weak-upgrade destruction fix(jobs): drop the key map and compaction upgrades outside the job set locks; upgrade the scope job set outside its lock --- .github/workflows/verification.yml | 2 +- CHANGELOG.md | 9 +- concurrency/src/reply/job_core.rs | 339 ++++++++++++++++++++ concurrency/src/reply/jobs.rs | 217 ++++--------- concurrency/src/reply/mod.rs | 5 +- concurrency/src/reply/turn.rs | 11 +- concurrency/src/system.rs | 6 +- concurrency/tests/job_core_loom.rs | 477 +++++++++++++++++++++++++++++ docs/TESTING.md | 18 ++ 9 files changed, 910 insertions(+), 174 deletions(-) create mode 100644 concurrency/src/reply/job_core.rs create mode 100644 concurrency/tests/job_core_loom.rs diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml index 3d13c8bd..03e0c71c 100644 --- a/.github/workflows/verification.yml +++ b/.github/workflows/verification.yml @@ -103,7 +103,7 @@ jobs: - name: Model-check the reply slot's packed-word protocol env: RUSTFLAGS: --cfg spawned_scheduler_loom - run: cargo test --locked --release -p spawned-concurrency --no-default-features --test reply_slot_loom --test registry_turn_loom + run: cargo test --locked --release -p spawned-concurrency --no-default-features --test reply_slot_loom --test registry_turn_loom --test job_core_loom # Data races in the runtime and its dependencies. The crates forbid # unsafe code, so ThreadSanitizer is the sanitizer that matters here. diff --git a/CHANGELOG.md b/CHANGELOG.md index 518d93aa..530edf14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,13 @@ (ROADMAP rank 11), implementing [the design](docs/design/deferred-replies.md). Every request now has a reply slot: one packed atomic word (state, token, registry and running-job count) - whose transitions are checked with Loom (`reply_slot_loom`, 15 concurrent - pairs, 5 mutation controls). `Handler::handle` takes a `PendingReply<'_, R>` + whose transitions are checked with Loom (`reply_slot_loom`, 16 models), as + are the owner's turn-phase drop commit (`registry_turn_loom`) and the job + core (`job_core_loom`: admission rollback, completed-key replacement, emit + versus terminal notification and cancel, native completion versus cleanup + join, final destruction after a `Weak` upgrade), each with planted-bug + controls. The job model found two destructions under the job set's locks + (key-map clear at owner exit, weak-list compaction), now fixed. `Handler::handle` takes a `PendingReply<'_, R>` and returns `()`; `#[actor]` generates it, and `#[deferred_handler]` methods receive the obligation and may `defer()` it into a storable `ReplyTo` (bounded by `Actor::MAX_OUTSTANDING`, `Overloaded` at the bound). Dropped diff --git a/concurrency/src/reply/job_core.rs b/concurrency/src/reply/job_core.rs new file mode 100644 index 00000000..19eb65e0 --- /dev/null +++ b/concurrency/src/reply/job_core.rs @@ -0,0 +1,339 @@ +//! The racy core of pipe-to-self jobs (docs/design/deferred-replies.md §8, +//! §9; `verification/tla/DeferredJobs.tla`): one job's cancellation flag, +//! one-shot terminal notification and emit credits, and the owner's job set +//! (key map, running count, native budget). +//! +//! Shared with the Loom model (`tests/job_core_loom.rs`), which supplies the +//! synchronization types through `super::sync`, including a model of +//! `Arc`/`Weak` that observes the final destruction of a job. The protocol +//! never runs user code under its locks: `enqueue` closures only push an +//! envelope, and every strong reference a lock section takes (a replaced key, +//! an upgraded `Weak`) is dropped after the lock is released. + +use super::sync::{ + AtomicBool, AtomicUsize, Condvar, JobArc as Arc, JobWeak as Weak, Mutex, MutexGuard, Ordering, +}; +use std::collections::HashMap; +use std::hash::Hash; + +/// The job is cancelled or already notified: an emit was not enqueued. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("the job is gone (cancelled, replaced or past its terminal notification)")] +pub struct JobGone; + +/// The terminal notification: a one-shot, taken by the result, a caught +/// panic or the job deadline, and suppressed by cancellation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Notification { + Pending, + Notified, + Suppressed, +} + +struct Inner { + cancelled: bool, + notification: Notification, + running: bool, + emit_credits: u16, +} + +impl Inner { + fn open(&self) -> bool { + !self.cancelled && self.notification == Notification::Pending + } +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(|p| p.into_inner()) +} + +/// One job's flag, notification and emit credits. +pub(crate) struct JobCore { + inner: Mutex, + emitters: Condvar, + cancelled_fast: AtomicBool, +} + +impl JobCore { + pub(crate) fn new(emit_credits: u16) -> Self { + Self { + inner: Mutex::new(Inner { + cancelled: false, + notification: Notification::Pending, + running: true, + emit_credits, + }), + emitters: Condvar::new(), + cancelled_fast: AtomicBool::new(false), + } + } + + pub(crate) fn is_cancelled(&self) -> bool { + self.cancelled_fast.load(Ordering::Acquire) + } + + /// Set the cancellation flag: suppresses a pending notification and + /// wakes blocked emitters. Never undoes effects. The caller wakes async + /// emitters afterwards. + pub(crate) fn cancel(&self) { + { + let mut inner = lock(&self.inner); + inner.cancelled = true; + if inner.notification == Notification::Pending { + inner.notification = Notification::Suppressed; + } + // Set under the lock, so a dispatch that sees an enqueued + // message's job as not cancelled ran before this cancel. + self.cancelled_fast.store(true, Ordering::Release); + } + self.emitters.notify_all(); + } + + /// Take the terminal notification and run `enqueue` under the job's + /// lock. Returns whether it was taken; `enqueue` is not called when the + /// job was cancelled or already notified. Whatever `enqueue` hands back + /// (an envelope the mailbox refused) is returned to be dropped by the + /// caller after the lock is released. When taken, the caller wakes async + /// emitters. + pub(crate) fn notify(&self, enqueue: impl FnOnce() -> Option) -> (bool, Option) { + let unsent = { + let mut inner = lock(&self.inner); + if !inner.open() { + return (false, None); + } + inner.notification = Notification::Notified; + enqueue() + }; + self.emitters.notify_all(); + (true, unsent) + } + + /// Whether the terminal notification can still be taken (the job + /// deadline is still due). + pub(crate) fn is_pending(&self) -> bool { + lock(&self.inner).open() + } + + #[allow(dead_code)] // used by the Loom model + pub(crate) fn notification(&self) -> Notification { + lock(&self.inner).notification + } + + #[allow(dead_code)] // used by the Loom model + pub(crate) fn emit_credits(&self) -> u16 { + lock(&self.inner).emit_credits + } + + /// Execution ended (physically). + pub(crate) fn finished(&self) { + lock(&self.inner).running = false; + } + + pub(crate) fn is_running(&self) -> bool { + lock(&self.inner).running + } + + /// Emit step 1 (blocking): take an emit credit, waiting while all are in + /// flight. Fails with `JobGone` once the job is cancelled or notified. + pub(crate) fn acquire_blocking(&self) -> Result<(), JobGone> { + let mut inner = lock(&self.inner); + loop { + if !inner.open() { + return Err(JobGone); + } + if inner.emit_credits > 0 { + inner.emit_credits -= 1; + return Ok(()); + } + inner = self.emitters.wait(inner).unwrap_or_else(|p| p.into_inner()); + } + } + + /// Emit step 1 (async): take a credit, or call `register` (store the + /// waker) under the job's lock and return `None`. Registration under the + /// lock orders it against the credit return and the cancel, which wake + /// the registered wakers after their update. + #[cfg_attr(not(feature = "async"), allow(dead_code))] + pub(crate) fn try_acquire(&self, register: impl FnOnce()) -> Option> { + let mut inner = lock(&self.inner); + if !inner.open() { + return Some(Err(JobGone)); + } + if inner.emit_credits > 0 { + inner.emit_credits -= 1; + return Some(Ok(())); + } + register(); + None + } + + /// Emit step 2: under the job's lock, enqueue unless the job is + /// cancelled or notified (then the credit returns here). An envelope the + /// mailbox refused comes back to be dropped after the lock is released; + /// its credit guard returns the credit then. + pub(crate) fn enqueue_emit( + &self, + enqueue: impl FnOnce() -> Result<(), X>, + ) -> Result<(), (JobGone, Option)> { + let mut inner = lock(&self.inner); + if !inner.open() { + inner.emit_credits += 1; + drop(inner); + self.emitters.notify_all(); + return Err((JobGone, None)); + } + enqueue().map_err(|unsent| (JobGone, Some(unsent))) + } + + /// Emit step 3: a credit comes back when its message is dispatched, + /// fenced or dropped with the mailbox. The caller wakes async emitters. + pub(crate) fn return_emit_credit(&self) { + lock(&self.inner).emit_credits += 1; + self.emitters.notify_all(); + } +} + +/// What the job set needs from a job. +pub(crate) trait Job { + fn cancel(&self); + fn is_running(&self) -> bool; + fn finished(&self); +} + +/// Compact the weak job list when it reaches this length (and every power of +/// two above it). +const COMPACT_AT: usize = 32; + +/// An owner instance's jobs: key map, running count, native budget. +pub(crate) struct JobSetCore { + jobs: Mutex>>, + keys: Mutex>>, + running: Mutex, + idle: Condvar, + native: AtomicUsize, + max_native: usize, +} + +impl JobSetCore { + pub(crate) fn new(max_native: usize) -> Self { + Self { + jobs: Mutex::new(Vec::new()), + keys: Mutex::new(HashMap::new()), + running: Mutex::new(0), + idle: Condvar::new(), + native: AtomicUsize::new(0), + max_native, + } + } + + /// Reserve a native slot (`MAX_NATIVE_JOBS`), rejecting at the bound. + pub(crate) fn reserve_native(&self) -> bool { + self.native + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |n| { + (n < self.max_native).then_some(n + 1) + }) + .is_ok() + } + + pub(crate) fn release_native(&self) { + let previous = self.native.fetch_sub(1, Ordering::AcqRel); + debug_assert!(previous > 0, "native job budget underflow"); + } + + #[allow(dead_code)] // used by the Loom model + pub(crate) fn native(&self) -> usize { + self.native.load(Ordering::Acquire) + } + + /// Admit a job: replace (cancel) the previous one with the same key, + /// whatever its notification state, and count it as running. The + /// replaced job and the references compaction upgraded are dropped after + /// the locks are released. + pub(crate) fn admit(&self, state: &Arc, key: Option) { + let previous = key.and_then(|key| lock(&self.keys).insert(key, state.clone())); + if let Some(previous) = &previous { + previous.cancel(); + } + let mut upgraded: Vec> = Vec::new(); + { + let mut jobs = lock(&self.jobs); + if jobs.len() >= COMPACT_AT && jobs.len().is_power_of_two() { + jobs.retain(|job| match job.upgrade() { + Some(job) => { + let keep = job.is_running(); + upgraded.push(job); + keep + } + None => false, + }); + } + jobs.push(Arc::downgrade(state)); + } + *lock(&self.running) += 1; + drop(upgraded); + drop(previous); + } + + /// A job's execution ended. Returns whether the set became idle (the + /// caller wakes async joiners). + pub(crate) fn ended(&self, state: &J) -> bool { + state.finished(); + // The key keeps its finished job until replaced: a replacement must + // still cancel it, so its queued terminal message and emits are + // fenced (`NoStaleKeyed`). The map holds one job per key. + let idle = { + let mut running = lock(&self.running); + *running -= 1; + *running == 0 + }; + if idle { + self.idle.notify_all(); + } + idle + } + + pub(crate) fn running(&self) -> usize { + *lock(&self.running) + } + + /// Owner exit: cancel every job (flag set, blocked emitters woken). The + /// key map's references and the upgraded jobs drop outside the locks. + pub(crate) fn cancel_all(&self) { + let jobs = std::mem::take(&mut *lock(&self.jobs)); + let keys = std::mem::take(&mut *lock(&self.keys)); + for job in jobs.iter().filter_map(Weak::upgrade) { + job.cancel(); + } + drop(keys); + } + + /// Wait until no job is running. `wait` parks on the idle condition + /// variable and returns `Err` with the guard to give up (the cleanup + /// deadline passed). Returns how many jobs are still running. + pub(crate) fn join_with<'a>( + &'a self, + mut wait: impl FnMut( + &'a Condvar, + MutexGuard<'a, usize>, + ) -> Result, MutexGuard<'a, usize>>, + ) -> usize { + let mut running = lock(&self.running); + while *running > 0 { + match wait(&self.idle, running) { + Ok(guard) => running = guard, + Err(guard) => return *guard, + } + } + 0 + } + + /// Take and release the set's locks: the Loom model calls it from a + /// job's final destruction, which deadlocks if that destruction runs + /// under one of them. + #[allow(dead_code)] // used by the Loom model + pub(crate) fn probe_locks(&self) { + drop(lock(&self.jobs)); + drop(lock(&self.keys)); + drop(lock(&self.running)); + } +} diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 338202a3..5c700b91 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -21,9 +21,7 @@ use super::deadline::Expirable; use super::registry::Registry; use super::slot::CreditPool; use crate::ActorId; -use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::{Arc, Condvar, Mutex, MutexGuard, Weak}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; /// Default emit credits per job (`JobOptions::emit_credits`). pub const DEFAULT_EMIT_CREDITS: u16 = 16; @@ -155,31 +153,13 @@ impl PipeReplyError { } } -/// The job is cancelled or already notified: an emit was not enqueued. -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] -#[error("the job is gone (cancelled, replaced or past its terminal notification)")] -pub struct JobGone; +pub use super::job_core::JobGone; +use super::job_core::{Job, JobCore, JobSetCore}; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Notification { - Pending, - Notified, - Suppressed, -} - -struct Inner { - cancelled: bool, - notification: Notification, - running: bool, - emit_credits: u16, -} - -/// One job's shared state. +/// One job's shared state: the core protocol plus the async wake-ups. pub(crate) struct JobState { owner: ActorId, - inner: Mutex, - emitters: Condvar, - cancelled_fast: AtomicBool, + core: JobCore, /// Async cancellation for `tasks` jobs (drops the work future). #[cfg(feature = "async")] cancel: spawned_rt::tasks::CancellationToken, @@ -196,14 +176,7 @@ impl JobState { pub(crate) fn new(owner: ActorId, emit_credits: u16) -> Arc { Arc::new(Self { owner, - inner: Mutex::new(Inner { - cancelled: false, - notification: Notification::Pending, - running: true, - emit_credits, - }), - emitters: Condvar::new(), - cancelled_fast: AtomicBool::new(false), + core: JobCore::new(emit_credits), #[cfg(feature = "async")] cancel: spawned_rt::tasks::CancellationToken::new(), #[cfg(feature = "async")] @@ -216,7 +189,7 @@ impl JobState { } pub(crate) fn is_cancelled(&self) -> bool { - self.cancelled_fast.load(Ordering::Acquire) + self.core.is_cancelled() } #[cfg(feature = "async")] @@ -227,15 +200,7 @@ impl JobState { /// Set the cancellation flag: suppresses a pending notification and /// wakes blocked emitters. Never undoes effects. pub(crate) fn cancel(&self) { - { - let mut inner = lock(&self.inner); - inner.cancelled = true; - if inner.notification == Notification::Pending { - inner.notification = Notification::Suppressed; - } - } - self.cancelled_fast.store(true, Ordering::Release); - self.emitters.notify_all(); + self.core.cancel(); #[cfg(feature = "async")] { self.cancel.cancel(); @@ -252,48 +217,23 @@ impl JobState { } /// Take the terminal notification and run `enqueue` under the job's - /// lock. Returns whether it was taken; `enqueue` is not called when the - /// job was cancelled or already notified. Whatever `enqueue` hands back - /// (an envelope the mailbox refused) is returned to be dropped by the - /// caller after the lock is released. + /// lock (see [`JobCore::notify`]). pub(crate) fn notify(&self, enqueue: impl FnOnce() -> Option) -> (bool, Option) { - let unsent = { - let mut inner = lock(&self.inner); - if inner.cancelled || inner.notification != Notification::Pending { - return (false, None); - } - inner.notification = Notification::Notified; - enqueue() - }; - self.emitters.notify_all(); + let (taken, unsent) = self.core.notify(enqueue); #[cfg(feature = "async")] - self.wake_emitters(); - (true, unsent) - } - - /// Execution ended (physically). - pub(crate) fn finished(&self) { - lock(&self.inner).running = false; + if taken { + self.wake_emitters(); + } + (taken, unsent) } pub(crate) fn is_running(&self) -> bool { - lock(&self.inner).running + self.core.is_running() } - /// Emit step 1 (blocking): take an emit credit, waiting while all are in - /// flight. Fails with `JobGone` once the job is cancelled or notified. + /// Emit step 1 (blocking). pub(crate) fn acquire_blocking(&self) -> Result<(), JobGone> { - let mut inner = lock(&self.inner); - loop { - if inner.cancelled || inner.notification != Notification::Pending { - return Err(JobGone); - } - if inner.emit_credits > 0 { - inner.emit_credits -= 1; - return Ok(()); - } - inner = self.emitters.wait(inner).unwrap_or_else(|p| p.into_inner()); - } + self.core.acquire_blocking() } /// Emit step 1 (async). @@ -302,46 +242,47 @@ impl JobState { &self, cx: &mut std::task::Context<'_>, ) -> std::task::Poll> { - let mut inner = lock(&self.inner); - if inner.cancelled || inner.notification != Notification::Pending { - return std::task::Poll::Ready(Err(JobGone)); - } - if inner.emit_credits > 0 { - inner.emit_credits -= 1; - return std::task::Poll::Ready(Ok(())); + let acquired = self.core.try_acquire(|| { + let mut wakers = lock(&self.emit_wakers); + if !wakers.iter().any(|w| w.will_wake(cx.waker())) { + wakers.push(cx.waker().clone()); + } + }); + match acquired { + Some(result) => std::task::Poll::Ready(result), + None => std::task::Poll::Pending, } - lock(&self.emit_wakers).push(cx.waker().clone()); - std::task::Poll::Pending } - /// Emit step 2: under the job's lock, enqueue unless the job is - /// cancelled or notified (then the credit returns here). An envelope the - /// mailbox refused comes back to be dropped after the lock is released; - /// its credit guard returns the credit then. + /// Emit step 2 (see [`JobCore::enqueue_emit`]). pub(crate) fn enqueue_emit( &self, enqueue: impl FnOnce() -> Result<(), X>, ) -> Result<(), (JobGone, Option)> { - let mut inner = lock(&self.inner); - if inner.cancelled || inner.notification != Notification::Pending { - inner.emit_credits += 1; - drop(inner); - self.emitters.notify_all(); - return Err((JobGone, None)); - } - enqueue().map_err(|unsent| (JobGone, Some(unsent))) + self.core.enqueue_emit(enqueue) } /// Emit step 3: a credit comes back when its message is dispatched, /// fenced or dropped with the mailbox. pub(crate) fn return_emit_credit(&self) { - lock(&self.inner).emit_credits += 1; - self.emitters.notify_all(); + self.core.return_emit_credit(); #[cfg(feature = "async")] self.wake_emitters(); } } +impl Job for JobState { + fn cancel(&self) { + JobState::cancel(self); + } + fn is_running(&self) -> bool { + self.core.is_running() + } + fn finished(&self) { + self.core.finished(); + } +} + /// A handle to a spawned job. Dropping it leaves the job owned by the /// actor's job set (it is not detached). #[derive(Clone)] @@ -384,12 +325,7 @@ impl JobHandle { /// An owner instance's jobs: key map, running count, native budget. pub(crate) struct JobSet { owner: ActorId, - jobs: Mutex>>, - keys: Mutex>>, - running: Mutex, - idle: Condvar, - native: AtomicUsize, - max_native: usize, + core: JobSetCore, #[cfg(feature = "async")] idle_async: Mutex>, } @@ -398,12 +334,7 @@ impl JobSet { pub(crate) fn new(owner: ActorId, max_native: usize) -> Self { Self { owner, - jobs: Mutex::new(Vec::new()), - keys: Mutex::new(HashMap::new()), - running: Mutex::new(0), - idle: Condvar::new(), - native: AtomicUsize::new(0), - max_native, + core: JobSetCore::new(max_native), #[cfg(feature = "async")] idle_async: Mutex::new(Vec::new()), } @@ -415,49 +346,22 @@ impl JobSet { /// Reserve a native slot (`MAX_NATIVE_JOBS`), rejecting at the bound. pub(crate) fn reserve_native(&self) -> bool { - self.native - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |n| { - (n < self.max_native).then_some(n + 1) - }) - .is_ok() + self.core.reserve_native() } pub(crate) fn release_native(&self) { - self.native.fetch_sub(1, Ordering::AcqRel); + self.core.release_native(); } /// Admit a job: replace (cancel) the previous one with the same key and /// count it as running. pub(crate) fn admit(&self, state: &Arc, key: Option) { - if let Some(key) = key { - let previous = lock(&self.keys).insert(key, state.clone()); - if let Some(previous) = previous { - previous.cancel(); - } - } - { - let mut jobs = lock(&self.jobs); - if jobs.len() >= 32 && jobs.len().is_power_of_two() { - jobs.retain(|j| j.upgrade().is_some_and(|j| j.is_running())); - } - jobs.push(Arc::downgrade(state)); - } - *lock(&self.running) += 1; + self.core.admit(state, key); } /// A job's execution ended. pub(crate) fn ended(&self, state: &Arc) { - state.finished(); - // The key keeps its finished job until replaced: a replacement must - // still cancel it, so its queued terminal message and emits are - // fenced (`NoStaleKeyed`). The map holds one job per key. - let idle = { - let mut running = lock(&self.running); - *running -= 1; - *running == 0 - }; - if idle { - self.idle.notify_all(); + if self.core.ended(state) { #[cfg(feature = "async")] for waker in std::mem::take(&mut *lock(&self.idle_async)) { waker.wake(); @@ -466,34 +370,27 @@ impl JobSet { } pub(crate) fn running(&self) -> usize { - *lock(&self.running) + self.core.running() } /// Owner exit: cancel every job (flag set, blocked emitters woken). pub(crate) fn cancel_all(&self) { - let jobs = std::mem::take(&mut *lock(&self.jobs)); - lock(&self.keys).clear(); - for job in jobs.iter().filter_map(Weak::upgrade) { - job.cancel(); - } + self.core.cancel_all(); } /// Join the cancelled jobs, bounded by `deadline`; returns how many are /// still running (unfinished at stop). pub(crate) fn join_blocking(&self, deadline: std::time::Instant) -> usize { - let mut running = lock(&self.running); - while *running > 0 { + self.core.join_with(|idle, running| { let now = std::time::Instant::now(); if now >= deadline { - break; + return Err(running); } - running = self - .idle + Ok(idle .wait_timeout(running, deadline - now) .unwrap_or_else(|p| p.into_inner()) - .0; - } - *running + .0) + }) } /// Async join, bounded by `timeout` on the runtime's clock. @@ -558,11 +455,7 @@ pub(crate) struct JobDeadline { impl Expirable for JobDeadline { fn pending_deadline(&self) -> Option { - let pending = { - let inner = lock(&self.state.inner); - !inner.cancelled && inner.notification == Notification::Pending - }; - pending.then_some(self.at) + self.state.core.is_pending().then_some(self.at) } fn expire(&self, _now: u64) { let notify = lock(&self.notify).take(); diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 0ede7a20..fe201a78 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -7,6 +7,7 @@ //! gets exactly one terminal discharge. pub(crate) mod deadline; +pub(crate) mod job_core; pub mod jobs; mod pending; pub(crate) mod registry; @@ -29,8 +30,8 @@ use std::time::Duration; /// Production synchronization types for [`slot`]; the Loom model supplies its /// own (`tests/reply_slot_loom.rs`). mod sync { - pub(crate) use std::sync::atomic::{AtomicU64, AtomicU8, AtomicUsize, Ordering}; - pub(crate) use std::sync::{Arc, Condvar, Mutex, MutexGuard}; + pub(crate) use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, AtomicUsize, Ordering}; + pub(crate) use std::sync::{Arc, Arc as JobArc, Condvar, Mutex, MutexGuard, Weak as JobWeak}; pub(crate) use std::thread::yield_now; pub(crate) type SlotOwner = Option>; diff --git a/concurrency/src/reply/turn.rs b/concurrency/src/reply/turn.rs index 6b5509a9..5bac605e 100644 --- a/concurrency/src/reply/turn.rs +++ b/concurrency/src/reply/turn.rs @@ -117,11 +117,12 @@ impl TurnPhase { if current >= EXITING { return Vec::new(); } - match self - .phase - .0 - .compare_exchange(current, EXITING, Ordering::AcqRel, Ordering::Acquire) - { + match self.phase.0.compare_exchange( + current, + EXITING, + Ordering::AcqRel, + Ordering::Acquire, + ) { Ok(previous) if previous == IN_TURN_PENDING => return take(&self.drops), Ok(_) => return Vec::new(), Err(actual) => current = actual, diff --git a/concurrency/src/system.rs b/concurrency/src/system.rs index 3e337bb2..ccf0db63 100644 --- a/concurrency/src/system.rs +++ b/concurrency/src/system.rs @@ -931,8 +931,10 @@ impl Scope { } /// Jobs still running after the actor's death (job cleanup incomplete). fn unfinished_jobs(&self) -> usize { - lock(&self.jobs) - .as_ref() + // Upgrade outside the lock: the upgraded set may be the last + // reference, and its destruction must not run under it. + let jobs = lock(&self.jobs).clone(); + jobs.as_ref() .and_then(std::sync::Weak::upgrade) .map_or(0, |jobs| jobs.running()) } diff --git a/concurrency/tests/job_core_loom.rs b/concurrency/tests/job_core_loom.rs new file mode 100644 index 00000000..8139eb18 --- /dev/null +++ b/concurrency/tests/job_core_loom.rs @@ -0,0 +1,477 @@ +//! Loom model of the pipe-to-self job core (`src/reply/job_core.rs`, +//! docs/design/deferred-replies.md §8, §9, `verification/tla/DeferredJobs.tla`): +//! admission and its rollback, keyed replacement of a completed job, emits +//! racing the terminal notification and cancellation, native completion +//! racing the owner's cleanup join, and the final destruction of a job whose +//! last strong reference comes from a `Weak` upgrade. +//! +//! `Arc`/`Weak` for jobs are modelled below (Loom has no `Weak`): the strong +//! count is a Loom atomic and upgrade is the same CAS loop as `std`, so the +//! thread that drops the last strong reference is explored. A job's final +//! destruction takes the job set's locks, which deadlocks (and Loom reports +//! it) if the destruction runs under one of them. +#![cfg(spawned_scheduler_loom)] + +mod sync { + pub(crate) use loom::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; + pub(crate) use loom::sync::{Arc, Condvar, Mutex, MutexGuard}; + pub(crate) use loom::thread::yield_now; + pub(crate) type SlotOwner = Option<()>; + pub(crate) use super::model_arc::{JobArc, JobWeak}; + + /// Loom has no timed wait; the model never parks on a slot. + pub(crate) fn wait_ticks<'a, T>( + condvar: &Condvar, + guard: MutexGuard<'a, T>, + ticks: Option, + ) -> MutexGuard<'a, T> { + assert!(ticks.is_none(), "timed park is not modelled"); + condvar.wait(guard).unwrap_or_else(|p| p.into_inner()) + } +} + +/// `std::sync::{Arc, Weak}` for jobs, with a hook at the final strong drop. +mod model_arc { + use loom::sync::atomic::{AtomicUsize, Ordering}; + use std::any::Any; + + struct Shared { + strong: AtomicUsize, + value: T, + } + + pub(crate) struct JobArc(loom::sync::Arc>); + pub(crate) struct JobWeak(loom::sync::Arc>); + + impl JobArc { + pub(crate) fn new(value: T) -> Self { + Self(loom::sync::Arc::new(Shared { + strong: AtomicUsize::new(1), + value, + })) + } + pub(crate) fn downgrade(this: &Self) -> JobWeak { + JobWeak(this.0.clone()) + } + } + impl Clone for JobArc { + fn clone(&self) -> Self { + self.0.strong.fetch_add(1, Ordering::Relaxed); + Self(self.0.clone()) + } + } + impl std::ops::Deref for JobArc { + type Target = T; + fn deref(&self) -> &T { + &self.0.value + } + } + impl Drop for JobArc { + fn drop(&mut self) { + if self.0.strong.fetch_sub(1, Ordering::AcqRel) == 1 { + if let Some(job) = (&self.0.value as &dyn Any).downcast_ref::() { + job.destroyed(); + } + } + } + } + impl JobWeak { + /// `std::sync::Weak::upgrade`: increment unless already zero. + pub(crate) fn upgrade(&self) -> Option> { + let mut n = self.0.strong.load(Ordering::Relaxed); + loop { + if n == 0 { + return None; + } + match self + .0 + .strong + .compare_exchange(n, n + 1, Ordering::Acquire, Ordering::Relaxed) + { + Ok(_) => return Some(JobArc(self.0.clone())), + Err(now) => n = now, + } + } + } + } +} + +#[allow(dead_code)] +#[path = "../src/reply/job_core.rs"] +mod job_core; + +#[allow(dead_code)] +#[path = "../src/reply/slot.rs"] +mod slot; + +use job_core::{Job, JobCore, JobGone, JobSetCore, Notification}; +use loom::sync::atomic::{AtomicUsize, Ordering}; +use loom::sync::{Arc, Mutex}; +use model_arc::JobArc; +use slot::CreditPool; +use std::sync::atomic::Ordering as StdOrdering; + +type Set = JobSetCore; + +fn model(f: impl Fn() + Sync + Send + 'static) { + let mut builder = loom::model::Builder::new(); + builder.preemption_bound = Some(3); + builder.check(f); +} + +/// A job: the real core plus a destruction probe. +struct TestJob { + id: u32, + core: JobCore, + /// Taken at the final destruction, which then takes the set's locks. + set: Mutex>>, + destroyed: Arc, + gone: std::sync::atomic::AtomicBool, +} + +impl TestJob { + fn new( + id: u32, + credits: u16, + set: Option<&Arc>, + destroyed: &Arc, + ) -> JobArc { + JobArc::new(Self { + id, + core: JobCore::new(credits), + set: Mutex::new(set.cloned()), + destroyed: destroyed.clone(), + gone: std::sync::atomic::AtomicBool::new(false), + }) + } + fn destroyed(&self) { + assert!( + !self.gone.swap(true, StdOrdering::AcqRel), + "job {} destroyed twice", + self.id + ); + self.destroyed.fetch_add(1, Ordering::AcqRel); + let set = self.set.lock().unwrap().take(); + if let Some(set) = set { + set.probe_locks(); + } + } +} + +impl Job for TestJob { + fn cancel(&self) { + self.core.cancel(); + } + fn is_running(&self) -> bool { + self.core.is_running() + } + fn finished(&self) { + self.core.finished(); + } +} + +/// Messages a job enqueues to its owner. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Msg { + Emit(u32, u32), + Terminal(u32), +} + +#[derive(Default)] +struct Mailbox(Mutex>); +impl Mailbox { + fn push(&self, msg: Msg) { + self.0.lock().unwrap().push(msg); + } + fn take(&self) -> Vec { + std::mem::take(&mut *self.0.lock().unwrap()) + } +} + +/// Admission at the bound: a blocking job reserves a credit then a native +/// slot, and rolls both back when its thread cannot be spawned, racing a +/// running job that ends (releasing its native slot and credit) and a second +/// admission. The bounds are never exceeded and nothing leaks. +#[test] +fn admission_rollback_vs_job_end() { + model(|| { + // Two credits: the running job holds one, so both admissions can + // reach the native check at once. + let pool = Arc::new(CreditPool::new(2)); + let set = Arc::new(Set::new(1)); + let holding = Arc::new(AtomicUsize::new(0)); + let destroyed = Arc::new(AtomicUsize::new(0)); + // A running job holds the credit and the native slot. + assert!(pool.reserve(1) && set.reserve_native()); + let running = TestJob::new(0, 1, None, &destroyed); + set.admit(&running, None); + holding.fetch_add(1, Ordering::AcqRel); + + let ender = { + let (pool, set, holding) = (pool.clone(), set.clone(), holding.clone()); + loom::thread::spawn(move || { + holding.fetch_sub(1, Ordering::AcqRel); + set.release_native(); + pool.release(1); + set.ended(&running); + }) + }; + // Admission whose spawn fails: roll back what was reserved. + let admitter = { + let (pool, set, holding) = (pool.clone(), set.clone(), holding.clone()); + loom::thread::spawn(move || { + if pool.reserve(1) { + if set.reserve_native() { + assert_eq!( + holding.fetch_add(1, Ordering::AcqRel), + 0, + "native bound exceeded" + ); + holding.fetch_sub(1, Ordering::AcqRel); + set.release_native(); + } + pool.release(1); + } + }) + }; + // A second admission on this thread: admitted, or rolled back. + let mut second = None; + if pool.reserve(1) { + if set.reserve_native() { + assert_eq!( + holding.fetch_add(1, Ordering::AcqRel), + 0, + "native bound exceeded" + ); + let job = TestJob::new(1, 1, None, &destroyed); + set.admit(&job, None); + second = Some(job); + } else { + pool.release(1); + } + } + ender.join().unwrap(); + admitter.join().unwrap(); + if let Some(job) = second { + holding.fetch_sub(1, Ordering::AcqRel); + set.release_native(); + pool.release(1); + set.ended(&job); + } + assert_eq!(set.running(), 0); + assert_eq!(set.native(), 0, "a native slot leaked"); + assert_eq!(pool.used(), 0, "a credit leaked"); + }); +} + +/// A keyed job completes (terminal message enqueued, execution ended) while +/// the owner replaces its key: the replaced job is cancelled whatever its +/// notification state, so its terminal message is fenced at dispatch. +#[test] +fn completed_key_replacement() { + model(|| { + let set = Arc::new(Set::new(1)); + let mailbox = Arc::new(Mailbox::default()); + let destroyed = Arc::new(AtomicUsize::new(0)); + let old = TestJob::new(0, 1, None, &destroyed); + set.admit(&old, Some(7)); + let worker = { + let (set, mailbox, old) = (set.clone(), mailbox.clone(), old.clone()); + loom::thread::spawn(move || { + old.core.notify(|| { + mailbox.push(Msg::Terminal(0)); + None::<()> + }); + set.ended(&old); + }) + }; + let new = TestJob::new(1, 1, None, &destroyed); + set.admit(&new, Some(7)); + assert!( + old.core.is_cancelled(), + "the replaced job was not cancelled" + ); + assert!(!new.core.is_cancelled()); + worker.join().unwrap(); + // Dispatch after the replacement: the old terminal message is fenced. + for msg in mailbox.take() { + assert_eq!(msg, Msg::Terminal(0)); + assert!( + old.core.is_cancelled(), + "a replaced job's terminal message was dispatched" + ); + } + assert_ne!(old.core.notification(), Notification::Pending); + set.cancel_all(); + }); +} + +/// An emitter (one credit, two emits) races the terminal notification and, +/// optionally, a cancel. No emit is enqueued after the terminal message, a +/// blocked emitter is always released, and every credit comes back. +#[test] +fn emit_vs_notification_and_cancel() { + for cancel in [false, true] { + model(move || { + let mailbox = Arc::new(Mailbox::default()); + let destroyed = Arc::new(AtomicUsize::new(0)); + let job = TestJob::new(0, 1, None, &destroyed); + let emitter = { + let (job, mailbox) = (job.clone(), mailbox.clone()); + loom::thread::spawn(move || -> Result<(), JobGone> { + for i in 0..2 { + job.core.acquire_blocking()?; + job.core + .enqueue_emit(|| { + mailbox.push(Msg::Emit(0, i)); + Ok::<(), ()>(()) + }) + .map_err(|(gone, _)| gone)?; + } + Ok(()) + }) + }; + let canceller = cancel.then(|| { + let job = job.clone(); + loom::thread::spawn(move || job.core.cancel()) + }); + let (taken, _) = job.core.notify(|| { + mailbox.push(Msg::Terminal(0)); + None::<()> + }); + if let Some(canceller) = canceller { + canceller.join().unwrap(); + } + // Dispatch: each emit's credit returns. + let mut terminal_seen = false; + for msg in mailbox.take() { + match msg { + Msg::Terminal(_) => terminal_seen = true, + Msg::Emit(..) => { + assert!( + !terminal_seen, + "an emit was enqueued after the terminal message" + ); + job.core.return_emit_credit(); + } + } + } + assert_eq!(terminal_seen, taken); + if !cancel { + assert!(taken, "nothing suppressed the notification"); + } + // The emitter stops at the terminal notification or the cancel + // (or after both emits); nothing is enqueued after the dispatch. + let _ = emitter.join().unwrap(); + for msg in mailbox.take() { + panic!("enqueued after dispatch: {msg:?}"); + } + assert_eq!(job.core.emit_credits(), 1, "an emit credit was lost"); + }); + } +} + +/// A native job completes (native slot released, terminal notification, +/// execution ended) while the owner exits: cancel every job, then join. +/// The join is always woken, and once it returns no job runs, the native +/// slot is free and the job's notification is decided. +#[test] +fn native_completion_vs_cleanup() { + model(|| { + let set = Arc::new(Set::new(1)); + let mailbox = Arc::new(Mailbox::default()); + let destroyed = Arc::new(AtomicUsize::new(0)); + assert!(set.reserve_native()); + let job = TestJob::new(0, 1, None, &destroyed); + set.admit(&job, None); + let native = { + let (set, mailbox, job) = (set.clone(), mailbox.clone(), job.clone()); + loom::thread::spawn(move || { + set.release_native(); + job.core.notify(|| { + mailbox.push(Msg::Terminal(0)); + None::<()> + }); + set.ended(&job); + }) + }; + set.cancel_all(); + let unfinished = set.join_with(|idle, running| Ok(idle.wait(running).unwrap())); + assert_eq!(unfinished, 0); + assert_eq!( + set.native(), + 0, + "the join returned before the native slot was released" + ); + assert!(!job.core.is_running()); + assert_ne!(job.core.notification(), Notification::Pending); + native.join().unwrap(); + }); +} + +/// The owner's cleanup upgrades its weak job list while the job thread drops +/// the last other strong reference: the job is destroyed exactly once, by +/// whichever thread drops last, and never under the set's locks. +#[test] +fn weak_upgrade_final_destruction_at_cancel_all() { + model(|| { + let set = Arc::new(Set::new(1)); + let destroyed = Arc::new(AtomicUsize::new(0)); + let job = TestJob::new(0, 1, Some(&set), &destroyed); + set.admit(&job, Some(1)); + let worker = { + let set = set.clone(); + loom::thread::spawn(move || { + set.ended(&job); + drop(job); + }) + }; + set.cancel_all(); + worker.join().unwrap(); + assert_eq!( + destroyed.load(Ordering::Acquire), + 1, + "the job was not destroyed exactly once" + ); + }); +} + +/// Compaction of the weak job list (at 32 entries) upgrades a finished job +/// whose last other reference is being dropped: if the upgraded reference is +/// the last, the destruction runs after the lock is released. +#[test] +fn weak_upgrade_final_destruction_at_admission() { + let mut builder = loom::model::Builder::new(); + builder.preemption_bound = Some(2); + builder.check(|| { + let set = Arc::new(Set::new(1)); + let destroyed = Arc::new(AtomicUsize::new(0)); + let filler = Arc::new(AtomicUsize::new(0)); + // 31 entries already dead, plus the racing job: 32 at admission. + for id in 0..31 { + let dead = TestJob::new(id, 1, None, &filler); + set.admit(&dead, None); + set.ended(&dead); + } + let job = TestJob::new(0, 1, Some(&set), &destroyed); + set.admit(&job, None); + let worker = { + let set = set.clone(); + loom::thread::spawn(move || { + set.ended(&job); + drop(job); + }) + }; + // Compacts the weak list, upgrading the racing job. + let next = TestJob::new(1, 1, None, &destroyed); + set.admit(&next, Some(1)); + worker.join().unwrap(); + assert_eq!( + destroyed.load(Ordering::Acquire), + 1, + "the job was not destroyed exactly once" + ); + set.ended(&next); + set.cancel_all(); + }); +} diff --git a/docs/TESTING.md b/docs/TESTING.md index f6b9b70e..685eb215 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -511,6 +511,24 @@ The custom cfg isolates instrumentation from Tokio's own Loom configuration. These models cover registration/cancellation races and reentrant callbacks, not the entire actor runtime. Do not run ordinary actor tests with this cfg. +The reply and job cores of deferred replies share their source with Loom +models that inject Loom's synchronization types (`super::sync`): + +```sh +RUSTFLAGS="--cfg spawned_scheduler_loom" cargo test --release -p spawned-concurrency \ + --no-default-features --test reply_slot_loom --test registry_turn_loom --test job_core_loom +``` + +`reply_slot_loom` covers the packed slot word (reply, expiry, shortening, +abandon, close, defer at the bound, linked jobs, parked receivers); +`registry_turn_loom` the in-turn drop commit against turn end, exit and death; +`job_core_loom` job admission and rollback at the credit and native bounds, +keyed replacement of a completed job, emits against the terminal notification +and cancel, native completion against the owner's cleanup join, and a job's +final destruction after a `Weak` upgrade (modelled `Arc`/`Weak`; the +destruction takes the job set's locks, so running it under one is a Loom +deadlock). Each model was checked against a planted bug. + Run sustained actor creation, traffic, monitoring, and shutdown: ```sh From 216a0a17cf3774482c079b3c9a6c121bcc0b04a7 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 01:16:05 +0000 Subject: [PATCH 055/208] verify(jobs): lifecycle-trace events for the job core, TraceJobs.tla and job trace validation with negative controls --- .github/workflows/verification.yml | 2 +- CHANGELOG.md | 6 +- concurrency/src/child_handle.rs | 4 + concurrency/src/lib.rs | 2 +- concurrency/src/lifecycle_trace.rs | 57 ++++- concurrency/src/reply/job_core.rs | 13 +- concurrency/src/reply/jobs.rs | 122 ++++++++-- concurrency/src/tasks/jobs.rs | 25 +- concurrency/src/threads/jobs.rs | 15 +- concurrency/tests/job_trace.rs | 338 ++++++++++++++++++++++++++++ verification/tla/README.md | 27 +++ verification/tla/TraceJobs.tla | 61 +++++ verification/tla/negative_checks.py | 42 ++++ verification/tla/trace_check.py | 47 +++- 14 files changed, 724 insertions(+), 37 deletions(-) create mode 100644 concurrency/tests/job_trace.rs create mode 100644 verification/tla/TraceJobs.tla diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml index 03e0c71c..e9ed2951 100644 --- a/.github/workflows/verification.yml +++ b/.github/workflows/verification.yml @@ -30,7 +30,7 @@ jobs: - name: Record supervisor lifecycle traces env: LIFECYCLE_TRACE_DIR: ${{ github.workspace }}/lifecycle-traces - run: cargo test --locked -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace + run: cargo test --locked -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace - name: Model-check (fast configuration) and validate traces run: nix run .#check-tla -- --traces lifecycle-traces --negative-controls # Which model actions, projected states and transitions the recorded diff --git a/CHANGELOG.md b/CHANGELOG.md index 530edf14..001be468 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,11 @@ versus terminal notification and cancel, native completion versus cleanup join, final destruction after a `Weak` upgrade), each with planted-bug controls. The job model found two destructions under the job set's locks - (key-map clear at owner exit, weak-list compaction), now fixed. `Handler::handle` takes a `PendingReply<'_, R>` + (key-map clear at owner exit, weak-list compaction), now fixed. With + `lifecycle-trace`, the job core records `DeferredJobs.tla` actions, and + `tests/job_trace.rs` traces (seven scenarios) are checked by TLC + (`TraceJobs.tla`, schema `jobs-v1`), with six edited traces as negative + controls. `Handler::handle` takes a `PendingReply<'_, R>` and returns `()`; `#[actor]` generates it, and `#[deferred_handler]` methods receive the obligation and may `defer()` it into a storable `ReplyTo` (bounded by `Actor::MAX_OUTSTANDING`, `Overloaded` at the bound). Dropped diff --git a/concurrency/src/child_handle.rs b/concurrency/src/child_handle.rs index 9d052d5e..06b5492a 100644 --- a/concurrency/src/child_handle.rs +++ b/concurrency/src/child_handle.rs @@ -15,6 +15,10 @@ static NEXT_ACTOR_ID: AtomicU64 = AtomicU64::new(1); pub struct ActorId(u64); impl ActorId { + #[cfg_attr(not(feature = "lifecycle-trace"), allow(dead_code))] + pub(crate) fn raw(self) -> u64 { + self.0 + } pub(crate) fn next() -> Self { Self(NEXT_ACTOR_ID.fetch_add(1, Ordering::Relaxed)) } diff --git a/concurrency/src/lib.rs b/concurrency/src/lib.rs index f3ce8471..4f6a0752 100644 --- a/concurrency/src/lib.rs +++ b/concurrency/src/lib.rs @@ -90,7 +90,7 @@ pub mod failpoints; pub mod child_handle; pub mod error; -#[cfg(feature = "async")] +#[cfg_attr(not(feature = "async"), allow(dead_code))] pub mod lifecycle_trace; pub mod link; pub mod message; diff --git a/concurrency/src/lifecycle_trace.rs b/concurrency/src/lifecycle_trace.rs index 418d5223..08b30d79 100644 --- a/concurrency/src/lifecycle_trace.rs +++ b/concurrency/src/lifecycle_trace.rs @@ -107,6 +107,41 @@ impl TreeEvent { } } +/// One pipe-to-self job action of `verification/tla/DeferredJobs.tla`, +/// recorded per owner instance. `job` is a process-wide job number +/// (normalized by the checker); `x` is the action's parameter: the message +/// kind (`emit`, `done`, `deadline`) and fence verdict for `Dispatch`, and +/// whether the terminal message was enqueued for `Finish`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct JobEvent { + pub action: &'static str, + pub job: u64, + pub x: &'static str, + pub keyed: bool, + pub deadline: bool, + pub fenced: bool, +} + +impl JobEvent { + pub(crate) fn new(action: &'static str, job: u64) -> Self { + Self { + action, + job, + x: "", + keyed: false, + deadline: false, + fenced: false, + } + } + /// JSON object understood by `verification/tla/trace_check.py`. + pub fn to_json(&self) -> String { + format!( + r#"{{"a":"{}","j":{},"x":"{}","keyed":{},"dl":{},"fenced":{}}}"#, + self.action, self.job, self.x, self.keyed, self.deadline, self.fenced + ) + } +} + pub(crate) fn flag(value: bool) -> &'static str { if value { "true" @@ -155,6 +190,23 @@ mod imp { static TREES: Mutex> = Mutex::new(Vec::new()); static SECTION: Mutex<()> = Mutex::new(()); static TRACES: Mutex>> = Mutex::new(BTreeMap::new()); + static JOBS: Mutex>> = Mutex::new(BTreeMap::new()); + + /// A job number for [`super::JobEvent::job`]. + pub(crate) fn next_job() -> u64 { + static NEXT: AtomicU64 = AtomicU64::new(1); + NEXT.fetch_add(1, Ordering::Relaxed) + } + + /// Record a job action of owner instance `owner` (inside the section). + pub(crate) fn job(owner: u64, event: impl FnOnce() -> super::JobEvent) { + atomic(|| lock(&JOBS).entry(owner).or_default().push(event())); + } + + /// Remove and return the job events recorded for an owner instance. + pub fn take_jobs(owner: u64) -> Vec { + lock(&JOBS).remove(&owner).unwrap_or_default() + } thread_local! { static DEPTH: Cell = const { Cell::new(0) }; static SUPPRESS_CLOSE: Cell = const { Cell::new(false) }; @@ -429,7 +481,10 @@ mod imp { pub(crate) fn failing(f: impl FnOnce() -> R) -> R { f() } + #[allow(dead_code)] // recording is compiled out without the feature + #[inline(always)] + pub(crate) fn job(_: u64, _: impl FnOnce() -> super::JobEvent) {} } #[cfg(feature = "lifecycle-trace")] -pub use imp::{take, take_tree}; +pub use imp::{take, take_jobs, take_tree}; diff --git a/concurrency/src/reply/job_core.rs b/concurrency/src/reply/job_core.rs index 19eb65e0..eb6dbe3c 100644 --- a/concurrency/src/reply/job_core.rs +++ b/concurrency/src/reply/job_core.rs @@ -73,11 +73,12 @@ impl JobCore { } /// Set the cancellation flag: suppresses a pending notification and - /// wakes blocked emitters. Never undoes effects. The caller wakes async - /// emitters afterwards. - pub(crate) fn cancel(&self) { - { + /// wakes blocked emitters. Never undoes effects. Returns whether this + /// call set the flag. The caller wakes async emitters afterwards. + pub(crate) fn cancel(&self) -> bool { + let newly = { let mut inner = lock(&self.inner); + let newly = !inner.cancelled; inner.cancelled = true; if inner.notification == Notification::Pending { inner.notification = Notification::Suppressed; @@ -85,8 +86,10 @@ impl JobCore { // Set under the lock, so a dispatch that sees an enqueued // message's job as not cancelled ran before this cancel. self.cancelled_fast.store(true, Ordering::Release); - } + newly + }; self.emitters.notify_all(); + newly } /// Take the terminal notification and run `enqueue` under the job's diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 5c700b91..c491f8a7 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -155,11 +155,15 @@ impl PipeReplyError { pub use super::job_core::JobGone; use super::job_core::{Job, JobCore, JobSetCore}; +use crate::lifecycle_trace as trace; /// One job's shared state: the core protocol plus the async wake-ups. pub(crate) struct JobState { owner: ActorId, core: JobCore, + /// The job's number in lifecycle traces. + #[cfg(feature = "lifecycle-trace")] + trace: u64, /// Async cancellation for `tasks` jobs (drops the work future). #[cfg(feature = "async")] cancel: spawned_rt::tasks::CancellationToken, @@ -177,6 +181,8 @@ impl JobState { Arc::new(Self { owner, core: JobCore::new(emit_credits), + #[cfg(feature = "lifecycle-trace")] + trace: crate::lifecycle_trace::next_job(), #[cfg(feature = "async")] cancel: spawned_rt::tasks::CancellationToken::new(), #[cfg(feature = "async")] @@ -184,10 +190,6 @@ impl JobState { }) } - pub(crate) fn owner(&self) -> ActorId { - self.owner - } - pub(crate) fn is_cancelled(&self) -> bool { self.core.is_cancelled() } @@ -197,10 +199,32 @@ impl JobState { &self.cancel } + /// Record a `DeferredJobs.tla` action of this job (lifecycle traces). + #[cfg_attr(not(feature = "lifecycle-trace"), allow(unused_variables))] + fn record(&self, event: impl FnOnce(u64) -> trace::JobEvent) { + #[cfg(feature = "lifecycle-trace")] + trace::job(self.owner.raw(), || event(self.trace)); + } + /// Set the cancellation flag: suppresses a pending notification and - /// wakes blocked emitters. Never undoes effects. + /// wakes blocked emitters. Never undoes effects. Recorded as the model's + /// `Cancel` (the owner's exit and key replacement cancel through the job + /// set instead, as part of `Exit` and `Start`). pub(crate) fn cancel(&self) { + trace::atomic(|| { + if self.core.cancel() { + self.record(|job| trace::JobEvent::new("Cancel", job)); + } + }); + self.wake_cancelled(); + } + + fn cancel_quietly(&self) { self.core.cancel(); + self.wake_cancelled(); + } + + fn wake_cancelled(&self) { #[cfg(feature = "async")] { self.cancel.cancel(); @@ -217,9 +241,25 @@ impl JobState { } /// Take the terminal notification and run `enqueue` under the job's - /// lock (see [`JobCore::notify`]). - pub(crate) fn notify(&self, enqueue: impl FnOnce() -> Option) -> (bool, Option) { - let (taken, unsent) = self.core.notify(enqueue); + /// lock (see [`JobCore::notify`]). `deadline` tells the job deadline from + /// the end of execution (the model's `JobDeadline` and `Finish`). + pub(crate) fn notify( + &self, + deadline: bool, + enqueue: impl FnOnce() -> Option, + ) -> (bool, Option) { + let (taken, unsent) = trace::atomic(|| { + let (taken, unsent) = self.core.notify(enqueue); + if !deadline { + self.record(|job| trace::JobEvent { + x: if taken { "notified" } else { "quiet" }, + ..trace::JobEvent::new("Finish", job) + }); + } else if taken { + self.record(|job| trace::JobEvent::new("JobDeadline", job)); + } + (taken, unsent) + }); #[cfg(feature = "async")] if taken { self.wake_emitters(); @@ -231,6 +271,31 @@ impl JobState { self.core.is_running() } + /// Execution of a cancelled async job ended without a notification (the + /// model's `Finish` when the notification was suppressed). + #[cfg(feature = "async")] + pub(crate) fn ended_cancelled(&self) { + self.record(|job| trace::JobEvent { + x: "quiet", + ..trace::JobEvent::new("Finish", job) + }); + } + + /// The dispatch fence: a message of this job dispatched by instance `at` + /// is dropped when the job is cancelled or belongs to another instance. + /// `kind` is `emit`, `done` or `deadline` (the model's `Dispatch`). + pub(crate) fn fenced(&self, at: ActorId, kind: &'static str) -> bool { + trace::atomic(|| { + let fenced = self.is_cancelled() || self.owner != at; + self.record(|job| trace::JobEvent { + x: kind, + fenced, + ..trace::JobEvent::new("Dispatch", job) + }); + fenced + }) + } + /// Emit step 1 (blocking). pub(crate) fn acquire_blocking(&self) -> Result<(), JobGone> { self.core.acquire_blocking() @@ -259,7 +324,13 @@ impl JobState { &self, enqueue: impl FnOnce() -> Result<(), X>, ) -> Result<(), (JobGone, Option)> { - self.core.enqueue_emit(enqueue) + trace::atomic(|| { + let result = self.core.enqueue_emit(enqueue); + if result.is_ok() { + self.record(|job| trace::JobEvent::new("Emit", job)); + } + result + }) } /// Emit step 3: a credit comes back when its message is dispatched, @@ -273,7 +344,7 @@ impl JobState { impl Job for JobState { fn cancel(&self) { - JobState::cancel(self); + self.cancel_quietly(); } fn is_running(&self) -> bool { self.core.is_running() @@ -326,6 +397,9 @@ impl JobHandle { pub(crate) struct JobSet { owner: ActorId, core: JobSetCore, + /// Whether the owner's exit was recorded (lifecycle traces). + #[cfg(feature = "lifecycle-trace")] + exited: std::sync::atomic::AtomicBool, #[cfg(feature = "async")] idle_async: Mutex>, } @@ -335,6 +409,8 @@ impl JobSet { Self { owner, core: JobSetCore::new(max_native), + #[cfg(feature = "lifecycle-trace")] + exited: std::sync::atomic::AtomicBool::new(false), #[cfg(feature = "async")] idle_async: Mutex::new(Vec::new()), } @@ -354,9 +430,17 @@ impl JobSet { } /// Admit a job: replace (cancel) the previous one with the same key and - /// count it as running. - pub(crate) fn admit(&self, state: &Arc, key: Option) { - self.core.admit(state, key); + /// count it as running (the model's `Start`). + pub(crate) fn admit(&self, state: &Arc, key: Option, deadline: bool) { + trace::atomic(|| { + let keyed = key.is_some(); + self.core.admit(state, key); + state.record(|job| trace::JobEvent { + keyed, + deadline, + ..trace::JobEvent::new("Start", job) + }); + }); } /// A job's execution ended. @@ -375,7 +459,13 @@ impl JobSet { /// Owner exit: cancel every job (flag set, blocked emitters woken). pub(crate) fn cancel_all(&self) { - self.core.cancel_all(); + trace::atomic(|| { + self.core.cancel_all(); + #[cfg(feature = "lifecycle-trace")] + if !self.exited.swap(true, std::sync::atomic::Ordering::AcqRel) { + trace::job(self.owner.raw(), || trace::JobEvent::new("Exit", 0)); + } + }); } /// Join the cancelled jobs, bounded by `deadline`; returns how many are @@ -535,10 +625,10 @@ mod tests { fn replacement_cancels_a_finished_keyed_job() { let set = JobSet::new(ActorId::next(), 1); let old = JobState::new(set.owner(), 1); - set.admit(&old, Some(JobKey::new("k"))); + set.admit(&old, Some(JobKey::new("k")), false); set.ended(&old); let new = JobState::new(set.owner(), 1); - set.admit(&new, Some(JobKey::new("k"))); + set.admit(&new, Some(JobKey::new("k")), false); assert!(old.is_cancelled(), "the finished job escaped replacement"); assert!(!new.is_cancelled()); } diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index e3a47705..deced057 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -54,7 +54,7 @@ where ctx: &'a Context, ) -> Pin + Send + 'a>> { let EmitEnvelope { msg, credit } = *self; - let fenced = credit.0.is_cancelled() || credit.0.owner() != ctx.id(); + let fenced = credit.0.fenced(ctx.id(), "emit"); // The credit returns at dispatch (or here, when fenced). drop(credit); if fenced { @@ -91,11 +91,16 @@ where ctx: &'a Context, ) -> Pin + Send + 'a>> { let TerminalEnvelope { job, done, outcome } = *self; + let kind = if matches!(outcome, JobOutcome::Err(JobError::DeadlineExceeded)) { + "deadline" + } else { + "done" + }; // Everything runs inside the returned future, so a panicking `done` // is a handler panic caught by the run loop (`stopped()`, turn end // and job cancellation still run). Box::pin(async move { - if job.is_cancelled() || job.owner() != ctx.id() { + if job.fenced(ctx.id(), kind) { // Fenced: `done` and its captures (a linked token) are // discarded in the turn, through the mailbox drop path. ctx.shared.sender.failures().discard((done, outcome)); @@ -126,7 +131,8 @@ where { /// Take the terminal notification and enqueue `(done, outcome)`. fn notify(&self, outcome: JobOutcome) { - let (_, unsent) = self.state.notify(|| { + let deadline = matches!(outcome, JobOutcome::Err(JobError::DeadlineExceeded)); + let (_, unsent) = self.state.notify(deadline, || { let done = self.done.lock().unwrap_or_else(|p| p.into_inner()).take()?; let envelope: Box + Send> = Box::new(TerminalEnvelope { job: self.state.clone(), @@ -305,7 +311,9 @@ impl Context { R: Send + 'static, W: Future + Send + 'static, { - self.shared.jobs.admit(&state, opts.key.clone()); + self.shared + .jobs + .admit(&state, opts.key.clone(), opts.deadline.is_some()); let run = Arc::new(Run { state: state.clone(), ctx: self.clone(), @@ -339,7 +347,10 @@ impl Context { } // Cancelled (or the opt-in caller give-up): the work future is // dropped here, and with it any downstream receiver. - future::Either::Right(_) => state.cancel(), + future::Either::Right(_) => { + state.cancel(); + state.ended_cancelled(); + } } }); handle @@ -395,7 +406,9 @@ impl Context { }); match spawned { Ok(_thread) => { - self.shared.jobs.admit(&state, opts.key.clone()); + self.shared + .jobs + .admit(&state, opts.key.clone(), opts.deadline.is_some()); let deadline = self.arm_deadline(&opts, &run); let ended = Ended(self.clone(), state.clone()); let _ = start.send((ended, deadline)); diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index bd28c86e..50dc425b 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -45,7 +45,7 @@ where { fn handle(self: Box, actor: &mut A, ctx: &Context) { let EmitEnvelope { msg, credit } = *self; - let fenced = credit.0.is_cancelled() || credit.0.owner() != ctx.id(); + let fenced = credit.0.fenced(ctx.id(), "emit"); // The credit returns at dispatch (or here, when fenced). drop(credit); if fenced { @@ -74,7 +74,12 @@ where { fn handle(self: Box, actor: &mut A, ctx: &Context) { let TerminalEnvelope { job, done, outcome } = *self; - if job.is_cancelled() || job.owner() != ctx.id() { + let kind = if matches!(outcome, JobOutcome::Err(JobError::DeadlineExceeded)) { + "deadline" + } else { + "done" + }; + if job.fenced(ctx.id(), kind) { // Fenced: `done` and its captures (a linked token) drop in the // turn, by the drop rules. ctx.sender.failures().discard((done, outcome)); @@ -104,7 +109,8 @@ where { /// Take the terminal notification and enqueue `(done, outcome)`. fn notify(&self, outcome: JobOutcome) { - let (_, unsent) = self.state.notify(|| { + let deadline = matches!(outcome, JobOutcome::Err(JobError::DeadlineExceeded)); + let (_, unsent) = self.state.notify(deadline, || { let done = self.done.lock().unwrap_or_else(|p| p.into_inner()).take()?; let envelope: Box + Send> = Box::new(TerminalEnvelope { job: self.state.clone(), @@ -302,7 +308,8 @@ impl Context { }); match spawned { Ok(_thread) => { - self.jobs.admit(&state, opts.key.clone()); + self.jobs + .admit(&state, opts.key.clone(), opts.deadline.is_some()); let deadline = self.arm_deadline(&opts, &run); let ended = Ended(self.clone(), state.clone()); let _ = start.send((ended, deadline)); diff --git a/concurrency/tests/job_trace.rs b/concurrency/tests/job_trace.rs new file mode 100644 index 00000000..cd4ac49b --- /dev/null +++ b/concurrency/tests/job_trace.rs @@ -0,0 +1,338 @@ +//! Records pipe-to-self job traces from real actors for TLA+ trace +//! validation against `verification/tla/DeferredJobs.tla` +//! (verification/tla/README.md). Each scenario writes `jobs_.json` to +//! `$LIFECYCLE_TRACE_DIR` (default `target/lifecycle-traces`); +//! `verification/tla/trace_check.py` checks them (schema `jobs-v1`). +#![cfg(feature = "lifecycle-trace")] + +use spawned_concurrency::{ + lifecycle_trace, + message::Message, + tasks::{ + Actor, ActorRef, ActorStart, Context, Handler, JobCtx, JobHandle, JobOptions, JobOutcome, + PendingReply, + }, +}; +use spawned_rt::tasks::{oneshot, sleep}; +use std::{ + path::PathBuf, + sync::{Arc, Mutex}, + time::Duration, +}; + +fn trace_dir() -> PathBuf { + std::env::var_os("LIFECYCLE_TRACE_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../target/lifecycle-traces") + }) +} + +#[derive(Default)] +struct Log(Mutex>); +impl Log { + fn push(&self, entry: impl Into) { + self.0.lock().unwrap().push(entry.into()); + } + fn snapshot(&self) -> Vec { + self.0.lock().unwrap().clone() + } +} + +struct Owner { + log: Arc, + handles: Vec, + gates: Vec>, + /// The gated old keyed job, handed to the test. + old: Arc)>>>, +} + +impl Actor for Owner {} + +struct Done(String); +impl Message for Done { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: Done, _: PendingReply<'_, ()>, _: &Context) { + self.log.push(msg.0); + } +} + +struct Emitted(u32); +impl Message for Emitted { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: Emitted, _: PendingReply<'_, ()>, _: &Context) { + self.log.push(format!("emit:{}", msg.0)); + } +} + +fn done(name: &'static str) -> impl FnOnce(JobOutcome) -> Done { + move |outcome| { + Done(match outcome { + JobOutcome::Ok(v) => format!("{name}:ok:{v:?}"), + JobOutcome::Err(e) => format!("{name}:{e}"), + }) + } +} + +#[derive(Clone, Copy)] +enum Scenario { + Plain, + KeyedFenced, + Emits, + Gated, + Deadline, + Blocking, +} + +struct Run(Scenario); +impl Message for Run { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: Run, reply: PendingReply<'_, ()>, ctx: &Context) { + match msg.0 { + Scenario::Plain => { + ctx.pipe(async { 1u32 }, done("plain")).unwrap(); + } + Scenario::KeyedFenced => { + let (open, gate) = oneshot::channel::<()>(); + let old = ctx + .pipe_keyed( + "k", + async move { gate.await.map(|()| 1u32).unwrap_or(0) }, + done("old"), + ) + .unwrap(); + *self.old.lock().unwrap() = Some((old, open)); + } + Scenario::Emits => { + let opts = JobOptions { + emit_credits: 1, + ..Default::default() + }; + ctx.pipe_with_ctx( + opts, + |job: JobCtx| async move { + for i in 0..3 { + if job.emit(Emitted(i)).await.is_err() { + break; + } + } + 3u32 + }, + done("emits"), + ) + .unwrap(); + } + Scenario::Gated => { + let (open, gate) = oneshot::channel::<()>(); + let handle = ctx + .pipe( + async move { gate.await.map(|()| 1u32).unwrap_or(0) }, + done("gated"), + ) + .unwrap(); + self.handles.push(handle); + self.gates.push(open); + } + Scenario::Deadline => { + let opts = JobOptions { + deadline: Some(spawned_rt::time::Instant::now() + Duration::from_millis(30)), + ..Default::default() + }; + ctx.pipe_with( + opts, + async { + sleep(Duration::from_millis(120)).await; + 1u32 + }, + done("slow"), + ) + .unwrap(); + } + Scenario::Blocking => { + let opts = JobOptions { + emit_credits: 2, + ..Default::default() + }; + ctx.pipe_blocking_with( + opts, + |job: &JobCtx| { + let _ = job.emit_blocking(Emitted(0)); + let _ = job.emit_blocking(Emitted(1)); + 5u32 + }, + done("blocking"), + ) + .unwrap(); + } + } + reply.reply(()); + } +} + +/// Hold the turn until `go`, then replace key `k`: the old job finished +/// meanwhile, so its terminal message is queued behind this turn and fenced. +struct ReplaceWhen(Arc); +impl Message for ReplaceWhen { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: ReplaceWhen, _: PendingReply<'_, ()>, ctx: &Context) { + while !msg.0.load(std::sync::atomic::Ordering::SeqCst) { + std::thread::sleep(Duration::from_millis(1)); + } + ctx.pipe_keyed("k", async { 2u32 }, done("new")).unwrap(); + } +} + +/// Cancel the stored handles, then open the gates. +struct CancelStored; +impl Message for CancelStored { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: CancelStored, reply: PendingReply<'_, ()>, _: &Context) { + for handle in self.handles.drain(..) { + handle.cancel(); + } + for gate in self.gates.drain(..) { + let _ = gate.send(()); + } + reply.reply(()); + } +} + +fn start() -> (ActorRef, Arc) { + start_with(Arc::default()) +} + +fn start_with( + old: Arc)>>>, +) -> (ActorRef, Arc) { + let log = Arc::new(Log::default()); + let actor = Owner { + old, + log: log.clone(), + handles: Vec::new(), + gates: Vec::new(), + } + .start(); + (actor, log) +} + +async fn wait_for(log: &Log, entry: &str) { + for _ in 0..400 { + if log.snapshot().iter().any(|e| e == entry) { + return; + } + sleep(Duration::from_millis(5)).await; + } + panic!("{entry} never logged: {:?}", log.snapshot()); +} + +/// Stop the owner, wait for its cleanup (the bounded job join), and write +/// its job trace. +async fn finish(name: &str, actor: ActorRef, credits: u16) -> Vec { + let id: u64 = actor.id().to_string().parse().unwrap(); + actor.context().stop(); + actor.join().await; + let events = lifecycle_trace::take_jobs(id); + assert!( + events.iter().any(|e| e.action == "Exit"), + "{name}: the owner's exit was not recorded" + ); + let actions: Vec = events.iter().map(|e| e.action.to_string()).collect(); + let json = format!( + "{{\"name\":\"jobs_{name}\",\"schema\":\"jobs-v1\",\"credits\":{credits},\"keys\":1,\"events\":[\n{}\n]}}\n", + events + .iter() + .map(|e| e.to_json()) + .collect::>() + .join(",\n") + ); + let dir = trace_dir(); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join(format!("jobs_{name}.json")), json).unwrap(); + actions +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn plain_job_result_is_dispatched() { + let (actor, log) = start(); + actor.request(Run(Scenario::Plain)).await.unwrap(); + wait_for(&log, "plain:ok:1").await; + let actions = finish("plain", actor, 16).await; + assert_eq!(actions, ["Start", "Finish", "Dispatch", "Exit"]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn replaced_finished_keyed_job_is_fenced() { + let old = Arc::default(); + let (actor, log) = start_with(Arc::clone(&old)); + actor.request(Run(Scenario::KeyedFenced)).await.unwrap(); + let go = Arc::new(std::sync::atomic::AtomicBool::new(false)); + actor.send(ReplaceWhen(go.clone())).unwrap(); + let (handle, open) = old.lock().unwrap().take().unwrap(); + open.send(()).unwrap(); + while handle.is_running() { + sleep(Duration::from_millis(1)).await; + } + go.store(true, std::sync::atomic::Ordering::SeqCst); + wait_for(&log, "new:ok:2").await; + finish("keyed_fenced", actor, 16).await; + assert!(!log.snapshot().iter().any(|e| e.starts_with("old"))); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn emits_precede_the_terminal_message() { + let (actor, log) = start(); + actor.request(Run(Scenario::Emits)).await.unwrap(); + wait_for(&log, "emits:ok:3").await; + finish("emits", actor, 1).await; + assert_eq!(log.snapshot(), ["emit:0", "emit:1", "emit:2", "emits:ok:3"]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cancelled_job_is_quiet() { + let (actor, log) = start(); + actor.request(Run(Scenario::Gated)).await.unwrap(); + actor.request(CancelStored).await.unwrap(); + sleep(Duration::from_millis(30)).await; + finish("cancelled", actor, 16).await; + assert!(log.snapshot().is_empty(), "{:?}", log.snapshot()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn job_deadline_reports_while_work_runs() { + let (actor, log) = start(); + actor.request(Run(Scenario::Deadline)).await.unwrap(); + wait_for(&log, "slow:job deadline exceeded").await; + sleep(Duration::from_millis(200)).await; + let actions = finish("deadline", actor, 16).await; + assert_eq!( + actions, + ["Start", "JobDeadline", "Dispatch", "Finish", "Exit"] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn owner_exit_cancels_a_running_job() { + let (actor, _log) = start(); + actor.request(Run(Scenario::Gated)).await.unwrap(); + let actions = finish("exit_running", actor, 16).await; + assert_eq!(actions, ["Start", "Exit", "Finish"]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn blocking_job_emits_then_completes() { + let (actor, log) = start(); + actor.request(Run(Scenario::Blocking)).await.unwrap(); + wait_for(&log, "blocking:ok:5").await; + finish("blocking", actor, 2).await; + assert_eq!(log.snapshot(), ["emit:0", "emit:1", "blocking:ok:5"]); +} diff --git a/verification/tla/README.md b/verification/tla/README.md index 6e4c7899..27443207 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -572,3 +572,30 @@ Measured 2026-11-10 (TLC 2.19, fede): | `MCDeferredJobsDeep` (3 jobs, 2 instances, 1 emit each; safety) | 5,876,658 | 12,792,039 | 23 | 25 s, 12 workers | All 22 named controls find their counterexample, each in a few seconds. + +### Job trace validation + +With `lifecycle-trace`, the job core (`concurrency/src/reply/jobs.rs`) records +one `DeferredJobs.tla` action per owner instance: `Start` (keyed, with a job +deadline), `Emit`, `Finish` (whether it enqueued the terminal message), +`JobDeadline`, `Cancel`, `Exit` and `Dispatch` (kind and fence verdict). Each +is recorded in the trace section together with its effect (the flag change +under the job lock, the enqueue, the fence read), so the log is a +linearization. `concurrency/tests/job_trace.rs` records seven scenarios (plain +result, a finished keyed job replaced and fenced, emits with one credit, +handle cancel, job deadline with the work still running, owner exit with a +running job, a blocking job emitting); `trace_check.py` checks them (schema +`jobs-v1`) against `TraceJobs.tla` with one owner instance and one key. The +permit bound is not checked (`JobBound = J`): the code returns a job's credit +just before `Finish`, and rejected admissions are not model actions. + +```bash +cargo test -p spawned-concurrency --features lifecycle-trace --test job_trace +python3 verification/tla/trace_check.py target/lifecycle-traces +``` + +`negative_checks.py` requires six edited job traces to be rejected: the +replaced job's terminal message delivered, an emit after the terminal +message, a second emit with its credit in flight, a second terminal message +after the deadline's, a quiet finish without a cancel, and a dispatch after +the owner's exit. diff --git a/verification/tla/TraceJobs.tla b/verification/tla/TraceJobs.tla new file mode 100644 index 00000000..d0425061 --- /dev/null +++ b/verification/tla/TraceJobs.tla @@ -0,0 +1,61 @@ +------------------------------ MODULE TraceJobs ------------------------------ +(***************************************************************************) +(* Trace validation for DeferredJobs.tla. *) +(* *) +(* With the `lifecycle-trace` feature, the job core records one event per *) +(* model action for each owner instance (concurrency/src/reply/jobs.rs): *) +(* [a |-> action, j |-> job (1..J in start order), x |-> parameter, *) +(* keyed, dl, fenced]. Each event is recorded in the trace section *) +(* together with its effect (the flag change, the enqueue, or the fence *) +(* read at dispatch), so the log is a linearization. *) +(* *) +(* Start(j) admission (keyed: replaces the key; dl: has a job *) +(* deadline) *) +(* Emit(j) an emit enqueued *) +(* Finish(j) execution ended; x = "notified" if it enqueued the *) +(* terminal message, "quiet" if the notification was *) +(* already taken or suppressed *) +(* JobDeadline(j) the job deadline took the notification *) +(* Cancel(j) handle cancel (or the opt-in caller give-up) *) +(* Exit the owner's exit cancelled the job set *) +(* Dispatch(j) the mailbox head, of kind x, fenced or delivered *) +(* *) +(* One trace is one owner instance (MaxGen = 1) and at most one key. The *) +(* model's permit bound is not checked (JobBound = J): the code releases *) +(* a job's credit just before Finish, and rejected admissions are not *) +(* model actions. The trace is accepted iff some behavior consumes the *) +(* whole log (TLC reports TraceIncomplete violated). *) +(***************************************************************************) +EXTENDS DeferredJobs, TLC + +CONSTANT Trace + +VARIABLE i + +traceVars == <> + +TraceInit == Init /\ i = 0 + +Step(e) == + CASE e.a = "Start" -> e.j \in Jobs /\ Start(e.j) /\ hasDl'[e.j] = e.dl + /\ (e.j \in Keyed) = e.keyed + [] e.a = "Emit" -> e.j \in Jobs /\ Emit(e.j) + [] e.a = "Finish" -> e.j \in Jobs /\ Finish(e.j) + /\ (termSent'[e.j] > termSent[e.j]) = (e.x = "notified") + [] e.a = "JobDeadline" -> e.j \in Jobs /\ JobDeadline(e.j) + [] e.a = "Cancel" -> e.j \in Jobs /\ Cancel(e.j) + [] e.a = "Exit" -> Exit + [] e.a = "Dispatch" -> /\ mbox # <<>> + /\ Head(mbox).j = e.j /\ Head(mbox).kind = e.x + /\ Fenced(Head(mbox)) = e.fenced + /\ Dispatch + [] OTHER -> FALSE + +TraceNext == i < Len(Trace) /\ Step(Trace[i + 1]) /\ i' = i + 1 + +TraceSpec == TraceInit /\ [][TraceNext]_traceVars + +\* Violated exactly when the whole log has been matched: TLC reporting this +\* invariant as violated means the trace is ACCEPTED. +TraceIncomplete == i < Len(Trace) +============================================================================= diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index a30bcd4e..ff924e3b 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -149,6 +149,47 @@ def named_fault(module, fault, invariant, tlc): ] +def job_trace_controls(traces, tlc): + """Edits of recorded pipe-to-self job traces that DeferredJobs.tla must + reject: each erases or reorders what the job core enforces.""" + by_name = {t["name"]: t for t in traces if t.get("schema") == "jobs-v1"} + + def deliver_replaced(events): + next(e for e in events if e["a"] == "Dispatch" and e["fenced"])["fenced"] = False + require_rejected(by_name["jobs_keyed_fenced"], deliver_replaced, + "replaced keyed job's terminal message delivered", tlc) + + def emit_after_terminal(events): + finish = next(i for i, e in enumerate(events) if e["a"] == "Finish") + last_emit = max(i for i, e in enumerate(events) if e["a"] == "Emit") + events.insert(last_emit, events.pop(finish)) + require_rejected(by_name["jobs_emits"], emit_after_terminal, + "emit enqueued after the terminal message", tlc) + + def skip_credit_return(events): + events.remove(next(e for e in events if e["a"] == "Dispatch")) + require_rejected(by_name["jobs_emits"], skip_credit_return, + "second emit with its credit still in flight", tlc) + + def second_terminal(events): + next(e for e in events if e["a"] == "Finish")["x"] = "notified" + require_rejected(by_name["jobs_deadline"], second_terminal, + "terminal message after the job deadline's", tlc) + + def forget_cancel(events): + events.remove(next(e for e in events if e["a"] == "Cancel")) + require_rejected(by_name["jobs_cancelled"], forget_cancel, + "quiet finish of a job that was never cancelled", tlc) + + def dispatch_after_exit(events): + exit_at = next(i for i, e in enumerate(events) if e["a"] == "Exit") + dispatch = next(e for e in events if e["a"] == "Dispatch") + events.remove(dispatch) + events.insert(exit_at, dispatch) + require_rejected(by_name["jobs_plain"], dispatch_after_exit, + "dispatch after the owner's exit", tlc) + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("traces", type=Path) @@ -205,6 +246,7 @@ def fail_instead_of_retry(events): model_fault("MCNested", "Propagation", args.tlc, temporal=True) for module, fault, invariant in DEFERRED_CONTROLS: named_fault(module, fault, invariant, args.tlc) + job_trace_controls(traces, args.tlc) if __name__ == "__main__": main() diff --git a/verification/tla/trace_check.py b/verification/tla/trace_check.py index d39a40ae..8962e284 100644 --- a/verification/tla/trace_check.py +++ b/verification/tla/trace_check.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -"""Check recorded supervisor lifecycle traces against Supervision.tla. +"""Check recorded lifecycle traces against Supervision.tla (and, by schema, +Cohorts.tla, NestedSupervision.tla and DeferredJobs.tla). Each JSON trace (written by concurrency/tests/supervision_trace.rs with the `lifecycle-trace` feature) looks like: @@ -47,10 +48,12 @@ def tla_event(event, cohort=False): def generate(trace, workdir): schema = trace.get("schema", "one-for-one-v1") - if schema not in {"one-for-one-v1", "cohort-v1", "tree-v1"}: + if schema not in {"one-for-one-v1", "cohort-v1", "tree-v1", "jobs-v1"}: raise ValueError(f"unsupported trace schema: {schema}") if schema == "tree-v1": return generate_tree(trace, workdir) + if schema == "jobs-v1": + return generate_jobs(trace, workdir) cohort = schema == "cohort-v1" modules = ("Cohorts.tla", "TraceCohorts.tla") if cohort else MODELS for model in modules: @@ -106,6 +109,46 @@ def generate_tree(trace, workdir): "INVARIANT Safety\nINVARIANT TraceIncomplete\n") +def generate_jobs(trace, workdir): + """One owner instance's pipe-to-self jobs against DeferredJobs.tla.""" + for model in ("DeferredJobs.tla", "TraceJobs.tla"): + shutil.copy(os.path.join(HERE, model), workdir) + if int(trace.get("keys", 0)) > 1: + raise ValueError("DeferredJobs.tla models one key per trace") + # Number jobs 1..J in the order they appear; Exit carries job 0. + jobs = {} + for e in trace["events"]: + if int(e["j"]): + jobs.setdefault(int(e["j"]), len(jobs) + 1) + if not jobs: + raise ValueError("a jobs trace needs at least one job") + keyed = sorted({jobs[int(e["j"])] for e in trace["events"] if e["a"] == "Start" and e["keyed"]}) + emits = {} + for e in trace["events"]: + if e["a"] == "Emit": + emits[int(e["j"])] = emits.get(int(e["j"]), 0) + 1 + + def flag(value): + return "TRUE" if value else "FALSE" + + records = ",\n ".join( + "[a |-> {}, j |-> {}, x |-> {}, keyed |-> {}, dl |-> {}, fenced |-> {}]".format( + tla_string(e["a"]), jobs.get(int(e["j"]), 0), tla_string(e["x"]), + flag(e["keyed"]), flag(e["dl"]), flag(e["fenced"])) + for e in trace["events"]) + with open(os.path.join(workdir, "TraceRun.tla"), "w") as out: + out.write("---- MODULE TraceRun ----\nEXTENDS TraceJobs\n" + "TraceKeyed == {" + ", ".join(map(str, keyed)) + "}\n" + f"TraceLog == <<\n {records}\n>>\n====\n") + with open(os.path.join(workdir, "TraceRun.cfg"), "w") as out: + out.write("INIT TraceInit\nNEXT TraceNext\nCONSTANTS\n" + f" J = {len(jobs)}\n JobBound = {len(jobs)}\n MaxGen = 1\n" + f" MaxEmit = {max(emits.values(), default=0)}\n" + f" Credits = {int(trace['credits'])}\n Fault = \"None\"\n" + " Keyed <- TraceKeyed\n Trace <- TraceLog\n" + "INVARIANT Safety\nINVARIANT TraceIncomplete\n") + + def longest_prefix(dump_path): best = 0 try: From e68ed391cc1de9906fd23426e30a95f60380a149 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 01:16:17 +0000 Subject: [PATCH 056/208] test(jobs): name the trace test handoff type --- concurrency/tests/job_trace.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/concurrency/tests/job_trace.rs b/concurrency/tests/job_trace.rs index cd4ac49b..dd4b82b8 100644 --- a/concurrency/tests/job_trace.rs +++ b/concurrency/tests/job_trace.rs @@ -28,6 +28,9 @@ fn trace_dir() -> PathBuf { }) } +/// The gated old keyed job and its gate, handed from the turn to the test. +type OldJob = Arc)>>>; + #[derive(Default)] struct Log(Mutex>); impl Log { @@ -44,7 +47,7 @@ struct Owner { handles: Vec, gates: Vec>, /// The gated old keyed job, handed to the test. - old: Arc)>>>, + old: OldJob, } impl Actor for Owner {} @@ -211,9 +214,7 @@ fn start() -> (ActorRef, Arc) { start_with(Arc::default()) } -fn start_with( - old: Arc)>>>, -) -> (ActorRef, Arc) { +fn start_with(old: OldJob) -> (ActorRef, Arc) { let log = Arc::new(Log::default()); let actor = Owner { old, From 954cadb68cafeded88b8b3d78f34fb9d6e38fe39 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 01:23:42 +0000 Subject: [PATCH 057/208] verify(replies): lifecycle-trace events for reply slots, TraceReplies.tla and reply trace validation with negative controls --- .github/workflows/verification.yml | 2 +- CHANGELOG.md | 5 +- concurrency/src/lifecycle_trace.rs | 55 +++++- concurrency/src/reply/mod.rs | 30 +++ concurrency/src/reply/registry.rs | 14 +- concurrency/src/reply/slot.rs | 32 +++- concurrency/tests/job_core_loom.rs | 4 + concurrency/tests/reply_slot_loom.rs | 4 + concurrency/tests/reply_trace.rs | 261 +++++++++++++++++++++++++++ verification/tla/README.md | 25 ++- verification/tla/TraceReplies.tla | 67 +++++++ verification/tla/negative_checks.py | 31 ++++ verification/tla/trace_check.py | 37 +++- verification/tla/trace_coverage.py | 12 +- 14 files changed, 563 insertions(+), 16 deletions(-) create mode 100644 concurrency/tests/reply_trace.rs create mode 100644 verification/tla/TraceReplies.tla diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml index e9ed2951..670c7313 100644 --- a/.github/workflows/verification.yml +++ b/.github/workflows/verification.yml @@ -30,7 +30,7 @@ jobs: - name: Record supervisor lifecycle traces env: LIFECYCLE_TRACE_DIR: ${{ github.workspace }}/lifecycle-traces - run: cargo test --locked -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace + run: cargo test --locked -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace - name: Model-check (fast configuration) and validate traces run: nix run .#check-tla -- --traces lifecycle-traces --negative-controls # Which model actions, projected states and transitions the recorded diff --git a/CHANGELOG.md b/CHANGELOG.md index 001be468..d56f0572 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,10 @@ `lifecycle-trace`, the job core records `DeferredJobs.tla` actions, and `tests/job_trace.rs` traces (seven scenarios) are checked by TLC (`TraceJobs.tla`, schema `jobs-v1`), with six edited traces as negative - controls. `Handler::handle` takes a `PendingReply<'_, R>` + controls. Reply slots record `DeferredReplies.tla` actions (send, + dispatch, the terminal transition inside the slot CAS, exit, death); + `tests/reply_trace.rs` traces (nine scenarios) are checked against + `TraceReplies.tla`, with four negative controls. `Handler::handle` takes a `PendingReply<'_, R>` and returns `()`; `#[actor]` generates it, and `#[deferred_handler]` methods receive the obligation and may `defer()` it into a storable `ReplyTo` (bounded by `Actor::MAX_OUTSTANDING`, `Overloaded` at the bound). Dropped diff --git a/concurrency/src/lifecycle_trace.rs b/concurrency/src/lifecycle_trace.rs index 08b30d79..789dc786 100644 --- a/concurrency/src/lifecycle_trace.rs +++ b/concurrency/src/lifecycle_trace.rs @@ -142,6 +142,28 @@ impl JobEvent { } } +/// One reply-slot action of `verification/tla/DeferredReplies.tla`, recorded +/// per owner instance: `Send`, `Dispatch`, `Discharge` (with the terminal +/// kind in `x`: `ok`, `timeout`, `dropped`, `stopped`, `overloaded`, +/// `abandoned`), `Exit` and `Death`. `req` identifies the slot (normalized by +/// the checker at each `Send`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReplyEvent { + pub action: &'static str, + pub req: u64, + pub x: &'static str, +} + +impl ReplyEvent { + /// JSON object understood by `verification/tla/trace_check.py`. + pub fn to_json(&self) -> String { + format!( + r#"{{"a":"{}","r":{},"x":"{}"}}"#, + self.action, self.req, self.x + ) + } +} + pub(crate) fn flag(value: bool) -> &'static str { if value { "true" @@ -203,6 +225,34 @@ mod imp { atomic(|| lock(&JOBS).entry(owner).or_default().push(event())); } + static REPLIES: Mutex>> = Mutex::new(BTreeMap::new()); + static DEAD: Mutex> = + Mutex::new(std::collections::BTreeSet::new()); + + /// Record a reply-slot action of owner instance `owner`. + pub(crate) fn reply(owner: u64, action: &'static str, req: u64, x: &'static str) { + atomic(|| { + // The owner's death closes everything still open in one model + // step (`Death`); the closes it performs are not separate events. + if action == "Discharge" && x == "stopped" && lock(&DEAD).contains(&owner) { + return; + } + if action == "Death" && !lock(&DEAD).insert(owner) { + return; + } + lock(&REPLIES) + .entry(owner) + .or_default() + .push(super::ReplyEvent { action, req, x }); + }); + } + + /// Remove and return the reply events recorded for an owner instance. + pub fn take_replies(owner: u64) -> Vec { + lock(&DEAD).remove(&owner); + lock(&REPLIES).remove(&owner).unwrap_or_default() + } + /// Remove and return the job events recorded for an owner instance. pub fn take_jobs(owner: u64) -> Vec { lock(&JOBS).remove(&owner).unwrap_or_default() @@ -483,8 +533,11 @@ mod imp { } #[allow(dead_code)] // recording is compiled out without the feature #[inline(always)] + pub(crate) fn reply(_: u64, _: &'static str, _: u64, _: &'static str) {} + #[allow(dead_code)] // recording is compiled out without the feature + #[inline(always)] pub(crate) fn job(_: u64, _: impl FnOnce() -> super::JobEvent) {} } #[cfg(feature = "lifecycle-trace")] -pub use imp::{take, take_jobs, take_tree}; +pub use imp::{take, take_jobs, take_replies, take_tree}; diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index fe201a78..a02a1dd7 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -35,6 +35,23 @@ mod sync { pub(crate) use std::thread::yield_now; pub(crate) type SlotOwner = Option>; + /// Run a slot transition as one step of the lifecycle-trace + /// linearization (the identity without the feature). + #[inline(always)] + pub(crate) fn section(f: impl FnOnce() -> T) -> T { + crate::lifecycle_trace::atomic(f) + } + + /// A slot left `OPEN` (`kind` is the model's terminal name). + #[inline(always)] + #[cfg_attr(not(feature = "lifecycle-trace"), allow(unused_variables))] + pub(crate) fn traced_terminal(owner: &SlotOwner, slot: usize, kind: &'static str) { + #[cfg(feature = "lifecycle-trace")] + if let Some(registry) = owner { + crate::lifecycle_trace::reply(registry.owner().raw(), "Discharge", slot as u64, kind); + } + } + /// Wait on `condvar` for at most `ticks` clock ticks (nanoseconds), or /// until notified when `None`. pub(crate) fn wait_ticks<'a, T>( @@ -122,6 +139,10 @@ pub(crate) fn reply_slot( owner: Option<&Arc>, ) -> (SlotHandle, ReplyReceiver) { let slot = Arc::new(Slot::with_owner(deadline.ticks(), owner.cloned())); + #[cfg(feature = "lifecycle-trace")] + if let Some(owner) = owner { + crate::lifecycle_trace::reply(owner.owner().raw(), "Send", Arc::as_ptr(&slot) as u64, ""); + } (SlotHandle(Some(slot.clone())), ReplyReceiver::new(slot)) } @@ -194,6 +215,15 @@ impl std::fmt::Debug for PendingReply<'_, R> { impl<'t, R: Send + 'static> PendingReply<'t, R> { pub(crate) fn new(slot: Option>>, registry: &'t Arc) -> Self { + #[cfg(feature = "lifecycle-trace")] + if let Some(slot) = &slot { + crate::lifecycle_trace::reply( + registry.owner().raw(), + "Dispatch", + Arc::as_ptr(slot) as u64, + "", + ); + } Self { slot, registry: Some(registry), diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index 530c0815..e2cbed2b 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -252,11 +252,16 @@ impl Registry { if self.turn.load() == DEAD { return; } - self.exit_began - .lock() - .unwrap_or_else(|p| p.into_inner()) - .get_or_insert_with(spawned_rt::time::Instant::now); + let first = { + let mut began = self.exit_began.lock().unwrap_or_else(|p| p.into_inner()); + let first = began.is_none(); + began.get_or_insert_with(spawned_rt::time::Instant::now); + first + }; Self::commit(self.turn.exiting(), Outcome::Stopped); + if first { + crate::lifecycle_trace::reply(self.owner.raw(), "Exit", 0, ""); + } } /// What remains of `timeout`, measured from the start of exit. @@ -272,6 +277,7 @@ impl Registry { /// Death: every open token listed becomes `STOPPED`, wherever it is. /// Returns how many were closed. pub(crate) fn close(&self) -> usize { + crate::lifecycle_trace::reply(self.owner.raw(), "Death", 0, ""); Self::commit(self.turn.close(), Outcome::Stopped); let tokens = std::mem::take(&mut lock(&self.lists).tokens); let closed = tokens diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 0549a2c0..72458646 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -77,6 +77,20 @@ pub(crate) enum Terminal { Abandoned, } +/// The model's name (`DeferredReplies.tla`) of the terminal state a slot +/// leaves `OPEN` for; `WRITING` is the reply's linearization point. +fn trace_name(state: u64) -> &'static str { + match state { + WRITING | REPLIED => "ok", + DROPPED => "dropped", + OVERLOADED => "overloaded", + TIMED_OUT => "timeout", + STOPPED => "stopped", + ABANDONED => "abandoned", + _ => "unknown", + } +} + impl Terminal { pub(crate) fn of(state: u64) -> Option { Some(match state { @@ -283,10 +297,20 @@ impl Slot { } } } - match self - .word - .compare_exchange(current, next, Ordering::AcqRel, Ordering::Acquire) - { + let exchanged = super::sync::section(|| { + let exchanged = + self.word + .compare_exchange(current, next, Ordering::AcqRel, Ordering::Acquire); + if exchanged.is_ok() && state(current) == OPEN && state(next) != OPEN { + super::sync::traced_terminal( + &self.owner, + self as *const Self as usize, + trace_name(state(next)), + ); + } + exchanged + }); + match exchanged { Ok(_) => return Ok((current, next)), Err(actual) => { if let (Some(pool), true) = (pool, up > 0) { diff --git a/concurrency/tests/job_core_loom.rs b/concurrency/tests/job_core_loom.rs index 8139eb18..620104d0 100644 --- a/concurrency/tests/job_core_loom.rs +++ b/concurrency/tests/job_core_loom.rs @@ -17,6 +17,10 @@ mod sync { pub(crate) use loom::sync::{Arc, Condvar, Mutex, MutexGuard}; pub(crate) use loom::thread::yield_now; pub(crate) type SlotOwner = Option<()>; + pub(crate) fn section(f: impl FnOnce() -> T) -> T { + f() + } + pub(crate) fn traced_terminal(_: &SlotOwner, _: usize, _: &'static str) {} pub(crate) use super::model_arc::{JobArc, JobWeak}; /// Loom has no timed wait; the model never parks on a slot. diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index c099c7cd..87ff94a6 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -12,6 +12,10 @@ mod sync { pub(crate) use loom::sync::{Arc, Condvar, Mutex, MutexGuard}; pub(crate) use loom::thread::yield_now; pub(crate) type SlotOwner = Option<()>; + pub(crate) fn section(f: impl FnOnce() -> T) -> T { + f() + } + pub(crate) fn traced_terminal(_: &SlotOwner, _: usize, _: &'static str) {} /// Loom has no timed wait; tests park only on slots without a deadline. pub(crate) fn wait_ticks<'a, T>( diff --git a/concurrency/tests/reply_trace.rs b/concurrency/tests/reply_trace.rs new file mode 100644 index 00000000..f2d6b099 --- /dev/null +++ b/concurrency/tests/reply_trace.rs @@ -0,0 +1,261 @@ +//! Records reply-slot traces from real actors for TLA+ trace validation +//! against `verification/tla/DeferredReplies.tla` (verification/tla/README.md). +//! Each scenario writes `replies_.json` to `$LIFECYCLE_TRACE_DIR` +//! (default `target/lifecycle-traces`); `verification/tla/trace_check.py` +//! checks them (schema `replies-v1`). +#![cfg(feature = "lifecycle-trace")] + +use spawned_concurrency::{ + lifecycle_trace, + message::Message, + tasks::{Actor, ActorRef, ActorStart, Context, Deadline, Handler, PendingReply, ReplyTo}, + ActorError, +}; +use spawned_rt::tasks::sleep; +use std::{path::PathBuf, time::Duration}; + +const BOUND: usize = 1; + +fn trace_dir() -> PathBuf { + std::env::var_os("LIFECYCLE_TRACE_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../target/lifecycle-traces") + }) +} + +#[derive(Default)] +struct Owner { + stored: Vec>, +} + +impl Actor for Owner { + const MAX_OUTSTANDING: usize = BOUND; +} + +/// Replied in its turn. +struct Echo(u32); +impl Message for Echo { + type Result = u32; +} +impl Handler for Owner { + async fn handle(&mut self, msg: Echo, reply: PendingReply<'_, u32>, _: &Context) { + reply.reply(msg.0); + } +} + +/// Deferred and stored for a later turn (`Overloaded` at the bound). +struct Hold; +impl Message for Hold { + type Result = u32; +} +impl Handler for Owner { + async fn handle(&mut self, _: Hold, reply: PendingReply<'_, u32>, _: &Context) { + if let Ok(token) = reply.defer() { + self.stored.push(token); + } + } +} + +/// Never answered: the unused obligation answers `ReplyDropped` at turn end. +struct Ignore; +impl Message for Ignore { + type Result = u32; +} +impl Handler for Owner { + async fn handle(&mut self, _: Ignore, _reply: PendingReply<'_, u32>, _: &Context) {} +} + +/// Deferred, then the turn panics: the token answers `ActorStopped`. +struct Boom; +impl Message for Boom { + type Result = u32; +} +impl Handler for Owner { + async fn handle(&mut self, _: Boom, reply: PendingReply<'_, u32>, _: &Context) { + let _token = reply.defer(); + panic!("boom"); + } +} + +/// A sent (not requested) message: answer, or drop, every stored token. +enum Flush { + Reply, + Drop, +} +impl Message for Flush { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: Flush, _: PendingReply<'_, ()>, _: &Context) { + for token in self.stored.drain(..) { + match msg { + Flush::Reply => { + token.reply(7); + } + Flush::Drop => drop(token), + } + } + } +} + +/// Wait until every message sent before has been handled. +struct Sync; +impl Message for Sync { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: Sync, reply: PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } +} + +fn id(actor: &ActorRef) -> u64 { + actor.id().to_string().parse().unwrap() +} + +/// Write the owner's reply trace once it is dead. +fn write(name: &str, owner: u64) -> Vec { + let events = lifecycle_trace::take_replies(owner); + assert_eq!( + events.last().map(|e| e.action), + Some("Death"), + "{name}: the owner's death was not recorded" + ); + let json = format!( + "{{\"name\":\"replies_{name}\",\"schema\":\"replies-v1\",\"bound\":{BOUND},\"events\":[\n{}\n]}}\n", + events + .iter() + .map(|e| e.to_json()) + .collect::>() + .join(",\n") + ); + let dir = trace_dir(); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join(format!("replies_{name}.json")), json).unwrap(); + events + .iter() + .map(|e| { + format!( + "{}{}", + e.action, + if e.x.is_empty() { + String::new() + } else { + format!(":{}", e.x) + } + ) + }) + .collect() +} + +async fn stop(name: &str, actor: ActorRef) -> Vec { + let owner = id(&actor); + actor.context().stop(); + actor.join().await; + write(name, owner) +} + +#[tokio::test] +async fn reply_in_turn() { + let actor = Owner::default().start(); + let got = actor.request(Echo(3)).await; + assert!(matches!(got, Ok(3)), "{got:?}"); + let events = stop("in_turn", actor).await; + assert_eq!( + events, + ["Send", "Dispatch", "Discharge:ok", "Exit", "Death"] + ); +} + +#[tokio::test] +async fn stored_token_answered_later() { + let actor = Owner::default().start(); + let held = actor.request_raw(Hold).unwrap(); + actor.request(Sync).await.unwrap(); + actor.send(Flush::Reply).unwrap(); + let got = held.await; + assert!(matches!(got, Ok(7)), "{got:?}"); + stop("later_reply", actor).await; +} + +#[tokio::test] +async fn stored_token_dropped_later() { + let actor = Owner::default().start(); + let held = actor.request_raw(Hold).unwrap(); + actor.request(Sync).await.unwrap(); + actor.send(Flush::Drop).unwrap(); + let got = held.await; + assert!(matches!(got, Err(ActorError::ReplyDropped)), "{got:?}"); + stop("later_drop", actor).await; +} + +#[tokio::test] +async fn unused_obligation_answers_dropped() { + let actor = Owner::default().start(); + let got = actor.request(Ignore).await; + assert!(matches!(got, Err(ActorError::ReplyDropped)), "{got:?}"); + stop("unused", actor).await; +} + +#[tokio::test] +async fn defer_at_the_bound_is_overloaded() { + let actor = Owner::default().start(); + let first = actor.request_raw(Hold).unwrap(); + let got = actor.request(Hold).await; + assert!(matches!(got, Err(ActorError::Overloaded)), "{got:?}"); + actor.send(Flush::Reply).unwrap(); + let got = first.await; + assert!(matches!(got, Ok(7)), "{got:?}"); + stop("overloaded", actor).await; +} + +#[tokio::test] +async fn stored_token_expires_at_its_deadline() { + let actor = Owner::default().start(); + let deadline = Deadline::At(spawned_rt::time::Instant::now() + Duration::from_millis(30)); + let held = actor.request_raw_until(Hold, deadline).unwrap(); + let got = held.await; + assert!(matches!(got, Err(ActorError::RequestTimeout)), "{got:?}"); + actor.send(Flush::Reply).unwrap(); + actor.request(Sync).await.unwrap(); + stop("expired", actor).await; +} + +#[tokio::test] +async fn caller_gives_up() { + let actor = Owner::default().start(); + let held = actor.request_raw(Hold).unwrap(); + actor.request(Sync).await.unwrap(); + drop(held); + actor.send(Flush::Reply).unwrap(); + actor.request(Sync).await.unwrap(); + stop("abandoned", actor).await; +} + +#[tokio::test] +async fn death_closes_stored_tokens() { + let actor = Owner::default().start(); + let held = actor.request_raw(Hold).unwrap(); + actor.request(Sync).await.unwrap(); + let owner = id(&actor); + actor.context().stop(); + let got = held.await; + assert!(matches!(got, Err(ActorError::ActorStopped)), "{got:?}"); + actor.join().await; + let events = write("death", owner); + assert!( + !events.iter().any(|e| e == "Discharge:stopped"), + "{events:?}" + ); +} + +#[tokio::test] +async fn panic_in_turn_answers_stopped() { + let actor = Owner::default().start(); + let got = actor.request(Boom).await; + assert!(matches!(got, Err(ActorError::ActorStopped)), "{got:?}"); + actor.join().await; + sleep(Duration::from_millis(10)).await; + write("panic", id(&actor)); +} diff --git a/verification/tla/README.md b/verification/tla/README.md index 27443207..1fcfd339 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -40,7 +40,7 @@ runs real actors through the scenarios below and writes each trace as JSON; TLC then checks that each trace is a behavior of `Supervision.tla`. ```bash -cargo test -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace +cargo test -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace nix run .#check-tla -- --traces-only target/lifecycle-traces --negative-controls # or --traces DIR to also model-check nix develop -c python3 verification/tla/trace_check.py target/lifecycle-traces ``` @@ -104,7 +104,7 @@ the recorded runs never exercise? `trace_coverage.py` reports it per model with trace validation (`Supervision`, `Cohorts`, `NestedSupervision`). ```bash -cargo test -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace +cargo test -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace nix develop -c python3 verification/tla/trace_coverage.py --traces dedicated=target/lifecycle-traces \ --model-keys verification/tla/coverage-model.json.gz --baseline verification/tla/coverage-baseline.json # every test binary: each finished supervisor dumps its traces @@ -599,3 +599,24 @@ replaced job's terminal message delivered, an emit after the terminal message, a second emit with its credit in flight, a second terminal message after the deadline's, a quiet finish without a cancel, and a dispatch after the owner's exit. + +### Reply trace validation + +The reply transport records `DeferredReplies.tla` actions per owner +instance: `Send` (slot created), `Dispatch` (its turn starts), `Discharge` +(the slot's one `OPEN` -> terminal transition, recorded in the trace section +together with the CAS in `Slot::cas`; a reply counts at `WRITING`), `Exit` +(after the registry commits the turn's drops) and `Death` (the closes that +death and mailbox disposal perform are that one model step and are not +logged). `TraceReplies.tla` takes the turn-internal steps that leave no trace +silently (a `Defer` that takes a credit, `Store`, `DropInTurn`, a turn end +that answers nothing, the clock); a `Discharge` must be the model action that +discharges exactly that request with that kind. Linked jobs and the +downstream actor are not traced (their notifications and job ends are not +logged). `concurrency/tests/reply_trace.rs` records nine scenarios (reply in +the turn, stored then answered or dropped from a later non-request turn, an +unused obligation, `Overloaded` at the bound, deadline expiry by the +service, the caller giving up, death with a stored token, a panicking turn); +`negative_checks.py` requires four edited traces to be rejected (`Overloaded` +below the bound, a reply after death, a second discharge, a drop before the +turn). diff --git a/verification/tla/TraceReplies.tla b/verification/tla/TraceReplies.tla new file mode 100644 index 00000000..9ae1cd1c --- /dev/null +++ b/verification/tla/TraceReplies.tla @@ -0,0 +1,67 @@ +----------------------------- MODULE TraceReplies ----------------------------- +(***************************************************************************) +(* Trace validation for DeferredReplies.tla (reply slots, without linked *) +(* jobs and the downstream actor). *) +(* *) +(* With the `lifecycle-trace` feature, the reply transport records per *) +(* owner instance: Send (slot created), Dispatch (its turn starts), *) +(* Discharge (the slot's one OPEN -> terminal transition, recorded in the *) +(* trace section together with the CAS; a reply counts at WRITING), Exit *) +(* (the registry enters exiting, after committing the turn's drops) and *) +(* Death (the registry closes; the closes it and the mailbox disposal *) +(* perform belong to this one model step and are not logged). *) +(* *) +(* Turn-internal steps leave no trace and are taken silently: Defer that *) +(* takes a credit (or finds the slot closed), Store, DropInTurn, a turn *) +(* end that answers nothing, and the clock. A Discharge is the model *) +(* action that discharges exactly that request with that kind. The model's *) +(* turns are request turns: later replies run between them (from *) +(* non-request turns, or stopped()). *) +(* *) +(* trace_check.py instantiates N (requests), BoundA (MAX_OUTSTANDING) and *) +(* a clock generous enough for the trace; deadlines are chosen by TLC. *) +(***************************************************************************) +EXTENDS DeferredReplies, Sequences, TLC + +CONSTANT Trace + +VARIABLE i + +traceVars == <> + +TraceInit == Init /\ i = 0 + +Only(r, k) == slot[r] = "open" /\ slot'[r] = k + /\ \A q \in Req \ {r} : slot'[q] = slot[q] + +Discharging(r) == + \/ ReplyNow(r) \/ LaterReply(r) \/ LaterDrop(r) \/ Expire(r) + \/ CallerGivesUp(r) \/ Defer(r) \/ EndTurn \/ Panic + +Step(e) == + CASE e.a = "Send" -> e.r \in Req /\ Send(e.r) + [] e.a = "Dispatch" -> e.r \in Req /\ DispatchA(e.r) + [] e.a = "Discharge" -> e.r \in Req /\ Discharging(e.r) /\ Only(e.r, e.x) + [] e.a = "Exit" -> \/ ExitBegin + \/ Panic /\ UNCHANGED slot + \/ phase = "exiting" /\ UNCHANGED vars + [] e.a = "Death" -> Death + [] OTHER -> FALSE + +\* Steps that leave no trace: they answer no request. +Silent == + /\ \/ Tick \/ EndTurn + \/ \E r \in Req : Defer(r) \/ Store(r) \/ DropInTurn(r) + /\ UNCHANGED slot + +TraceNext == + /\ i < Len(Trace) + /\ \/ Step(Trace[i + 1]) /\ UNCHANGED earlyClosed /\ i' = i + 1 + \/ Silent /\ UNCHANGED earlyClosed /\ UNCHANGED i + +TraceSpec == TraceInit /\ [][TraceNext]_traceVars + +\* Violated exactly when the whole log has been matched: TLC reporting this +\* invariant as violated means the trace is ACCEPTED. +TraceIncomplete == i < Len(Trace) +============================================================================= diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index ff924e3b..2512a090 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -149,6 +149,36 @@ def named_fault(module, fault, invariant, tlc): ] +def reply_trace_controls(traces, tlc): + """Edits of recorded reply-slot traces that DeferredReplies.tla must + reject.""" + by_name = {t["name"]: t for t in traces if t.get("schema") == "replies-v1"} + + def overload_below_bound(events): + next(e for e in events if e["a"] == "Discharge")["x"] = "overloaded" + require_rejected(by_name["replies_in_turn"], overload_below_bound, + "Overloaded with a free credit", tlc) + + def reply_after_death(events): + held = next(e for e in events if e["a"] == "Send") + events.append({"a": "Discharge", "r": held["r"], "x": "ok"}) + require_rejected(by_name["replies_death"], reply_after_death, + "stored token answered after death", tlc) + + def discharge_twice(events): + first = next(i for i, e in enumerate(events) if e["a"] == "Discharge") + events.insert(first + 1, dict(events[first], x="ok")) + require_rejected(by_name["replies_expired"], discharge_twice, + "second discharge of an expired request", tlc) + + def dropped_before_dispatch(events): + dispatch = next(i for i, e in enumerate(events) if e["a"] == "Dispatch") + drop = next(i for i, e in enumerate(events) if e.get("x") == "dropped") + events.insert(dispatch, events.pop(drop)) + require_rejected(by_name["replies_unused"], dropped_before_dispatch, + "unused obligation dropped before its turn", tlc) + + def job_trace_controls(traces, tlc): """Edits of recorded pipe-to-self job traces that DeferredJobs.tla must reject: each erases or reorders what the job core enforces.""" @@ -247,6 +277,7 @@ def fail_instead_of_retry(events): for module, fault, invariant in DEFERRED_CONTROLS: named_fault(module, fault, invariant, args.tlc) job_trace_controls(traces, args.tlc) + reply_trace_controls(traces, args.tlc) if __name__ == "__main__": main() diff --git a/verification/tla/trace_check.py b/verification/tla/trace_check.py index 8962e284..b3e3bb0c 100644 --- a/verification/tla/trace_check.py +++ b/verification/tla/trace_check.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """Check recorded lifecycle traces against Supervision.tla (and, by schema, -Cohorts.tla, NestedSupervision.tla and DeferredJobs.tla). +Cohorts.tla, NestedSupervision.tla, DeferredJobs.tla and DeferredReplies.tla). Each JSON trace (written by concurrency/tests/supervision_trace.rs with the `lifecycle-trace` feature) looks like: @@ -48,12 +48,14 @@ def tla_event(event, cohort=False): def generate(trace, workdir): schema = trace.get("schema", "one-for-one-v1") - if schema not in {"one-for-one-v1", "cohort-v1", "tree-v1", "jobs-v1"}: + if schema not in {"one-for-one-v1", "cohort-v1", "tree-v1", "jobs-v1", "replies-v1"}: raise ValueError(f"unsupported trace schema: {schema}") if schema == "tree-v1": return generate_tree(trace, workdir) if schema == "jobs-v1": return generate_jobs(trace, workdir) + if schema == "replies-v1": + return generate_replies(trace, workdir) cohort = schema == "cohort-v1" modules = ("Cohorts.tla", "TraceCohorts.tla") if cohort else MODELS for model in modules: @@ -109,6 +111,37 @@ def generate_tree(trace, workdir): "INVARIANT Safety\nINVARIANT TraceIncomplete\n") +def generate_replies(trace, workdir): + """One owner instance's reply slots against DeferredReplies.tla.""" + for model in ("DeferredReplies.tla", "TraceReplies.tla"): + shutil.copy(os.path.join(HERE, model), workdir) + # A slot address can be reused once freed: each Send starts a request. + current, n, records = {}, 0, [] + for e in trace["events"]: + r = int(e["r"]) + if e["a"] == "Send": + n += 1 + current[r] = n + req = current.get(r, 0) if r else 0 + if r and not req: + raise ValueError(f"event for an unsent request: {e}") + records.append("[a |-> {}, r |-> {}, x |-> {}]".format( + tla_string(e["a"]), req, tla_string(e["x"]))) + if not n: + raise ValueError("a replies trace needs at least one request") + # Deadlines are chosen by TLC; the clock only needs room for the trace. + clock = 2 + sum(1 for e in trace["events"] if e.get("x") == "timeout") + with open(os.path.join(workdir, "TraceRun.tla"), "w") as out: + out.write("---- MODULE TraceRun ----\nEXTENDS TraceReplies\nTraceLog == <<\n " + + ",\n ".join(records) + "\n>>\n====\n") + with open(os.path.join(workdir, "TraceRun.cfg"), "w") as out: + out.write("INIT TraceInit\nNEXT TraceNext\nCONSTANTS\n" + f" N = {n}\n BoundA = {int(trace['bound'])}\n BoundB = 1\n" + f" MaxTime = {clock}\n MaxWait = {clock}\n Fault = \"None\"\n" + " Trace <- TraceLog\n" + "INVARIANT Safety\nINVARIANT TraceIncomplete\n") + + def generate_jobs(trace, workdir): """One owner instance's pipe-to-self jobs against DeferredJobs.tla.""" for model in ("DeferredJobs.tla", "TraceJobs.tla"): diff --git a/verification/tla/trace_coverage.py b/verification/tla/trace_coverage.py index e79c110f..20e00698 100644 --- a/verification/tla/trace_coverage.py +++ b/verification/tla/trace_coverage.py @@ -260,6 +260,16 @@ def trace_keys(path, tlc): shutil.rmtree(workdir, ignore_errors=True) +# Trace schemas validated by trace_check.py but without a coverage model +# (the deferred-replies models): skipped here. +UNCOVERED_SCHEMAS = {"jobs-v1", "replies-v1"} + + +def covered(path): + with open(path) as f: + return json.load(f).get("schema") not in UNCOVERED_SCHEMAS + + def collect(paths): files = [] for p in paths: @@ -267,7 +277,7 @@ def collect(paths): files += sorted(os.path.join(p, f) for f in os.listdir(p) if f.endswith(".json")) elif os.path.exists(p): files.append(p) - return files + return [f for f in files if covered(f)] # --------------------------------------------------------------------------- From 009355dbd08507b5c013d6490f9bdba054b73c6a Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 01:29:39 +0000 Subject: [PATCH 058/208] feat(metrics): spawned_deferred_* and spawned_jobs_* metric families --- CHANGELOG.md | 8 +- concurrency/src/instrumentation.rs | 29 ++- concurrency/src/metrics.rs | 107 ++++++++++- concurrency/src/reply/jobs.rs | 41 ++++- concurrency/src/reply/mod.rs | 12 +- concurrency/src/reply/slot.rs | 3 +- concurrency/src/tasks/jobs.rs | 8 +- concurrency/src/threads/jobs.rs | 8 +- concurrency/tests/job_core_loom.rs | 2 +- concurrency/tests/metrics_deferred.rs | 255 ++++++++++++++++++++++++++ concurrency/tests/reply_slot_loom.rs | 2 +- docs/GUIDE.md | 7 + docs/MIGRATION.md | 4 +- 13 files changed, 466 insertions(+), 20 deletions(-) create mode 100644 concurrency/tests/metrics_deferred.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index d56f0572..7c9901c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,13 @@ controls. Reply slots record `DeferredReplies.tla` actions (send, dispatch, the terminal transition inside the slot CAS, exit, death); `tests/reply_trace.rs` traces (nine scenarios) are checked against - `TraceReplies.tla`, with four negative controls. `Handler::handle` takes a `PendingReply<'_, R>` + `TraceReplies.tla`, with four negative controls. With `metrics`, the + `spawned_deferred_*` family (tokens, pending, replied/expired/dropped/ + stopped/abandoned, overloaded, sampled defer-to-reply latency) and the + `spawned_jobs_*` family (started, running, completed/panicked/deadline + exceeded, cancelled while running, fenced, rejected at a bound, emits, + unfinished at stop); `tests/metrics_deferred.rs` checks each is counted + once. `Handler::handle` takes a `PendingReply<'_, R>` and returns `()`; `#[actor]` generates it, and `#[deferred_handler]` methods receive the obligation and may `defer()` it into a storable `ReplyTo` (bounded by `Actor::MAX_OUTSTANDING`, `Overloaded` at the bound). Dropped diff --git a/concurrency/src/instrumentation.rs b/concurrency/src/instrumentation.rs index f402c771..88a5a524 100644 --- a/concurrency/src/instrumentation.rs +++ b/concurrency/src/instrumentation.rs @@ -1,7 +1,7 @@ //! Internal instrumentation; disabled builds use zero-sized no-op guards. #[cfg(feature = "metrics")] pub(crate) use crate::metrics::{ - accepted, reply, ActorObservation, Mailbox, MessageObservation, RequestObservation, + accepted, hooks, reply, ActorObservation, Mailbox, MessageObservation, RequestObservation, }; #[cfg(not(feature = "metrics"))] @@ -29,6 +29,33 @@ mod disabled { #[inline] pub(crate) fn finish(self, _: &Result) {} } + + /// Deferred-reply and job accounting: no-ops without `metrics`. + pub(crate) mod hooks { + pub(crate) type DeferStamp = (); + #[inline(always)] + pub(crate) fn deferred() -> DeferStamp {} + #[inline(always)] + pub(crate) fn deferred_replied(_: &DeferStamp) {} + #[inline(always)] + pub(crate) fn left_open(_: &'static str, _: bool) {} + #[inline(always)] + pub(crate) fn job_started() {} + #[inline(always)] + pub(crate) fn job_ended() {} + #[inline(always)] + pub(crate) fn job_notified(_: &'static str) {} + #[inline(always)] + pub(crate) fn job_cancelled() {} + #[inline(always)] + pub(crate) fn job_fenced() {} + #[inline(always)] + pub(crate) fn job_rejected() {} + #[inline(always)] + pub(crate) fn job_emitted() {} + #[inline(always)] + pub(crate) fn jobs_unfinished(_: usize) {} + } } #[cfg(not(feature = "metrics"))] pub(crate) use disabled::*; diff --git a/concurrency/src/metrics.rs b/concurrency/src/metrics.rs index 080845cf..f6c73ebe 100644 --- a/concurrency/src/metrics.rs +++ b/concurrency/src/metrics.rs @@ -159,8 +159,8 @@ macro_rules! counters { /// Process aggregates. Reads are independent atomic samples, not a /// transaction. Conservation equations hold after operations quiesce. #[derive(Debug, Clone)] - pub struct Snapshot { $( pub $name: u64, )* pub queue_wait: HistogramSnapshot, pub handler_duration: HistogramSnapshot, pub request_latency: HistogramSnapshot } - pub fn snapshot() -> Snapshot { Snapshot { $( $name: COUNTERS.$name.load(Relaxed), )* queue_wait: QUEUE_WAIT.snapshot(), handler_duration: HANDLER_DURATION.snapshot(), request_latency: REQUEST_LATENCY.snapshot() } } + pub struct Snapshot { $( pub $name: u64, )* pub queue_wait: HistogramSnapshot, pub handler_duration: HistogramSnapshot, pub request_latency: HistogramSnapshot, pub deferred_reply_latency: HistogramSnapshot } + pub fn snapshot() -> Snapshot { Snapshot { $( $name: COUNTERS.$name.load(Relaxed), )* queue_wait: QUEUE_WAIT.snapshot(), handler_duration: HANDLER_DURATION.snapshot(), request_latency: REQUEST_LATENCY.snapshot(), deferred_reply_latency: DEFERRED_REPLY_LATENCY.snapshot() } } fn render_counters(out: &mut String, s: &Snapshot) { $( let name = concat!("spawned_", stringify!($name)); @@ -196,10 +196,30 @@ counters!( requests_reply_dropped_total, requests_overloaded_total, requests_cancelled_total, + deferred_tokens_total, + deferred_pending, + deferred_replied_total, + deferred_expired_total, + deferred_dropped_total, + deferred_stopped_total, + deferred_abandoned_total, + deferred_overloaded_total, + jobs_started_total, + jobs_running, + jobs_completed_total, + jobs_panicked_total, + jobs_deadline_exceeded_total, + jobs_cancelled_total, + jobs_fenced_total, + jobs_rejected_total, + jobs_emits_total, + jobs_unfinished_at_stop_total, ); static QUEUE_WAIT: Histogram = Histogram::new(); static HANDLER_DURATION: Histogram = Histogram::new(); static REQUEST_LATENCY: Histogram = Histogram::new(); +static DEFERRED_REPLY_LATENCY: Histogram = Histogram::new(); +static DEFERRED_SEQUENCE: AtomicU64 = AtomicU64::new(0); static MESSAGE_SEQUENCE: AtomicU64 = AtomicU64::new(0); static REQUEST_SEQUENCE: AtomicU64 = AtomicU64::new(0); @@ -216,6 +236,10 @@ pub fn render_prometheus() -> String { &snapshot.handler_duration, ), ("spawned_request_latency_seconds", &snapshot.request_latency), + ( + "spawned_deferred_reply_latency_seconds", + &snapshot.deferred_reply_latency, + ), ] { writeln!(out, "# TYPE {name} histogram").unwrap(); let mut count = 0; @@ -245,6 +269,85 @@ pub fn render_prometheus() -> String { out } +/// Deferred-reply and pipe-to-self job accounting +/// (docs/design/deferred-replies.md; `spawned_deferred_*`, `spawned_jobs_*`). +pub(crate) mod hooks { + use super::{sample, Instant, COUNTERS, DEFERRED_REPLY_LATENCY, DEFERRED_SEQUENCE}; + use std::sync::atomic::Ordering::Relaxed; + + /// When a token was deferred, for a sampled reply latency. + pub(crate) type DeferStamp = Option; + + /// `defer()` took a credit: the slot now has a live token. + pub(crate) fn deferred() -> DeferStamp { + COUNTERS.deferred_tokens_total.fetch_add(1, Relaxed); + COUNTERS.deferred_pending.fetch_add(1, Relaxed); + sample(&DEFERRED_SEQUENCE) + } + + /// A deferred token delivered its reply. + pub(crate) fn deferred_replied(at: &DeferStamp) { + if let Some(at) = at { + DEFERRED_REPLY_LATENCY.record(at.elapsed()); + } + } + + /// A slot left `OPEN` for `kind` (the model's terminal name); `token` + /// tells whether a deferred token was live. + pub(crate) fn left_open(kind: &'static str, token: bool) { + if kind == "overloaded" { + COUNTERS.deferred_overloaded_total.fetch_add(1, Relaxed); + } + if !token { + return; + } + COUNTERS.deferred_pending.fetch_sub(1, Relaxed); + match kind { + "ok" => &COUNTERS.deferred_replied_total, + "timeout" => &COUNTERS.deferred_expired_total, + "dropped" => &COUNTERS.deferred_dropped_total, + "stopped" => &COUNTERS.deferred_stopped_total, + "abandoned" => &COUNTERS.deferred_abandoned_total, + _ => return, + } + .fetch_add(1, Relaxed); + } + + pub(crate) fn job_started() { + COUNTERS.jobs_started_total.fetch_add(1, Relaxed); + COUNTERS.jobs_running.fetch_add(1, Relaxed); + } + pub(crate) fn job_ended() { + COUNTERS.jobs_running.fetch_sub(1, Relaxed); + } + /// The terminal notification was taken: `ok`, `panicked` or `deadline`. + pub(crate) fn job_notified(end: &'static str) { + match end { + "ok" => &COUNTERS.jobs_completed_total, + "panicked" => &COUNTERS.jobs_panicked_total, + _ => &COUNTERS.jobs_deadline_exceeded_total, + } + .fetch_add(1, Relaxed); + } + pub(crate) fn job_cancelled() { + COUNTERS.jobs_cancelled_total.fetch_add(1, Relaxed); + } + pub(crate) fn job_fenced() { + COUNTERS.jobs_fenced_total.fetch_add(1, Relaxed); + } + pub(crate) fn job_rejected() { + COUNTERS.jobs_rejected_total.fetch_add(1, Relaxed); + } + pub(crate) fn job_emitted() { + COUNTERS.jobs_emits_total.fetch_add(1, Relaxed); + } + pub(crate) fn jobs_unfinished(n: usize) { + COUNTERS + .jobs_unfinished_at_stop_total + .fetch_add(n as u64, Relaxed); + } +} + #[derive(Clone, Default)] pub(crate) struct Mailbox(Arc); impl Mailbox { diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index c491f8a7..5385a8a9 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -213,6 +213,7 @@ impl JobState { pub(crate) fn cancel(&self) { trace::atomic(|| { if self.core.cancel() { + self.count_cancel(); self.record(|job| trace::JobEvent::new("Cancel", job)); } }); @@ -220,10 +221,19 @@ impl JobState { } fn cancel_quietly(&self) { - self.core.cancel(); + if self.core.cancel() { + self.count_cancel(); + } self.wake_cancelled(); } + /// `spawned_jobs_cancelled_total` counts jobs cancelled while running. + fn count_cancel(&self) { + if self.core.is_running() { + crate::instrumentation::hooks::job_cancelled(); + } + } + fn wake_cancelled(&self) { #[cfg(feature = "async")] { @@ -241,15 +251,19 @@ impl JobState { } /// Take the terminal notification and run `enqueue` under the job's - /// lock (see [`JobCore::notify`]). `deadline` tells the job deadline from - /// the end of execution (the model's `JobDeadline` and `Finish`). + /// lock (see [`JobCore::notify`]). `end` is `ok`, `panicked` (the end + /// of execution, the model's `Finish`) or `deadline` (`JobDeadline`). pub(crate) fn notify( &self, - deadline: bool, + end: &'static str, enqueue: impl FnOnce() -> Option, ) -> (bool, Option) { + let deadline = end == "deadline"; let (taken, unsent) = trace::atomic(|| { let (taken, unsent) = self.core.notify(enqueue); + if taken { + crate::instrumentation::hooks::job_notified(end); + } if !deadline { self.record(|job| trace::JobEvent { x: if taken { "notified" } else { "quiet" }, @@ -287,6 +301,9 @@ impl JobState { pub(crate) fn fenced(&self, at: ActorId, kind: &'static str) -> bool { trace::atomic(|| { let fenced = self.is_cancelled() || self.owner != at; + if fenced { + crate::instrumentation::hooks::job_fenced(); + } self.record(|job| trace::JobEvent { x: kind, fenced, @@ -327,6 +344,7 @@ impl JobState { trace::atomic(|| { let result = self.core.enqueue_emit(enqueue); if result.is_ok() { + crate::instrumentation::hooks::job_emitted(); self.record(|job| trace::JobEvent::new("Emit", job)); } result @@ -422,7 +440,11 @@ impl JobSet { /// Reserve a native slot (`MAX_NATIVE_JOBS`), rejecting at the bound. pub(crate) fn reserve_native(&self) -> bool { - self.core.reserve_native() + let reserved = self.core.reserve_native(); + if !reserved { + crate::instrumentation::hooks::job_rejected(); + } + reserved } pub(crate) fn release_native(&self) { @@ -435,6 +457,7 @@ impl JobSet { trace::atomic(|| { let keyed = key.is_some(); self.core.admit(state, key); + crate::instrumentation::hooks::job_started(); state.record(|job| trace::JobEvent { keyed, deadline, @@ -445,6 +468,7 @@ impl JobSet { /// A job's execution ended. pub(crate) fn ended(&self, state: &Arc) { + crate::instrumentation::hooks::job_ended(); if self.core.ended(state) { #[cfg(feature = "async")] for waker in std::mem::take(&mut *lock(&self.idle_async)) { @@ -504,6 +528,7 @@ impl JobSet { /// Report jobs still running when the owner's cleanup deadline passed. pub(crate) fn report_unfinished(owner: ActorId, unfinished: usize) { if unfinished > 0 { + crate::instrumentation::hooks::jobs_unfinished(unfinished); tracing::warn!( event = "unfinished_at_stop", actor = %owner, @@ -524,7 +549,11 @@ impl JobCredit { pub(crate) fn reserve(registry: &Registry) -> Option { let pool = registry.pool().clone(); // `then`, not `then_some`: the guard must exist only when reserved. - pool.reserve(1).then(|| Self(Some(pool))) + let credit = pool.reserve(1).then(|| Self(Some(pool))); + if credit.is_none() { + crate::instrumentation::hooks::job_rejected(); + } + credit } } diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index a02a1dd7..256abc9b 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -42,10 +42,12 @@ mod sync { crate::lifecycle_trace::atomic(f) } - /// A slot left `OPEN` (`kind` is the model's terminal name). + /// A slot left `OPEN` (`kind` is the model's terminal name; `token` + /// whether a deferred token was live): metrics and lifecycle traces. #[inline(always)] #[cfg_attr(not(feature = "lifecycle-trace"), allow(unused_variables))] - pub(crate) fn traced_terminal(owner: &SlotOwner, slot: usize, kind: &'static str) { + pub(crate) fn left_open(owner: &SlotOwner, slot: usize, kind: &'static str, token: bool) { + crate::instrumentation::hooks::left_open(kind, token); #[cfg(feature = "lifecycle-trace")] if let Some(registry) = owner { crate::lifecycle_trace::reply(registry.owner().raw(), "Discharge", slot as u64, kind); @@ -277,6 +279,7 @@ impl<'t, R: Send + 'static> PendingReply<'t, R> { Ok(ReplyTo { slot: Some(slot), registry: registry.clone(), + stamp: crate::instrumentation::hooks::deferred(), }) } Err(slot::Refused::Closed) => Err(DeferError::Closed), @@ -338,6 +341,8 @@ impl Drop for PendingReply<'_, R> { pub struct ReplyTo { slot: Option>>, registry: Arc, + /// When it was deferred (sampled; zero-sized without `metrics`). + stamp: crate::instrumentation::hooks::DeferStamp, } impl std::fmt::Debug for ReplyTo { @@ -358,6 +363,9 @@ impl ReplyTo { let (delivery, unsent) = slot.reply(value, &now_ticks); drop(unsent); record(delivery); + if delivery == Delivery::Delivered { + crate::instrumentation::hooks::deferred_replied(&self.stamp); + } delivery } diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 72458646..76151ff6 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -302,10 +302,11 @@ impl Slot { self.word .compare_exchange(current, next, Ordering::AcqRel, Ordering::Acquire); if exchanged.is_ok() && state(current) == OPEN && state(next) != OPEN { - super::sync::traced_terminal( + super::sync::left_open( &self.owner, self as *const Self as usize, trace_name(state(next)), + current & TOKEN != 0, ); } exchanged diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index deced057..95d19941 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -131,8 +131,12 @@ where { /// Take the terminal notification and enqueue `(done, outcome)`. fn notify(&self, outcome: JobOutcome) { - let deadline = matches!(outcome, JobOutcome::Err(JobError::DeadlineExceeded)); - let (_, unsent) = self.state.notify(deadline, || { + let end = match &outcome { + JobOutcome::Ok(_) => "ok", + JobOutcome::Err(JobError::Panicked(_)) => "panicked", + JobOutcome::Err(JobError::DeadlineExceeded) => "deadline", + }; + let (_, unsent) = self.state.notify(end, || { let done = self.done.lock().unwrap_or_else(|p| p.into_inner()).take()?; let envelope: Box + Send> = Box::new(TerminalEnvelope { job: self.state.clone(), diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index 50dc425b..08aab7dd 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -109,8 +109,12 @@ where { /// Take the terminal notification and enqueue `(done, outcome)`. fn notify(&self, outcome: JobOutcome) { - let deadline = matches!(outcome, JobOutcome::Err(JobError::DeadlineExceeded)); - let (_, unsent) = self.state.notify(deadline, || { + let end = match &outcome { + JobOutcome::Ok(_) => "ok", + JobOutcome::Err(JobError::Panicked(_)) => "panicked", + JobOutcome::Err(JobError::DeadlineExceeded) => "deadline", + }; + let (_, unsent) = self.state.notify(end, || { let done = self.done.lock().unwrap_or_else(|p| p.into_inner()).take()?; let envelope: Box + Send> = Box::new(TerminalEnvelope { job: self.state.clone(), diff --git a/concurrency/tests/job_core_loom.rs b/concurrency/tests/job_core_loom.rs index 620104d0..958ea2a2 100644 --- a/concurrency/tests/job_core_loom.rs +++ b/concurrency/tests/job_core_loom.rs @@ -20,7 +20,7 @@ mod sync { pub(crate) fn section(f: impl FnOnce() -> T) -> T { f() } - pub(crate) fn traced_terminal(_: &SlotOwner, _: usize, _: &'static str) {} + pub(crate) fn left_open(_: &SlotOwner, _: usize, _: &'static str, _: bool) {} pub(crate) use super::model_arc::{JobArc, JobWeak}; /// Loom has no timed wait; the model never parks on a slot. diff --git a/concurrency/tests/metrics_deferred.rs b/concurrency/tests/metrics_deferred.rs new file mode 100644 index 00000000..c0ca92d0 --- /dev/null +++ b/concurrency/tests/metrics_deferred.rs @@ -0,0 +1,255 @@ +//! `spawned_deferred_*` and `spawned_jobs_*` metric families: every deferred +//! token and job outcome is counted once, and the gauges return to zero. +//! One test per binary: the counters are process-wide. +#![cfg(feature = "async")] +#![cfg(feature = "metrics")] +use spawned_concurrency::{ + message::Message, + metrics::{self, Config}, + tasks::{ + Actor, ActorRef, ActorStart, Context, Deadline, Handler, JobCtx, JobOptions, JobOutcome, + PendingReply, ReplyTo, + }, + ActorError, +}; +use spawned_rt::tasks::{oneshot, sleep}; +use std::time::Duration; + +#[derive(Default)] +struct Owner { + stored: Vec>, + gates: Vec>, +} +impl Actor for Owner { + const MAX_OUTSTANDING: usize = 2; + const MAX_NATIVE_JOBS: Option = Some(1); +} + +struct Hold; +impl Message for Hold { + type Result = u32; +} +impl Handler for Owner { + async fn handle(&mut self, _: Hold, reply: PendingReply<'_, u32>, _: &Context) { + if let Ok(token) = reply.defer() { + self.stored.push(token); + } + } +} + +enum Flush { + Reply, + Drop, +} +impl Message for Flush { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: Flush, reply: PendingReply<'_, ()>, _: &Context) { + for token in self.stored.drain(..) { + match msg { + Flush::Reply => { + token.reply(1); + } + Flush::Drop => drop(token), + } + } + reply.reply(()); + } +} + +struct Sync; +impl Message for Sync { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: Sync, reply: PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } +} + +struct Done; +impl Message for Done { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: Done, _: PendingReply<'_, ()>, _: &Context) {} +} +fn done(_: JobOutcome) -> Done { + Done +} + +enum Jobs { + Ok, + Panics, + Deadline, + Emits, + /// A native job finished, then cancelled: its queued result is fenced. + Fenced, + /// A gated native job holds the budget; a second one is rejected. + Budget, + Cancel, +} +impl Message for Jobs { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: Jobs, reply: PendingReply<'_, ()>, ctx: &Context) { + match msg { + Jobs::Ok => drop(ctx.pipe(async { 1 }, done).unwrap()), + Jobs::Panics => drop(ctx.pipe(async { panic!("job") }, done::<()>).unwrap()), + Jobs::Deadline => { + let opts = JobOptions { + deadline: Some(spawned_rt::time::Instant::now() + Duration::from_millis(10)), + ..Default::default() + }; + ctx.pipe_with(opts, sleep(Duration::from_millis(60)), done) + .unwrap(); + } + Jobs::Emits => { + ctx.pipe_with_ctx( + JobOptions::default(), + |job: JobCtx| async move { + job.emit(Done).await.unwrap(); + job.emit(Done).await.unwrap(); + }, + done, + ) + .unwrap(); + } + Jobs::Fenced => { + let job = ctx + .pipe_blocking_with(JobOptions::default(), |_: &JobCtx| 1, done) + .unwrap(); + while job.is_running() { + std::thread::sleep(Duration::from_millis(1)); + } + job.cancel(); + } + Jobs::Budget => { + let (open, gate) = std::sync::mpsc::channel::<()>(); + ctx.pipe_blocking_with( + JobOptions::default(), + move |_: &JobCtx| gate.recv().ok(), + done, + ) + .unwrap(); + assert!(ctx + .pipe_blocking_with(JobOptions::default(), |_: &JobCtx| (), done) + .is_err()); + open.send(()).unwrap(); + } + Jobs::Cancel => { + let (open, gate) = oneshot::channel::<()>(); + let job = ctx.pipe(async move { gate.await.is_ok() }, done).unwrap(); + job.cancel(); + self.gates.push(open); + } + } + reply.reply(()); + } +} + +/// Wait until no job runs and their messages have been dispatched. +async fn settle(actor: &ActorRef) { + for _ in 0..400 { + if metrics::snapshot().jobs_running == 0 { + break; + } + sleep(Duration::from_millis(5)).await; + } + actor.request(Sync).await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn deferred_and_job_families_count_each_outcome_once() { + metrics::configure(Config { + timing_sample_every: 1, + }) + .unwrap(); + let before = metrics::snapshot(); + let actor = Owner::default().start(); + + // Replied, and Overloaded at the bound. + let first = actor.request_raw(Hold).unwrap(); + let second = actor.request_raw(Hold).unwrap(); + assert!(matches!( + actor.request(Hold).await, + Err(ActorError::Overloaded) + )); + actor.request(Flush::Reply).await.unwrap(); + assert!(matches!(first.await, Ok(1))); + assert!(matches!(second.await, Ok(1))); + // Expired, dropped, abandoned. + let deadline = Deadline::At(spawned_rt::time::Instant::now() + Duration::from_millis(20)); + let expiring = actor.request_raw_until(Hold, deadline).unwrap(); + assert!(matches!(expiring.await, Err(ActorError::RequestTimeout))); + let dropped = actor.request_raw(Hold).unwrap(); + actor.request(Flush::Drop).await.unwrap(); + assert!(matches!(dropped.await, Err(ActorError::ReplyDropped))); + let abandoned = actor.request_raw(Hold).unwrap(); + actor.request(Sync).await.unwrap(); + drop(abandoned); + actor.request(Flush::Reply).await.unwrap(); + + for jobs in [ + Jobs::Ok, + Jobs::Panics, + Jobs::Deadline, + Jobs::Emits, + Jobs::Fenced, + Jobs::Budget, + Jobs::Cancel, + ] { + actor.request(jobs).await.unwrap(); + // One job at a time: they share the actor's two credits. + settle(&actor).await; + } + + // Stopped: a token still stored at death. + let stopped = actor.request_raw(Hold).unwrap(); + actor.request(Flush::Reply).await.unwrap(); + let late = actor.request_raw(Hold).unwrap(); + sleep(Duration::from_millis(20)).await; + actor.context().stop(); + assert!(matches!(late.await, Err(ActorError::ActorStopped))); + assert!(matches!(stopped.await, Ok(1))); + actor.join().await; + + let after = metrics::snapshot(); + let d = |f: fn(&metrics::Snapshot) -> u64| f(&after) - f(&before); + assert_eq!(d(|s| s.deferred_tokens_total), 7, "{after:#?}"); + assert_eq!(after.deferred_pending, before.deferred_pending); + assert_eq!(d(|s| s.deferred_replied_total), 3); + assert_eq!(d(|s| s.deferred_overloaded_total), 1); + assert_eq!(d(|s| s.deferred_expired_total), 1); + assert_eq!(d(|s| s.deferred_dropped_total), 1); + assert_eq!(d(|s| s.deferred_abandoned_total), 1); + assert_eq!(d(|s| s.deferred_stopped_total), 1); + assert_eq!( + after.deferred_reply_latency.count - before.deferred_reply_latency.count, + 3 + ); + assert_eq!(d(|s| s.jobs_started_total), 7); + assert_eq!(after.jobs_running, before.jobs_running); + // Ok, Emits, Budget's gated job, Fenced's (taken before the cancel). + assert_eq!(d(|s| s.jobs_completed_total), 4); + assert_eq!(d(|s| s.jobs_panicked_total), 1); + assert_eq!(d(|s| s.jobs_deadline_exceeded_total), 1); + assert_eq!(d(|s| s.jobs_emits_total), 2); + assert_eq!(d(|s| s.jobs_fenced_total), 1); + assert_eq!(d(|s| s.jobs_rejected_total), 1); + // Only Cancel's job was running when cancelled (Fenced's had finished; + // the owner's exit finds none running). + assert_eq!(d(|s| s.jobs_cancelled_total), 1); + let text = metrics::render_prometheus(); + for family in [ + "spawned_deferred_pending ", + "spawned_deferred_replied_total ", + "spawned_deferred_reply_latency_seconds_count ", + "spawned_jobs_running ", + "spawned_jobs_fenced_total ", + ] { + assert!(text.contains(family), "{family} missing"); + } +} diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index 87ff94a6..28d509c1 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -15,7 +15,7 @@ mod sync { pub(crate) fn section(f: impl FnOnce() -> T) -> T { f() } - pub(crate) fn traced_terminal(_: &SlotOwner, _: usize, _: &'static str) {} + pub(crate) fn left_open(_: &SlotOwner, _: usize, _: &'static str, _: bool) {} /// Loom has no timed wait; tests park only on slots without a deadline. pub(crate) fn wait_ticks<'a, T>( diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 25b6c5e6..7a09b32a 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -1183,6 +1183,8 @@ counters and histograms still aggregate the whole process. | Request outcomes | Started, active, replied, timed out, actor stopped, and caller cancelled. Transport replies are successful even if the application result itself contains an error. | | Lifecycle | Actor instances admitted, active, and exited by Normal/Shutdown/Panic/Kill; handler starts, active handlers, completions, panics and interrupted futures. | | Delivery accounting | Enqueue acceptance/rejection, discarded queued messages, replies sent to channels, and reply receivers already dropped. Enqueue success does not promise handling or durable delivery. | +| Deferred replies (`spawned_deferred_*`) | `deferred_tokens_total` (successful `defer()`), `deferred_pending` (open slots with a live token, all actors), how each deferred slot ended: `replied`, `expired` (its deadline), `dropped`, `stopped` (owner exit or death), `abandoned` (caller gave up), each `_total`; `deferred_overloaded_total` (every `Overloaded` answer, at `defer()` or `ReplyTo::overloaded`); `deferred_reply_latency_seconds`, a sampled histogram from `defer()` to a delivered reply. | +| Jobs (`spawned_jobs_*`) | `jobs_started_total` (admitted), `jobs_running` (executing, cancelled or timed-out work included), terminal notifications `jobs_completed_total`, `jobs_panicked_total`, `jobs_deadline_exceeded_total`; `jobs_cancelled_total` (cancelled while still running: handle, key replacement, owner exit, caller give-up); `jobs_fenced_total` (job messages dropped at dispatch); `jobs_rejected_total` (at the credit bound or the native budget); `jobs_emits_total`; `jobs_unfinished_at_stop_total` (still running at the owner's cleanup deadline). | Counters end in `_total`. Duration histograms use fixed nanosecond buckets internally and seconds in Prometheus export; their count/sum/max describe sampled @@ -1203,6 +1205,11 @@ conservation relationships apply after operations quiesce: - handled = completed handlers + panicked handlers + aborted handlers + active; - observed requests = replied + timed out + actor stopped + cancelled + active; - spawned actor instances = exited + active. +- deferred tokens = replied + expired + dropped + stopped + abandoned + + pending (an `Overloaded` defer takes no token); +- started jobs = running + jobs whose execution ended (a job's terminal + notification is completed, panicked, deadline exceeded, or none when it was + cancelled first). An actor is active from creation of its lifecycle guard until terminal cleanup, including waiting for a blocking-pool slot, `started()` and `stopped()`. A runtime diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 2dfcacd9..4120bed4 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -46,7 +46,9 @@ provided). A `recv_timeout` on the old native receiver has no counterpart: the deadline is fixed when the request is sent. **Metrics.** `metrics::Snapshot` gains `requests_reply_dropped_total` and -`requests_overloaded_total`. +`requests_overloaded_total`, the `deferred_*` and `jobs_*` fields (see the +GUIDE's Metrics section) and the `deferred_reply_latency` histogram. Code +that builds a `Snapshot` with a struct literal must add them. **Lifecycle.** At death the registry closes (open deferred tokens answer `ActorStopped`), then the mailbox is disposed, then actor state is dropped: From 56f061e5c88c9e01c2db2f07d541da700a451513 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 01:32:32 +0000 Subject: [PATCH 059/208] test(sim): deferred reply and job races under PCT and bounded DFS --- CHANGELOG.md | 4 +- concurrency/tests/deferred_sim.rs | 342 ++++++++++++++++++++++++++++++ docs/TESTING.md | 14 ++ 3 files changed, 359 insertions(+), 1 deletion(-) create mode 100644 concurrency/tests/deferred_sim.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c9901c0..c0bac4f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,9 @@ `spawned_jobs_*` family (started, running, completed/panicked/deadline exceeded, cancelled while running, fenced, rejected at a bound, emits, unfinished at stop); `tests/metrics_deferred.rs` checks each is counted - once. `Handler::handle` takes a `PendingReply<'_, R>` + once. Simulator races (`tests/deferred_sim.rs`, PCT and bounded DFS): + reply vs expiry vs owner death vs caller drop, keyed replacement, emit vs + terminal and cancel. `Handler::handle` takes a `PendingReply<'_, R>` and returns `()`; `#[actor]` generates it, and `#[deferred_handler]` methods receive the obligation and may `defer()` it into a storable `ReplyTo` (bounded by `Actor::MAX_OUTSTANDING`, `Overloaded` at the bound). Dropped diff --git a/concurrency/tests/deferred_sim.rs b/concurrency/tests/deferred_sim.rs new file mode 100644 index 00000000..38c747d4 --- /dev/null +++ b/concurrency/tests/deferred_sim.rs @@ -0,0 +1,342 @@ +//! Simulator races for deferred replies and pipe-to-self jobs +//! (docs/design/deferred-replies.md), explored with PCT and bounded DFS over +//! real actors: a stored token's reply against its deadline, the owner's +//! death and the caller leaving; keyed replacement against the old job's +//! completion; emits against the terminal message and a cancel. Run with: +//! +//! ```sh +//! RUSTFLAGS="--cfg spawned_sim --cfg tokio_unstable" cargo test -p spawned-concurrency --test deferred_sim +//! ``` +#![cfg(all(spawned_sim, feature = "async"))] + +use spawned_concurrency::{ + message::Message, + tasks::{ + Actor, ActorRef, ActorStart, Context, Deadline, Delivery, Handler, JobCtx, JobHandle, + JobOptions, JobOutcome, PendingReply, ReplyTo, + }, + ActorError, +}; +use spawned_rt::tasks::{ + sim::explore::{explore, Strategy}, + sleep, spawn, +}; +use std::{ + future::Future, + sync::{Arc, Mutex}, + time::Duration, +}; + +type Log = Arc>>; + +fn push(log: &Log, entry: impl Into) { + log.lock().unwrap().push(entry.into()); +} + +fn entries(log: &Log) -> Vec { + log.lock().unwrap().clone() +} + +struct Owner { + log: Log, + stored: Vec>, + job: Option, +} +impl Actor for Owner {} + +fn owner(log: &Log) -> ActorRef { + Owner { + log: log.clone(), + stored: Vec::new(), + job: None, + } + .start() +} + +struct Hold; +impl Message for Hold { + type Result = u32; +} +impl Handler for Owner { + async fn handle(&mut self, _: Hold, reply: PendingReply<'_, u32>, _: &Context) { + if let Ok(token) = reply.defer() { + self.stored.push(token); + } + } +} + +struct Sync; +impl Message for Sync { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: Sync, reply: PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } +} + +/// Answer every stored token and log how each delivery went. +struct Flush; +impl Message for Flush { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: Flush, _: PendingReply<'_, ()>, _: &Context) { + for token in self.stored.drain(..) { + let delivery = token.reply(7); + push(&self.log, format!("{delivery:?}")); + } + } +} + +struct StartKeyed(&'static str, u64); +impl Message for StartKeyed { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: StartKeyed, _: PendingReply<'_, ()>, ctx: &Context) { + let StartKeyed(name, delay) = msg; + ctx.pipe_keyed( + "k", + async move { + sleep(Duration::from_millis(delay)).await; + name + }, + |outcome: JobOutcome<&'static str>| Done(outcome.into_result().unwrap()), + ) + .unwrap(); + push(&self.log, format!("start:{name}")); + } +} + +struct Done(&'static str); +impl Message for Done { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: Done, _: PendingReply<'_, ()>, _: &Context) { + push(&self.log, format!("done:{}", msg.0)); + } +} + +struct Tick(u32); +impl Message for Tick { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: Tick, _: PendingReply<'_, ()>, _: &Context) { + push(&self.log, format!("tick:{}", msg.0)); + } +} + +struct StartEmitter; +impl Message for StartEmitter { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: StartEmitter, _: PendingReply<'_, ()>, ctx: &Context) { + let opts = JobOptions { + emit_credits: 1, + ..Default::default() + }; + let job = ctx + .pipe_with_ctx( + opts, + |job: JobCtx| async move { + for i in 0..3 { + if job.emit(Tick(i)).await.is_err() { + break; + } + sleep(Duration::from_millis(1)).await; + } + "emitter" + }, + |outcome: JobOutcome<&'static str>| Done(outcome.into_result().unwrap()), + ) + .unwrap(); + self.job = Some(job); + } +} + +struct CancelJob; +impl Message for CancelJob { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: CancelJob, _: PendingReply<'_, ()>, _: &Context) { + if let Some(job) = self.job.take() { + job.cancel(); + push(&self.log, "cancelled"); + } + } +} + +// --------------------------------------------------------------------------- +// Bodies +// --------------------------------------------------------------------------- + +/// A stored token's reply races its deadline, the owner's stop and (on odd +/// seeds) the caller dropping its receiver. The caller sees `Ok` exactly when +/// the reply reports `Delivered`, and always gets an answer. +async fn reply_vs_expiry_vs_death_vs_drop(seed: u64) { + let log = Log::default(); + let actor = owner(&log); + let d = Duration::from_millis(10); + let receiver = actor + .request_raw_until(Hold, Deadline::At(spawned_rt::time::Instant::now() + d)) + .unwrap(); + actor.request(Sync).await.unwrap(); + let replier = actor.clone(); + spawn(async move { + sleep(d).await; + let _ = replier.send(Flush); + }); + let stopper = actor.clone(); + spawn(async move { + sleep(d).await; + stopper.context().stop(); + }); + let awaited = seed % 2 == 0; + let caller = spawn(async move { + if awaited { + Some(receiver.await) + } else { + sleep(d).await; + drop(receiver); + None + } + }); + let result = caller.await.unwrap(); + actor.join().await; + let deliveries = entries(&log); + assert!(deliveries.len() <= 1, "{deliveries:?}"); + let delivered = deliveries.first().map(String::as_str) == Some("Delivered"); + match result { + Some(Ok(7)) => assert!(delivered, "Ok without a delivery: {deliveries:?}"), + Some(Ok(other)) => panic!("unexpected value {other}"), + Some(Err(e)) => { + assert!(!delivered, "{e:?} after a delivery"); + assert!( + matches!(e, ActorError::RequestTimeout | ActorError::ActorStopped), + "{e:?}" + ); + } + None => {} + } + if let Some(delivery) = deliveries.first() { + let expected = [ + format!("{:?}", Delivery::Delivered), + format!("{:?}", Delivery::Expired), + format!("{:?}", Delivery::CallerGone), + ]; + assert!(expected.contains(delivery), "{delivery}"); + } +} + +/// A keyed job is replaced while it completes: its result never reaches a +/// turn after the replacement started. +async fn keyed_replacement(seed: u64) { + let log = Log::default(); + let actor = owner(&log); + actor.send(StartKeyed("a", 5)).unwrap(); + let replacer = actor.clone(); + spawn(async move { + sleep(Duration::from_millis(4 + seed % 3)).await; + let _ = replacer.send(StartKeyed("b", 1)); + }); + for _ in 0..100 { + if entries(&log).iter().any(|e| e == "done:b") { + break; + } + sleep(Duration::from_millis(1)).await; + } + actor.request(Sync).await.unwrap(); + actor.context().stop(); + actor.join().await; + let log = entries(&log); + assert!(log.contains(&"done:b".to_string()), "{log:?}"); + let replaced = log.iter().position(|e| e == "start:b").unwrap(); + assert!( + !log[replaced..].contains(&"done:a".to_string()), + "a replaced job's result after its replacement: {log:?}" + ); +} + +/// Emits race the terminal message and a cancel: no tick after the terminal +/// message, and nothing from the job after the cancelling turn. +async fn emit_vs_terminal_and_cancel(seed: u64) { + let log = Log::default(); + let actor = owner(&log); + actor.send(StartEmitter).unwrap(); + let canceller = actor.clone(); + spawn(async move { + sleep(Duration::from_millis(seed % 5)).await; + let _ = canceller.send(CancelJob); + }); + sleep(Duration::from_millis(10)).await; + actor.request(Sync).await.unwrap(); + actor.context().stop(); + actor.join().await; + let log = entries(&log); + if let Some(done) = log.iter().position(|e| e.starts_with("done:")) { + assert!( + !log[done + 1..].iter().any(|e| e.starts_with("tick:")), + "an emit after the terminal message: {log:?}" + ); + } + if let Some(cancelled) = log.iter().position(|e| e == "cancelled") { + assert!( + log[cancelled + 1..].is_empty(), + "a cancelled job's message dispatched: {log:?}" + ); + } + let ticks: Vec<_> = log.iter().filter(|e| e.starts_with("tick:")).collect(); + let in_order = ticks + .iter() + .enumerate() + .all(|(i, t)| **t == format!("tick:{i}")); + assert!(in_order, "emits out of order: {log:?}"); +} + +// --------------------------------------------------------------------------- +// Driver +// --------------------------------------------------------------------------- + +const STRATEGIES: [(&str, usize); 2] = [("pct:d=3", 300), ("dfs:bound=3,depth=300", 300)]; + +fn check(name: &str, body: F) +where + F: Fn(u64) -> Fut, + Fut: Future, +{ + for (strategy, budget) in STRATEGIES { + let report = explore(&Strategy::parse(strategy).unwrap(), 0, budget, &body); + assert!( + report.found.is_none(), + "{name} under {strategy}: {:?}", + report.found + ); + println!( + "{name} {strategy}: {} runs, {} schedules", + report.runs, report.schedules + ); + } +} + +#[test] +fn reply_races_expiry_death_and_caller_drop() { + check( + "reply vs expiry vs death vs drop", + reply_vs_expiry_vs_death_vs_drop, + ); +} + +#[test] +fn keyed_replacement_fences_the_completing_job() { + check("keyed replacement", keyed_replacement); +} + +#[test] +fn emits_never_follow_the_terminal_message_or_a_cancel() { + check("emit vs terminal and cancel", emit_vs_terminal_and_cancel); +} diff --git a/docs/TESTING.md b/docs/TESTING.md index 685eb215..1ea47649 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -400,6 +400,20 @@ RUSTFLAGS="--cfg spawned_sim --cfg tokio_unstable" CARGO_TARGET_DIR=target/sim \ cargo test -p spawned-concurrency --test supervision --test sim ``` +`deferred_sim` explores deferred-reply and job races with PCT (`d=3`) and +bounded DFS (`bound=3`), 300 runs each: a stored token's reply against its +deadline, the owner's stop and the caller dropping its receiver (the caller +sees `Ok` exactly when the reply reports `Delivered`); keyed replacement +against the old job's completion (no result of a replaced job after its +replacement starts); emits against the terminal message and a cancel. Planted +bugs (no cancel fence at dispatch; a late reply reported `Delivered`) are +found within the first runs. + +```sh +RUSTFLAGS="--cfg spawned_sim --cfg tokio_unstable" CARGO_TARGET_DIR=target/sim \ + cargo test -p spawned-concurrency --test deferred_sim --test deferred_replies --test deferred_jobs +``` + ### Exploration strategies (`sim::explore`) The random walk can be swapped for a systematic strategy with From 80e9562f5d7de744b9044e1c2763d8fc5f6bc337 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 01:41:27 +0000 Subject: [PATCH 060/208] fix(trace): record deferred reply and job events only for watched owners --- concurrency/src/lifecycle_trace.rs | 28 ++++++++++++++++++++++++++-- concurrency/tests/job_trace.rs | 1 + concurrency/tests/reply_trace.rs | 25 ++++++++++++++++--------- verification/tla/README.md | 3 ++- 4 files changed, 45 insertions(+), 12 deletions(-) diff --git a/concurrency/src/lifecycle_trace.rs b/concurrency/src/lifecycle_trace.rs index 789dc786..c8166f0b 100644 --- a/concurrency/src/lifecycle_trace.rs +++ b/concurrency/src/lifecycle_trace.rs @@ -213,6 +213,21 @@ mod imp { static SECTION: Mutex<()> = Mutex::new(()); static TRACES: Mutex>> = Mutex::new(BTreeMap::new()); static JOBS: Mutex>> = Mutex::new(BTreeMap::new()); + /// Owner instances whose reply-slot and job events are recorded. + static WATCHED: Mutex> = + Mutex::new(std::collections::BTreeSet::new()); + + /// Record the reply-slot and job events of owner instance `owner` (an + /// `ActorId`'s number) until [`take_replies`] or [`take_jobs`]. Opt-in, so + /// other tests built with the feature neither accumulate events nor + /// allocate inside the trace section. + pub fn watch_deferred(owner: u64) { + lock(&WATCHED).insert(owner); + } + + fn watched(owner: u64) -> bool { + lock(&WATCHED).contains(&owner) + } /// A job number for [`super::JobEvent::job`]. pub(crate) fn next_job() -> u64 { @@ -222,7 +237,11 @@ mod imp { /// Record a job action of owner instance `owner` (inside the section). pub(crate) fn job(owner: u64, event: impl FnOnce() -> super::JobEvent) { - atomic(|| lock(&JOBS).entry(owner).or_default().push(event())); + atomic(|| { + if watched(owner) { + lock(&JOBS).entry(owner).or_default().push(event()); + } + }); } static REPLIES: Mutex>> = Mutex::new(BTreeMap::new()); @@ -232,6 +251,9 @@ mod imp { /// Record a reply-slot action of owner instance `owner`. pub(crate) fn reply(owner: u64, action: &'static str, req: u64, x: &'static str) { atomic(|| { + if !watched(owner) { + return; + } // The owner's death closes everything still open in one model // step (`Death`); the closes it performs are not separate events. if action == "Discharge" && x == "stopped" && lock(&DEAD).contains(&owner) { @@ -249,12 +271,14 @@ mod imp { /// Remove and return the reply events recorded for an owner instance. pub fn take_replies(owner: u64) -> Vec { + lock(&WATCHED).remove(&owner); lock(&DEAD).remove(&owner); lock(&REPLIES).remove(&owner).unwrap_or_default() } /// Remove and return the job events recorded for an owner instance. pub fn take_jobs(owner: u64) -> Vec { + lock(&WATCHED).remove(&owner); lock(&JOBS).remove(&owner).unwrap_or_default() } thread_local! { @@ -540,4 +564,4 @@ mod imp { } #[cfg(feature = "lifecycle-trace")] -pub use imp::{take, take_jobs, take_replies, take_tree}; +pub use imp::{take, take_jobs, take_replies, take_tree, watch_deferred}; diff --git a/concurrency/tests/job_trace.rs b/concurrency/tests/job_trace.rs index dd4b82b8..14b45f0c 100644 --- a/concurrency/tests/job_trace.rs +++ b/concurrency/tests/job_trace.rs @@ -223,6 +223,7 @@ fn start_with(old: OldJob) -> (ActorRef, Arc) { gates: Vec::new(), } .start(); + lifecycle_trace::watch_deferred(actor.id().to_string().parse().unwrap()); (actor, log) } diff --git a/concurrency/tests/reply_trace.rs b/concurrency/tests/reply_trace.rs index f2d6b099..0c2940e4 100644 --- a/concurrency/tests/reply_trace.rs +++ b/concurrency/tests/reply_trace.rs @@ -110,6 +110,13 @@ impl Handler for Owner { } } +/// Start an owner whose reply slots are traced. +fn start() -> ActorRef { + let actor = Owner::default().start(); + lifecycle_trace::watch_deferred(id(&actor)); + actor +} + fn id(actor: &ActorRef) -> u64 { actor.id().to_string().parse().unwrap() } @@ -158,7 +165,7 @@ async fn stop(name: &str, actor: ActorRef) -> Vec { #[tokio::test] async fn reply_in_turn() { - let actor = Owner::default().start(); + let actor = start(); let got = actor.request(Echo(3)).await; assert!(matches!(got, Ok(3)), "{got:?}"); let events = stop("in_turn", actor).await; @@ -170,7 +177,7 @@ async fn reply_in_turn() { #[tokio::test] async fn stored_token_answered_later() { - let actor = Owner::default().start(); + let actor = start(); let held = actor.request_raw(Hold).unwrap(); actor.request(Sync).await.unwrap(); actor.send(Flush::Reply).unwrap(); @@ -181,7 +188,7 @@ async fn stored_token_answered_later() { #[tokio::test] async fn stored_token_dropped_later() { - let actor = Owner::default().start(); + let actor = start(); let held = actor.request_raw(Hold).unwrap(); actor.request(Sync).await.unwrap(); actor.send(Flush::Drop).unwrap(); @@ -192,7 +199,7 @@ async fn stored_token_dropped_later() { #[tokio::test] async fn unused_obligation_answers_dropped() { - let actor = Owner::default().start(); + let actor = start(); let got = actor.request(Ignore).await; assert!(matches!(got, Err(ActorError::ReplyDropped)), "{got:?}"); stop("unused", actor).await; @@ -200,7 +207,7 @@ async fn unused_obligation_answers_dropped() { #[tokio::test] async fn defer_at_the_bound_is_overloaded() { - let actor = Owner::default().start(); + let actor = start(); let first = actor.request_raw(Hold).unwrap(); let got = actor.request(Hold).await; assert!(matches!(got, Err(ActorError::Overloaded)), "{got:?}"); @@ -212,7 +219,7 @@ async fn defer_at_the_bound_is_overloaded() { #[tokio::test] async fn stored_token_expires_at_its_deadline() { - let actor = Owner::default().start(); + let actor = start(); let deadline = Deadline::At(spawned_rt::time::Instant::now() + Duration::from_millis(30)); let held = actor.request_raw_until(Hold, deadline).unwrap(); let got = held.await; @@ -224,7 +231,7 @@ async fn stored_token_expires_at_its_deadline() { #[tokio::test] async fn caller_gives_up() { - let actor = Owner::default().start(); + let actor = start(); let held = actor.request_raw(Hold).unwrap(); actor.request(Sync).await.unwrap(); drop(held); @@ -235,7 +242,7 @@ async fn caller_gives_up() { #[tokio::test] async fn death_closes_stored_tokens() { - let actor = Owner::default().start(); + let actor = start(); let held = actor.request_raw(Hold).unwrap(); actor.request(Sync).await.unwrap(); let owner = id(&actor); @@ -252,7 +259,7 @@ async fn death_closes_stored_tokens() { #[tokio::test] async fn panic_in_turn_answers_stopped() { - let actor = Owner::default().start(); + let actor = start(); let got = actor.request(Boom).await; assert!(matches!(got, Err(ActorError::ActorStopped)), "{got:?}"); actor.join().await; diff --git a/verification/tla/README.md b/verification/tla/README.md index 1fcfd339..b27e0217 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -575,7 +575,8 @@ All 22 named controls find their counterexample, each in a few seconds. ### Job trace validation -With `lifecycle-trace`, the job core (`concurrency/src/reply/jobs.rs`) records +With `lifecycle-trace`, for owners a test opts in with +`lifecycle_trace::watch_deferred(id)`, the job core (`concurrency/src/reply/jobs.rs`) records one `DeferredJobs.tla` action per owner instance: `Start` (keyed, with a job deadline), `Emit`, `Finish` (whether it enqueued the terminal message), `JobDeadline`, `Cancel`, `Exit` and `Dispatch` (kind and fence verdict). Each From 176d924fe9386440da2abdec409d321b87eb2f9e Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:21:54 -0300 Subject: [PATCH 061/208] feat(exit): exit request with kill provenance, death point, Killed/NoProc, exit signals, fences (WIP) --- concurrency/src/actor_cleanup.rs | 165 +++----- concurrency/src/child_handle.rs | 273 ++++++------ concurrency/src/error.rs | 37 +- concurrency/src/lib.rs | 2 +- concurrency/src/link.rs | 684 ++++++++++++++++++++++--------- concurrency/src/mailbox.rs | 7 + concurrency/src/system.rs | 4 +- concurrency/src/tasks/actor.rs | 431 +++++++++++-------- concurrency/src/threads/actor.rs | 394 ++++++++++-------- 9 files changed, 1220 insertions(+), 777 deletions(-) diff --git a/concurrency/src/actor_cleanup.rs b/concurrency/src/actor_cleanup.rs index 69e376ff..98145fc1 100644 --- a/concurrency/src/actor_cleanup.rs +++ b/concurrency/src/actor_cleanup.rs @@ -2,7 +2,7 @@ use crate::{ child_handle::ActorId, error::ExitReason, - link::{LinkCleanup, LinkTable, LinkedExitReason}, + link::{LinkNotices, LinkTable, LinkedExitReason}, monitor::{MonitorCleanup, MonitorTable}, }; use spawned_rt::sync::watch; @@ -15,18 +15,18 @@ pub(crate) fn log_safely(log: impl FnOnce()) { } /// Constructed before dispatch, so even a never-polled actor has cleanup. -/// Completion owns its sender until links and outgoing monitors are closed. +/// [`die`](Self::die) is the death point D; dropping it is publication. pub(crate) struct ActorCleanup { id: ActorId, metrics: crate::instrumentation::ActorObservation, - links: LinkCleanup, + links: LinkTable, cause: LinkedExitReason, monitors: Option, completion: watch::Sender>, - fallback: ExitReason, reason: Option, + notices: Option, cancel: C, - mailbox_failures: Option>, + failures: Option>, } impl ActorCleanup { pub(crate) fn new( @@ -35,126 +35,94 @@ impl ActorCleanup { cause: LinkedExitReason, monitors: MonitorTable, completion: watch::Sender>, - fallback: ExitReason, cancel: C, ) -> Self { Self { id, metrics: crate::instrumentation::ActorObservation::new(), - links: LinkCleanup::new(id, links), + links, cause, monitors: Some(MonitorCleanup(monitors)), completion, - fallback, reason: None, + notices: None, cancel, - mailbox_failures: None, + failures: None, } } pub(crate) fn with_mailbox_failures( mut self, failures: std::sync::Arc, ) -> Self { - self.mailbox_failures = Some(failures); + self.failures = Some(failures); self } - pub(crate) fn complete(&mut self, reason: ExitReason) { - self.complete_inner(reason, false); + /// The death point D: the requested reason becomes final and the link + /// table closes, taking one notice per link (delivered at publication). + /// Idempotent; returns the final reason. + pub(crate) fn die(&mut self) -> ExitReason { + if let Some(reason) = &self.reason { + return reason.clone(); + } + let reason = crate::link::lock_request(&self.cause).die(); + self.close(reason) } - pub(crate) fn complete_secondary(&mut self, reason: ExitReason) { - self.complete_inner(reason, true); + fn close(&mut self, reason: ExitReason) -> ExitReason { + self.notices = Some(crate::link::close_links(self.id, &self.links, &reason)); + self.reason = Some(reason.clone()); + reason } - fn complete_inner(&mut self, mut reason: ExitReason, secondary: bool) { - let mut cause = self.cause.lock().unwrap_or_else(|p| p.into_inner()); - if secondary { - if let Some(committed) = cause.as_ref().filter(|c| **c != ExitReason::Normal) { - reason = committed.clone(); - } - } - if matches!(reason, ExitReason::Normal) { - if let Some(linked) = cause.as_ref() { - reason = linked.clone(); - } - if reason == ExitReason::Normal { - if let Some(message) = self.mailbox_failures.as_ref().and_then(|f| f.first()) { - reason = ExitReason::Panic(format!("queued message destructor: {message}")); - } - } + /// Record a cleanup failure after D (state or queued-envelope + /// destructor): it never changes the final reason. + pub(crate) fn cleanup_failed(&self, message: String) { + if let Some(failures) = &self.failures { + failures.record(message); } - // Freeze the completed outcome before any further cancellation callbacks. - *cause = Some(reason.clone()); - self.reason = Some(reason); } } impl Drop for ActorCleanup { fn drop(&mut self) { - let completed = self.reason.is_some(); - let failure = self.mailbox_failures.as_ref().and_then(|f| f.first()); - let mut reason = self.reason.take().unwrap_or_else(|| self.fallback.clone()); - { - let mut cause = self.cause.lock().unwrap_or_else(|p| p.into_inner()); - if !completed { - if let Some(committed) = cause.as_ref().filter(|c| **c != ExitReason::Normal) { - reason = committed.clone(); - } - } - // Queued destructors are secondary to committed Shutdown/Kill or - // a linked/handler panic. Only a completed Normal lifecycle gains - // a new panic outcome; aborted tasks keep their abort outcome. - if completed && reason == ExitReason::Normal { - if let Some(message) = &failure { - reason = ExitReason::Panic(format!("queued message destructor: {message}")); - } + // Never reached D: the runtime dropped the actor (G1). + let reason = match self.reason.clone() { + Some(reason) => reason, + None => { + let reason = crate::link::lock_request(&self.cause).runtime_drop(); + self.close(reason) } - *cause = Some(reason.clone()); - } - if let Some(message) = failure { + }; + if let Some(message) = self.failures.as_ref().and_then(|f| f.first()) { // Subscriber code must not unwind through a receiver/actor teardown. - if let Err(payload) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - tracing::warn!(event = "mailbox_cleanup_failed", actor = %self.id, %message, %reason, - "queued message destructor failed"); - })) { - crate::mailbox::dispose_panic(payload); - } + log_safely(|| { + tracing::warn!(event = "actor_cleanup_failed", actor = %self.id, %message, %reason, + "destructor failed after the death point"); + }); } (self.cancel)(); drop(self.monitors.take()); - self.links.complete(&reason); self.metrics.finish(&reason); + // Publication: wait_exit, join and monitors observe the final reason, + // then the link notices taken at D are delivered. self.completion.send_replace(Some(reason)); + if let Some(notices) = self.notices.take() { + notices.deliver(); + } } } -/// Destroy state after capturing the lifecycle outcome. A state destructor is -/// secondary to an already observed handler/start/stop panic, even before that -/// outcome has been committed to the shared exit slot. -pub(crate) fn finish_actor_state( - actor: A, - primary: Result, -) -> Result { +/// Destroy state after D. A destructor panic never changes the final reason; +/// it is returned as a cleanup failure. +pub(crate) fn finish_actor_state(actor: A) -> Option { let Err(payload) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| drop(actor))) else { - return primary; + return None; }; let message = crate::error::panic_message(&*payload); crate::mailbox::dispose_panic(payload); - let keep_primary = match &primary { - Ok(reason) => *reason != ExitReason::Normal, - Err(_) => true, - }; - if keep_primary { - if let Err(payload) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - tracing::warn!(event = "actor_state_cleanup_failed", actor_type = std::any::type_name::(), - ?primary, secondary = %message, "actor state destructor panicked after a terminal outcome"); - })) { - crate::mailbox::dispose_panic(payload); - } - primary - } else { - Err(ExitReason::Panic(format!( - "actor state destructor: {message}" - ))) - } + log_safely(|| { + tracing::warn!(event = "actor_state_cleanup_failed", actor_type = std::any::type_name::(), + secondary = %message, "actor state destructor panicked after the death point"); + }); + Some(format!("actor state destructor: {message}")) } /// Captured as one value by the dispatch closure. Declaration order makes @@ -184,9 +152,7 @@ impl Drop for ActorStartup { let message = crate::error::panic_message(&*panic); crate::mailbox::dispose_panic(panic); if let Some(cleanup) = &mut self.cleanup { - cleanup.complete_secondary(ExitReason::Panic(format!( - "actor startup payload destructor: {message}" - ))); + cleanup.cleanup_failed(format!("actor startup payload destructor: {message}")); } } } @@ -203,22 +169,11 @@ mod state_drop_tests { } } #[test] - fn destructor_failure_only_escalates_normal_even_without_a_committed_cause() { - for primary in [ - Ok(ExitReason::Normal), - Ok(ExitReason::Shutdown), - Ok(ExitReason::Kill), - Ok(ExitReason::Panic("handler failure".into())), - Err(ExitReason::Panic("lifecycle unwind".into())), - ] { - let expected = if primary == Ok(ExitReason::Normal) { - Err(ExitReason::Panic( - "actor state destructor: secondary state failure".into(), - )) - } else { - primary.clone() - }; - assert_eq!(finish_actor_state(Explodes, primary), expected); - } + fn destructor_failure_is_a_cleanup_failure_not_a_reason() { + assert_eq!( + finish_actor_state(Explodes), + Some("actor state destructor: secondary state failure".into()) + ); + assert_eq!(finish_actor_state(()), None); } } diff --git a/concurrency/src/child_handle.rs b/concurrency/src/child_handle.rs index 06b5492a..c24632f9 100644 --- a/concurrency/src/child_handle.rs +++ b/concurrency/src/child_handle.rs @@ -1,5 +1,5 @@ use crate::error::ExitReason; -use crate::link::{LinkTable, LinkedExitReason, SendExitFn, TrapExitFlag}; +use crate::link::{LinkTable, Origin, SignalTarget}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -15,6 +15,10 @@ static NEXT_ACTOR_ID: AtomicU64 = AtomicU64::new(1); pub struct ActorId(u64); impl ActorId { + /// The sender of an exit signal sent from outside any actor + /// ([`ChildHandle::exit`]). Never an actor's identity, never a parent. + pub const EXTERNAL: ActorId = ActorId(0); + #[cfg_attr(not(feature = "lifecycle-trace"), allow(dead_code))] pub(crate) fn raw(self) -> u64 { self.0 @@ -53,85 +57,43 @@ pub(crate) enum Completion { pub(crate) type CancelFn = Arc; /// Type-erased handle to a running actor. Provides lifecycle operations -/// (stop, liveness check, exit reason) without knowing the actor's concrete type. +/// (administrative stops, exit signals, liveness check, exit reason) without +/// knowing the actor's concrete type. /// /// Obtained via `ChildHandle::from(actor_ref)`. /// /// Unlike `ActorRef`, a `ChildHandle` cannot send messages directly — it -/// only provides supervision-related operations (stop, wait, check liveness, -/// link/monitor). +/// only provides supervision-related operations (stop, exit, kill, wait, +/// check liveness, link/monitor). #[derive(Clone)] pub struct ChildHandle { - id: ActorId, + target: Arc, cancel: CancelFn, completion: Completion, - // Link/trap_exit support — carried so `ctx.link()` can wire up signal - // propagation without knowing the underlying actor type. - trap_exit: TrapExitFlag, links: LinkTable, - linked_reason: LinkedExitReason, - send_exit: SendExitFn, mailbox_failures: Option>, } impl std::fmt::Debug for ChildHandle { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("ChildHandle") - .field("id", &self.id) + .field("id", &self.target.id) .finish_non_exhaustive() } } impl ChildHandle { - /// Create a ChildHandle for tasks mode. - #[allow(clippy::too_many_arguments)] - #[cfg(feature = "async")] - pub(crate) fn from_tasks( - id: ActorId, - cancellation_token: spawned_rt::tasks::CancellationToken, - completion_rx: spawned_rt::sync::watch::Receiver>, - trap_exit: TrapExitFlag, - links: LinkTable, - linked_reason: LinkedExitReason, - send_exit: SendExitFn, - ) -> Self { - let stop_reason = linked_reason.clone(); - Self { - id, - cancel: Arc::new(move || { - crate::link::normal_stop(&stop_reason, || cancellation_token.cancel()) - }), - completion: Completion::Tasks(completion_rx), - trap_exit, - links, - linked_reason, - send_exit, - mailbox_failures: None, - } - } - - /// Create a ChildHandle for threads mode. - #[allow(clippy::too_many_arguments)] - pub(crate) fn from_threads( - id: ActorId, - cancellation_token: spawned_rt::threads::CancellationToken, - completion: spawned_rt::sync::watch::Receiver>, - trap_exit: TrapExitFlag, - links: LinkTable, - linked_reason: LinkedExitReason, - send_exit: SendExitFn, - ) -> Self { - let stop_reason = linked_reason.clone(); + /// Create a ChildHandle from an actor's signal target, completion channel + /// and exit table. + pub(crate) fn new(target: Arc, completion: Completion, links: LinkTable) -> Self { + let stop = target.clone(); Self { - id, cancel: Arc::new(move || { - crate::link::normal_stop(&stop_reason, || cancellation_token.cancel()) + crate::link::normal_stop(&stop.request, || (stop.wake)()) }), - completion: Completion::Threads(completion), - trap_exit, + target, + completion, links, - linked_reason, - send_exit, mailbox_failures: None, } } @@ -143,28 +105,34 @@ impl ChildHandle { self } - /// First queued-message destructor failure, recorded independently of the - /// primary exit reason. Complete after actor termination; Shutdown, Kill - /// and an existing panic are never replaced by this secondary failure. - pub fn mailbox_cleanup_failure(&self) -> Option { + /// First cleanup failure after the death point (a panicking state or + /// queued-message destructor), recorded independently of the final exit + /// reason, which it never changes. Complete after publication. A + /// supervisor restarts a Transient child that has one. + pub fn cleanup_failure(&self) -> Option { self.mailbox_failures.as_ref().and_then(|f| f.first()) } - /// Crate-internal accessors used by `ctx.link()` to wire up signal propagation. - pub(crate) fn trap_exit_flag(&self) -> &TrapExitFlag { - &self.trap_exit + /// Deprecated alias of [`cleanup_failure`](Self::cleanup_failure). + #[deprecated(since = "0.8.0", note = "use `cleanup_failure()`")] + pub fn mailbox_cleanup_failure(&self) -> Option { + self.cleanup_failure() + } + + /// Crate-internal accessors used by `ctx.link()` and `ctx.monitor()`. + pub(crate) fn signal_target(&self) -> &Arc { + &self.target } pub(crate) fn links(&self) -> &LinkTable { &self.links } - pub(crate) fn linked_reason(&self) -> &LinkedExitReason { - &self.linked_reason - } - pub(crate) fn cancel_fn(&self) -> &CancelFn { - &self.cancel - } - pub(crate) fn send_exit_fn(&self) -> &SendExitFn { - &self.send_exit + /// Past the death point D: links, monitors, sends and requests report + /// `NoProc` from now on. + pub(crate) fn is_past_death(&self) -> bool { + self.links + .lock() + .unwrap_or_else(|p| p.into_inner()) + .is_dead() } // Each child_handle() creates this allocation; its clones share it. @@ -175,28 +143,59 @@ impl ChildHandle { /// The actor's unique identity. pub fn id(&self) -> ActorId { - self.id + self.target.id } - /// Signal the actor to stop. The actor will finish its current handler, - /// run `stopped()`, and exit. + /// Administrative stop (Spawned's `gen_server:stop`): the actor finishes + /// its current handler, discards its queue, runs `stopped()` and exits + /// `Normal`. Untrappable; returns at once (join with `wait_exit_*`). pub fn stop(&self) { (self.cancel)(); } - /// Direct cooperative stop with a chosen reason. Trapping only affects - /// linked exit notices and cannot intercept this request. Kill remains - /// trappable when this actor's final reason is propagated to its peers. + /// Administrative stop with a chosen reason, as [`stop`](Self::stop). + /// Not an exit signal: `trap_exit` cannot intercept it and `stopped()` + /// runs. `stop_with_reason(Kill)` ends `Kill`; use [`kill`](Self::kill) + /// for the untrappable kill signal that ends `Killed`. pub fn stop_with_reason(&self, reason: ExitReason) { - crate::link::stop_with_reason(&self.linked_reason, reason, || (self.cancel)()); + crate::link::stop_with_reason(&self.target.request, reason, || (self.target.wake)()); } - /// Request an ordered cooperative shutdown without replacing an earlier - /// committed cause. A cleanup panic remains an abnormal exit. + /// Administrative stop with reason `Shutdown`, as [`stop`](Self::stop). + /// The first requested reason wins over later stops. pub fn shutdown(&self) { - crate::link::stop_with_reason(&self.linked_reason, ExitReason::Shutdown, || { - (self.cancel)() - }); + self.stop_with_reason(ExitReason::Shutdown); + } + + /// Send an exit signal from outside any actor (OTP `exit_signal/2`, the + /// sender is [`ActorId::EXTERNAL`]): + /// + /// - `Kill` is untrappable: the actor ends `Killed` (see [`kill`](Self::kill)); + /// - `Normal` is ignored by a non-trapping actor; + /// - any other reason ends a non-trapping actor with that reason after + /// its current handler, **without** `stopped()`, discarding its queue; + /// - a trapping actor gets `Exit { from: EXTERNAL, reason }` queued + /// behind its messages. + /// + /// A signal to an actor past its death point is dropped. + pub fn exit(&self, reason: ExitReason) { + self.exit_from(ActorId::EXTERNAL, reason, false); + } + + /// Untrappable kill signal (`exit(Kill)`): the actor ends `Killed`, also + /// when it is already exiting (until its death point), and nothing + /// replaces that reason. The current handler, and a `stopped()` that + /// already began, still finish (Rust cannot preempt them); a `stopped()` + /// that has not begun is skipped. Links receive `Killed` (trappable), + /// monitors `Down { reason: Killed }`. + pub fn kill(&self) { + self.exit(ExitReason::Kill); + } + + /// Exit signal with an explicit sender; `from_parent` marks the internal + /// entry point of supervisors and systems (E3). + pub(crate) fn exit_from(&self, from: ActorId, reason: ExitReason, from_parent: bool) { + crate::link::deliver_signal(&self.target, from, reason, Origin::Direct, from_parent); } /// Returns `true` if the actor is still running. @@ -204,9 +203,10 @@ impl ChildHandle { self.exit_reason().is_none() } - /// Poll the exit reason. Returns `None` if the actor is still running. - /// A completion channel closed without a reason reports [`ExitReason::Kill`], - /// matching the wait methods (for example, after runtime shutdown). + /// Poll the exit reason. Returns `None` until the exit is published. + /// A completion channel closed without a reason reports + /// [`ExitReason::Killed`], matching the wait methods (for example, after + /// runtime shutdown). pub fn exit_reason(&self) -> Option { match &self.completion { #[cfg(feature = "async")] @@ -260,18 +260,20 @@ pub(crate) fn poll_exit_reason( let closed = rx.has_changed().is_err(); rx.borrow() .clone() - .or_else(|| closed.then_some(ExitReason::Kill)) + .or_else(|| closed.then_some(ExitReason::Killed)) } /// Async loop that polls a watch channel until it carries an exit reason. -/// Returns `ExitReason::Kill` if the watch sender is dropped without a reason. -async fn wait_loop(mut rx: spawned_rt::sync::watch::Receiver>) -> ExitReason { +/// Returns `ExitReason::Killed` if the watch sender is dropped without a reason. +pub(crate) async fn wait_loop( + mut rx: spawned_rt::sync::watch::Receiver>, +) -> ExitReason { loop { if let Some(reason) = rx.borrow_and_update().clone() { return reason; } if rx.changed().await.is_err() { - return ExitReason::Kill; + return ExitReason::Killed; } } } @@ -312,7 +314,7 @@ pub(crate) fn wait_for_thread_exit_blocking( /// making progress (deadlock). Use `wait_exit_async` from async context, or /// run on a multi-thread runtime. /// -/// Returns `ExitReason::Kill` if the watch sender is dropped without setting a +/// Returns `ExitReason::Killed` if the watch sender is dropped without setting a /// reason — this means the actor task was aborted externally (e.g., runtime /// shutdown) without going through the normal exit path. #[cfg(feature = "async")] @@ -350,7 +352,7 @@ fn wait_for_tasks_exit_blocking( impl PartialEq for ChildHandle { fn eq(&self, other: &Self) -> bool { - self.id == other.id + self.target.id == other.target.id } } @@ -358,7 +360,7 @@ impl Eq for ChildHandle {} impl std::hash::Hash for ChildHandle { fn hash(&self, state: &mut H) { - self.id.hash(state); + self.target.id.hash(state); } } @@ -367,6 +369,20 @@ mod tests { use super::*; use crate::link::{new_link_table, new_linked_exit_reason, new_trap_exit_flag}; + /// A signal target whose wake cancels `token`. + pub(crate) fn test_target(id: ActorId, token: spawned_rt::threads::CancellationToken) -> Arc { + Arc::new(SignalTarget { + id, + parent: None, + trap: new_trap_exit_flag(), + request: new_linked_exit_reason(), + wake: Arc::new(move || token.cancel()), + send_exit: Arc::new(|_| Ok(())), + send_parent_exit: Arc::new(|_| Ok(())), + send_held: Arc::new(|_, _| Ok(())), + }) + } + // Shared tasks-mode fixture: an Idler actor that does nothing and never // stops on its own. Used by multiple tests below. #[cfg(feature = "async")] @@ -382,15 +398,10 @@ mod tests { token: spawned_rt::threads::CancellationToken, completion: spawned_rt::sync::watch::Receiver>, ) -> ChildHandle { - let no_op_send_exit: SendExitFn = Arc::new(|_| Ok(())); - ChildHandle::from_threads( - ActorId::next(), - token, - completion, - new_trap_exit_flag(), + ChildHandle::new( + test_target(ActorId::next(), token), + Completion::Threads(completion), new_link_table(), - new_linked_exit_reason(), - no_op_send_exit, ) } @@ -440,24 +451,15 @@ mod tests { let (_completion_tx, completion) = spawned_rt::sync::watch::channel(None); let token = spawned_rt::threads::CancellationToken::new(); let id = ActorId::next(); - let no_op_send_exit: SendExitFn = Arc::new(|_| Ok(())); - let h1 = ChildHandle::from_threads( - id, - token.clone(), - completion.clone(), - new_trap_exit_flag(), + let h1 = ChildHandle::new( + test_target(id, token.clone()), + Completion::Threads(completion.clone()), new_link_table(), - new_linked_exit_reason(), - no_op_send_exit.clone(), ); - let h2 = ChildHandle::from_threads( - id, - token, - completion, - new_trap_exit_flag(), + let h2 = ChildHandle::new( + test_target(id, token), + Completion::Threads(completion), new_link_table(), - new_linked_exit_reason(), - no_op_send_exit, ); assert_eq!(h1, h2); } @@ -735,7 +737,7 @@ mod tests { #[cfg(test)] mod closed_completion_tests { use super::*; - use crate::link::{new_link_table, new_linked_exit_reason, new_trap_exit_flag}; + use crate::link::new_link_table; #[test] fn closed_without_reason_is_dead_and_all_waits_agree() { #[cfg(not(feature = "async"))] @@ -744,52 +746,39 @@ mod closed_completion_tests { let modes = [false, true]; for threads in modes { let (tx, rx) = spawned_rt::sync::watch::channel(None); - let child = if threads { - ChildHandle::from_threads( - ActorId::next(), - spawned_rt::threads::CancellationToken::new(), - rx, - new_trap_exit_flag(), - new_link_table(), - new_linked_exit_reason(), - Arc::new(|_| Ok(())), - ) + let target = super::tests::test_target( + ActorId::next(), + spawned_rt::threads::CancellationToken::new(), + ); + let completion = if threads { + Completion::Threads(rx) } else { #[cfg(not(feature = "async"))] unreachable!(); #[cfg(feature = "async")] - { - ChildHandle::from_tasks( - ActorId::next(), - spawned_rt::tasks::CancellationToken::new(), - rx, - new_trap_exit_flag(), - new_link_table(), - new_linked_exit_reason(), - Arc::new(|_| Ok(())), - ) - } + Completion::Tasks(rx) }; + let child = ChildHandle::new(target, completion, new_link_table()); assert!(child.is_alive()); drop(tx); - assert_eq!(child.exit_reason(), Some(ExitReason::Kill)); + assert_eq!(child.exit_reason(), Some(ExitReason::Killed)); assert!(!child.is_alive()); assert_eq!( futures::executor::block_on(child.wait_exit_async()), - ExitReason::Kill + ExitReason::Killed ); // Already-exited fast path must not enter another executor or reject // a current-thread runtime merely because no exit value was published. assert_eq!( futures::executor::block_on(async { child.wait_exit_blocking() }), - ExitReason::Kill + ExitReason::Killed ); let rt = tokio::runtime::Builder::new_current_thread() .enable_all() .build() .unwrap(); rt.block_on(async { - assert_eq!(child.wait_exit_blocking(), ExitReason::Kill); + assert_eq!(child.wait_exit_blocking(), ExitReason::Killed); }); } } diff --git a/concurrency/src/error.rs b/concurrency/src/error.rs index f9810660..a3c59a55 100644 --- a/concurrency/src/error.rs +++ b/concurrency/src/error.rs @@ -1,5 +1,9 @@ /// Reason an actor stopped. Used by supervision to decide whether to restart. +/// +/// Non-exhaustive, as OTP exit reasons are open terms: match with a wildcard +/// arm. #[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] pub enum ExitReason { /// Clean stop via `ctx.stop()` or `ChildHandle::stop()`. The run loop holds /// a sender to its own mailbox, so the mailbox never closes while it runs. @@ -8,11 +12,19 @@ pub enum ExitReason { Shutdown, /// Actor panicked in `started()`, a handler, or `stopped()`. Panic(String), - /// Direct kill outcome, also used when an async task is aborted externally. - /// Propagated link notices carrying Kill are trappable. An active handler may finish and the - /// actor still runs `stopped()`. An externally dropped future cannot guarantee - /// execution of that async cleanup hook. No blocking Rust code is preempted. + /// A self-exit with `Kill` (`ctx.stop_with_reason(Kill)`, OTP `exit(kill)`), + /// an administrative `stop_with_reason(Kill)`, or a `Kill` link notice. + /// Propagated over links as `Kill` (trappable), not converted to `Killed`. Kill, + /// Killed by an untrappable kill signal ([`ChildHandle::kill`], + /// `exit(Kill)`), or dropped by its runtime before it finished (OTP + /// `killed`). Links receive `Killed`, which is trappable. + /// + /// [`ChildHandle::kill`]: crate::ChildHandle::kill + Killed, + /// The target no longer existed when a link, monitor, send or request + /// was made (OTP `noproc`). + NoProc, } impl ExitReason { @@ -22,7 +34,10 @@ impl ExitReason { pub fn is_abnormal(&self) -> bool { match self { ExitReason::Normal | ExitReason::Shutdown => false, - ExitReason::Panic(_) | ExitReason::Kill => true, + ExitReason::Panic(_) + | ExitReason::Kill + | ExitReason::Killed + | ExitReason::NoProc => true, } } } @@ -34,10 +49,18 @@ impl std::fmt::Display for ExitReason { ExitReason::Shutdown => write!(f, "shutdown"), ExitReason::Panic(msg) => write!(f, "panic: {msg}"), ExitReason::Kill => write!(f, "kill"), + ExitReason::Killed => write!(f, "killed"), + ExitReason::NoProc => write!(f, "noproc"), } } } +/// The target of [`try_link`](crate::tasks::Context::try_link) no longer +/// existed (it was past its death point), as OTP's `noproc`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("noproc")] +pub struct NoProc; + /// Extract a human-readable message from a panic payload. pub(crate) fn panic_message(panic: &(dyn std::any::Any + Send)) -> String { if let Some(s) = panic.downcast_ref::<&str>() { @@ -106,6 +129,8 @@ mod tests { assert!(!ExitReason::Shutdown.is_abnormal()); assert!(ExitReason::Panic("boom".into()).is_abnormal()); assert!(ExitReason::Kill.is_abnormal()); + assert!(ExitReason::Killed.is_abnormal()); + assert!(ExitReason::NoProc.is_abnormal()); } #[test] @@ -114,6 +139,8 @@ mod tests { assert_eq!(ExitReason::Shutdown.to_string(), "shutdown"); assert_eq!(ExitReason::Panic("oops".into()).to_string(), "panic: oops"); assert_eq!(ExitReason::Kill.to_string(), "kill"); + assert_eq!(ExitReason::Killed.to_string(), "killed"); + assert_eq!(ExitReason::NoProc.to_string(), "noproc"); } #[test] diff --git a/concurrency/src/lib.rs b/concurrency/src/lib.rs index 4f6a0752..72f63607 100644 --- a/concurrency/src/lib.rs +++ b/concurrency/src/lib.rs @@ -107,7 +107,7 @@ pub mod tasks; pub mod threads; pub use child_handle::{ActorId, ChildHandle}; -pub use error::{ActorError, ExitReason}; +pub use error::{ActorError, ExitReason, NoProc}; pub use link::Exit; pub use monitor::{Down, MonitorRef}; pub use reply::{ diff --git a/concurrency/src/link.rs b/concurrency/src/link.rs index 48cf54a8..ba17c049 100644 --- a/concurrency/src/link.rs +++ b/concurrency/src/link.rs @@ -1,7 +1,7 @@ use crate::child_handle::ActorId; use crate::error::{ActorError, ExitReason}; use std::panic::{catch_unwind, AssertUnwindSafe}; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex, Weak}; // --------------------------------------------------------------------------- @@ -31,47 +31,254 @@ pub struct Exit { /// mailbox. Captures the peer's typed sender at link time. pub(crate) type SendExitFn = Arc Result<(), ActorError> + Send + Sync>; -/// Per-actor flag controlling how exit signals from linked actors are handled. -/// `false` (default): the receiver is cancelled. `true`: the receiver gets an -/// `Exit` message via `Actor::exit_received`. +/// Queues a signal held during `started()`; `true` for a parent signal. +pub(crate) type SendHeldFn = Arc Result<(), ActorError> + Send + Sync>; + +/// Per-actor flag controlling how exit signals are handled. `false` +/// (default): a non-`Normal` signal ends the receiver. `true`: the receiver +/// gets an `Exit` message via `Actor::exit_received` (only a kill is +/// untrappable). pub(crate) type TrapExitFlag = Arc; -/// Per-actor slot holding the exit reason of a linked actor whose death -/// triggered cancellation. When a non-trapping actor is cancelled by a link -/// signal, this slot is set so the actor's own exit reason propagates -/// transitively through further links. -pub(crate) type LinkedExitReason = Arc>>; +/// The exit request of one actor instance (design otp-exit-signals.md, +/// "Phases of an exit"): the requested reason, kill provenance, whether +/// `stopped()` is skipped, and the death point D after which nothing changes. +#[derive(Debug, Default)] +pub(crate) struct ExitRequest { + reason: Option, + /// A kill signal (or the supervisor's escalation) arrived before D. + killed: bool, + /// The exit was begun by a kill, an untrapped signal or link notice, or + /// `stopped()` must otherwise not start (decision 2). + skip_hook: bool, + hook_begun: bool, + /// Past the death point D: the reason is final. + dead: bool, + /// Phase 0: `started()` is running; non-kill signals are held. + starting: bool, + /// Reasons of the signals held during `started()`, in arrival order. + held: Vec, + /// Held signal envelopes become `Exit`s (the actor trapped when + /// `started()` returned); otherwise they are inert. + held_trapped: bool, +} + +impl ExitRequest { + /// The requested reason (final after D). + pub(crate) fn reason(&self) -> Option<&ExitReason> { + self.reason.as_ref() + } + pub(crate) fn is_dead(&self) -> bool { + self.dead + } + /// A self-exit or an administrative stop: the first requested reason + /// wins. Returns whether the run loop must be woken. + pub(crate) fn request(&mut self, reason: ExitReason) -> bool { + if self.dead { + return false; + } + if self.reason.is_none() { + self.reason = Some(reason); + } + true + } + /// An untrapped non-`Normal` exit signal or link notice: begins exit + /// without `stopped()`, or replaces the reason of an actor already + /// exiting (decision 8), unless a kill was received. + pub(crate) fn untrapped(&mut self, reason: ExitReason) -> bool { + if self.dead || self.killed { + return false; + } + if self.reason.is_none() && !self.hook_begun { + self.skip_hook = true; + } + self.reason = Some(reason); + true + } + /// An untrappable kill: `Killed` supersedes any reason until D and + /// skips a `stopped()` that has not begun. + pub(crate) fn kill(&mut self) -> bool { + if self.dead { + return false; + } + self.reason = Some(ExitReason::Killed); + self.killed = true; + if !self.hook_begun { + self.skip_hook = true; + } + true + } + /// A panic in `started()`, a handler or `stopped()`: begins exit, or + /// replaces the requested reason, unless a kill was received. + pub(crate) fn panic(&mut self, reason: ExitReason) { + if !self.dead && !self.killed { + self.reason = Some(reason); + } + } + /// `stopped()` is about to start: returns whether it runs. + pub(crate) fn begin_hook(&mut self) -> bool { + self.hook_begun = true; + !self.skip_hook + } + /// The death point D: the requested reason becomes final. + pub(crate) fn die(&mut self) -> ExitReason { + self.dead = true; + self.reason.get_or_insert(ExitReason::Normal).clone() + } + /// The runtime dropped the actor before D (G1): `Killed` unless a + /// non-`Normal` reason was already requested; then D. + pub(crate) fn runtime_drop(&mut self) -> ExitReason { + if !self.dead { + self.reason = match self.reason.take() { + None | Some(ExitReason::Normal) => Some(ExitReason::Killed), + Some(reason) => Some(reason), + }; + } + self.die() + } + pub(crate) fn begin_starting(&mut self) { + self.starting = true; + } + /// `started()` returned: resolve the held signals with the trap flag read + /// now. A non-trapping actor begins exit with the first non-`Normal` + /// one; a trapping actor dispatches each held envelope as an `Exit` in + /// its own place in the queue. Returns whether the run loop must stop. + pub(crate) fn end_starting(&mut self, trapping: bool) -> bool { + self.starting = false; + let held = std::mem::take(&mut self.held); + self.held_trapped = trapping; + if !trapping { + if let Some(reason) = held.into_iter().find(|r| *r != ExitReason::Normal) { + return self.untrapped(reason); + } + } + false + } + pub(crate) fn held_trapped(&self) -> bool { + self.held_trapped + } +} + +/// Per-actor exit request, shared by every handle of the instance (and by its +/// managed scope). +pub(crate) type LinkedExitReason = Arc>; -/// Create a new empty linked-exit-reason slot. +/// Create a new empty exit request. pub(crate) fn new_linked_exit_reason() -> LinkedExitReason { - Arc::new(Mutex::new(None)) + Arc::new(Mutex::new(ExitRequest::default())) +} + +pub(crate) fn lock_request(request: &LinkedExitReason) -> std::sync::MutexGuard<'_, ExitRequest> { + request.lock().unwrap_or_else(|p| p.into_inner()) +} + +/// Everything needed to deliver an exit signal to one actor instance without +/// knowing its type. +pub(crate) struct SignalTarget { + pub(crate) id: ActorId, + /// The actor's `start_linked` parent: its signals are parent exits. + pub(crate) parent: Option, + pub(crate) trap: TrapExitFlag, + pub(crate) request: LinkedExitReason, + /// Wakes the run loop (cancels its token) without recording a reason. + pub(crate) wake: Arc, + /// Queues an `Exit` for `exit_received`. + pub(crate) send_exit: SendExitFn, + /// Queues a parent exit: when dispatched, exit begins with its reason and + /// `stopped()` runs; `exit_received` never sees it (E3). + pub(crate) send_parent_exit: SendExitFn, + /// Queues a signal held during `started()` in its place in the mailbox + /// (the flag marks a parent signal). + pub(crate) send_held: SendHeldFn, +} + +/// Where an exit signal comes from. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Origin { + /// `ctx.exit` / `ChildHandle::exit` / `kill` (a `Kill` is untrappable). + Direct, + /// A death notice of a link, or a late `NoProc` (a `Kill` is trappable). + Link, +} + +/// Deliver an exit signal (OTP `exit_signal/2` semantics, design E1-E3). +/// `from_parent` marks the internal entry point of supervisors and systems. +pub(crate) fn deliver_signal( + target: &SignalTarget, + from: ActorId, + reason: ExitReason, + origin: Origin, + from_parent: bool, +) { + if origin == Origin::Direct && reason == ExitReason::Kill { + if lock_request(&target.request).kill() { + (target.wake)(); + } + return; + } + let from_parent = from_parent || target.parent == Some(from); + let mut request = lock_request(&target.request); + if request.dead { + return; + } + let exit = Exit { from, reason }; + if request.starting { + // Held in arrival order, and as an envelope in its mailbox place. + request.held.push(exit.reason.clone()); + let _ = (target.send_held)(exit, from_parent); + return; + } + // The trap flag is read when the signal arrives. + if target.trap.load(Ordering::Acquire) { + drop(request); + let send = if from_parent { + &target.send_parent_exit + } else { + &target.send_exit + }; + let _ = send(exit); // admission may be closed if the peer just died + return; + } + if exit.reason == ExitReason::Normal { + return; + } + if request.untrapped(exit.reason) { + drop(request); + (target.wake)(); + } } /// An entry in an actor's link table, representing one linked peer. pub(crate) struct LinkEntry { /// The peer's unique actor ID. pub peer_id: ActorId, - /// Delivers an exit signal to the peer. - pub signal: ExitSignalFn, - /// Reference to the peer's link table, so we can remove ourselves on death. + /// Edge generation: both records of one link share it; a late record has + /// its own. Delivery applies only to a record with this generation. + pub gen: u64, + /// The peer, to deliver this actor's death notice to it. + pub peer: Arc, + /// Reference to the peer's link table, for receiver-side delivery. pub peer_links: Weak>, + /// A pending late `NoProc` notice (the peer was past D at `link`). + pub late: bool, } -/// Type-erased function that signals exit to a peer. Decides whether to -/// cancel the peer or send an `Exit` message based on the peer's `trap_exit` -/// flag and the exit reason. -pub(crate) type ExitSignalFn = Arc; - -/// An actor's link table — a list of linked peers. +/// An actor's exit table: its link records and its death point. #[derive(Default)] pub(crate) struct LinkState { entries: Vec, - // Published under the same mutex as registration, before edges are drained. - // Completion channels can be published later without losing a racing link. + // Set at the death point D under the same mutex as registration. exited: Option, } pub(crate) type LinkTable = Arc>; +impl LinkState { + /// Past the death point D. + pub(crate) fn is_dead(&self) -> bool { + self.exited.is_some() + } +} + /// Create a new empty link table. pub(crate) fn new_link_table() -> LinkTable { Arc::new(Mutex::new(LinkState::default())) @@ -82,40 +289,7 @@ pub(crate) fn new_trap_exit_flag() -> TrapExitFlag { Arc::new(AtomicBool::new(false)) } -/// Build an `ExitSignalFn` that signals a peer based on its trap_exit flag -/// and a typed-erased send-exit function. -pub(crate) fn make_signal( - peer_trap_exit: TrapExitFlag, - peer_cancel: Arc, - peer_send_exit: SendExitFn, - peer_linked_reason: LinkedExitReason, -) -> ExitSignalFn { - Arc::new(move |sender_id, reason| { - // A propagated notice is trappable, including Kill. Only a direct - // stop_with_reason(Kill) bypasses the target's trap_exit flag. - // Normal exit signals are silently dropped unless the peer is trapping - let trapping = peer_trap_exit.load(Ordering::Acquire); - if matches!(reason, ExitReason::Normal) && !trapping { - return; - } - if trapping { - // Send Exit message to the peer's mailbox - let exit = Exit { - from: sender_id, - reason, - }; - let _ = peer_send_exit(exit); // mailbox may be closed if peer just died - } else { - // Peer is not trapping: record the linked reason then cancel - let mut slot = peer_linked_reason.lock().unwrap_or_else(|p| p.into_inner()); - if slot.is_none() { - *slot = Some(reason); - } - drop(slot); - peer_cancel(); - } - }) -} +static NEXT_EDGE: AtomicU64 = AtomicU64::new(1); /// Mutate both ends in one transaction. Callbacks and removed entries are /// handled after releasing both locks. Pointer order is stable while Arcs live. @@ -135,42 +309,69 @@ fn with_pair( } } -/// Register before actor execution starts, or atomically observe a closed peer. -/// Late operations through an already-closed own context are ignored. +/// The outcome of a link attempt. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum LinkOutcome { + Linked, + /// Nothing to do: self link, an existing record, or the caller is past D. + Unchanged, + /// The peer was past D. + NoProc, +} + +/// Link `own` and `peer` (OTP `link/1`). An existing record (live or +/// pending) makes it a no-op. Past the peer's D, `try_only` returns +/// `NoProc` without a record; otherwise a pending late record is created and +/// its `NoProc` notice is delivered through the receiver-side step, so a +/// concurrent `unlink` still fences it. pub(crate) fn register_link( - own_id: ActorId, - own: &LinkTable, - own_signal: ExitSignalFn, - peer_id: ActorId, - peer: &LinkTable, - peer_signal: ExitSignalFn, -) { - if own_id == peer_id || Arc::ptr_eq(own, peer) { - return; + own: &Arc, + own_links: &LinkTable, + peer: &Arc, + peer_links: &LinkTable, + try_only: bool, +) -> LinkOutcome { + if own.id == peer.id || Arc::ptr_eq(own_links, peer_links) { + return LinkOutcome::Unchanged; } fail_point!("link.before_install"); - let closed = with_pair(own, peer, |a, b| { - if a.exited.is_some() || a.entries.iter().any(|e| e.peer_id == peer_id) { - return None; + let gen = NEXT_EDGE.fetch_add(1, Ordering::Relaxed); + let outcome = with_pair(own_links, peer_links, |a, b| { + if a.exited.is_some() || a.entries.iter().any(|e| e.peer_id == peer.id) { + return LinkOutcome::Unchanged; } - if let Some(reason) = &b.exited { - return Some(reason.clone()); + if b.exited.is_some() { + if !try_only { + a.entries.push(LinkEntry { + peer_id: peer.id, + gen, + peer: peer.clone(), + peer_links: Arc::downgrade(peer_links), + late: true, + }); + } + return LinkOutcome::NoProc; } a.entries.push(LinkEntry { - peer_id, - signal: peer_signal.clone(), - peer_links: Arc::downgrade(peer), + peer_id: peer.id, + gen, + peer: peer.clone(), + peer_links: Arc::downgrade(peer_links), + late: false, }); b.entries.push(LinkEntry { - peer_id: own_id, - signal: own_signal.clone(), - peer_links: Arc::downgrade(own), + peer_id: own.id, + gen, + peer: own.clone(), + peer_links: Arc::downgrade(own_links), + late: false, }); - None + LinkOutcome::Linked }); - if let Some(reason) = closed { - deliver(&own_signal, peer_id, reason); + if outcome == LinkOutcome::NoProc && !try_only { + deliver_notice(own_links, own, peer.id, gen, ExitReason::NoProc); } + outcome } fn remove(table: &mut LinkState, id: ActorId) -> Option { @@ -181,8 +382,8 @@ fn remove(table: &mut LinkState, id: ActorId) -> Option { .map(|i| table.entries.swap_remove(i)) } -/// Unlink is atomic with registration. An exit already taken for delivery may -/// still arrive; unlink does not retract a signal already in flight. +/// Unlink removes both records, live or pending: once it returns, no notice +/// of that edge takes effect (E6). An `Exit` already queued stays. pub(crate) fn unregister_link( own_id: ActorId, own: &LinkTable, @@ -196,61 +397,89 @@ pub(crate) fn unregister_link( drop(removed); } -fn deliver(signal: &ExitSignalFn, id: ActorId, reason: ExitReason) { - // One failing internal cancellation callback must not prevent other peers - // from being notified, or panic again during actor-future destruction. - if let Err(payload) = catch_unwind(AssertUnwindSafe(|| signal(id, reason))) { +/// Receiver-side delivery: under the receiver's table lock, apply the notice +/// only if the record with that peer and generation is still present, and +/// remove it. `unlink` removes records under the same lock (the fence). +fn deliver_notice( + receiver_links: &LinkTable, + receiver: &SignalTarget, + from: ActorId, + gen: u64, + reason: ExitReason, +) { + let mut table = receiver_links.lock().unwrap_or_else(|p| p.into_inner()); + let Some(i) = table + .entries + .iter() + .position(|e| e.peer_id == from && e.gen == gen) + else { + return; + }; + let record = table.entries.swap_remove(i); + // One failing internal callback must not prevent other peers from being + // notified, or panic again during actor-future destruction. + if let Err(payload) = catch_unwind(AssertUnwindSafe(|| { + deliver_signal(receiver, from, reason, Origin::Link, false) + })) { crate::mailbox::dispose_panic(payload); - if let Err(payload) = catch_unwind(AssertUnwindSafe(|| { + crate::actor_cleanup::log_safely(|| { tracing::error!("Panic while delivering a linked exit"); - })) { - crate::mailbox::dispose_panic(payload); - } + }); } + drop(table); + drop(record); } -/// Close the registration state before draining. Terminal closure is idempotent. -pub(crate) fn propagate_exit(own_id: ActorId, own: &LinkTable, reason: &ExitReason) { - let entries = { - let mut state = own.lock().unwrap_or_else(|p| p.into_inner()); - if state.exited.is_some() { +/// Death notices taken at D, delivered at publication. +#[derive(Default)] +pub(crate) struct LinkNotices { + from: Option, + reason: Option, + entries: Vec, +} + +impl LinkNotices { + /// Deliver every notice (publication). Each is applied only if the + /// receiver still holds the record (E6 fence). + pub(crate) fn deliver(self) { + let (Some(from), Some(reason)) = (self.from, self.reason) else { return; + }; + for entry in self.entries { + if entry.late { + continue; + } + if let Some(peer_links) = entry.peer_links.upgrade() { + deliver_notice(&peer_links, &entry.peer, from, entry.gen, reason.clone()); + } } - state.exited = Some(reason.clone()); - std::mem::take(&mut state.entries) - }; - for entry in entries { - let removed = entry.peer_links.upgrade().and_then(|peer| { - let mut table = peer.lock().unwrap_or_else(|p| p.into_inner()); - remove(&mut table, own_id) - }); - drop(removed); - deliver(&entry.signal, own_id, reason.clone()); } } -/// Created before constructing the actor future/thread closure, so shutdown -/// before its first poll also closes links. Weak reverse edges avoid Arc cycles. -pub(crate) struct LinkCleanup { - id: ActorId, - links: LinkTable, -} -impl LinkCleanup { - pub(crate) fn new(id: ActorId, links: LinkTable) -> Self { - Self { id, links } +/// Close the exit table at D: set the final reason and take one notice per +/// link. Takes only this actor's own lock. Terminal closure is idempotent. +pub(crate) fn close_links(own_id: ActorId, own: &LinkTable, reason: &ExitReason) -> LinkNotices { + let mut state = own.lock().unwrap_or_else(|p| p.into_inner()); + if state.exited.is_some() { + return LinkNotices::default(); } - pub(crate) fn complete(&self, reason: &ExitReason) { - propagate_exit(self.id, &self.links, reason); + state.exited = Some(reason.clone()); + LinkNotices { + from: Some(own_id), + reason: Some(reason.clone()), + entries: std::mem::take(&mut state.entries), } } -impl Drop for LinkCleanup { - fn drop(&mut self) { - self.complete(&ExitReason::Kill); - } + +/// Close and deliver at once (tests and paths with nothing in between). +#[cfg(test)] +pub(crate) fn propagate_exit(own_id: ActorId, own: &LinkTable, reason: &ExitReason) { + close_links(own_id, own, reason).deliver(); } -/// A local stop commits its cause before cancelling. A linked failure recorded -/// first wins; a late linked failure cannot relabel an already-requested stop. +/// A self-exit or administrative stop commits its cause before waking. The +/// first requested reason wins over later self-exits and stops; a later +/// untrapped signal or kill can still replace it until D. pub(crate) fn normal_stop(reason: &LinkedExitReason, cancel: impl FnOnce()) { stop_with_reason(reason, ExitReason::Normal, cancel); } @@ -260,12 +489,9 @@ pub(crate) fn stop_with_reason( cause: ExitReason, cancel: impl FnOnce(), ) { - let mut slot = reason.lock().unwrap_or_else(|p| p.into_inner()); - if slot.is_none() { - *slot = Some(cause); + if lock_request(reason).request(cause) { + cancel(); } - drop(slot); - cancel(); } #[cfg(test)] @@ -332,11 +558,15 @@ mod tests { } let drops = Arc::new(AtomicUsize::new(0)); let callback_drops = drops.clone(); - let signal: ExitSignalFn = - Arc::new(move |_, _| std::panic::panic_any(Payload(callback_drops.clone()))); + let mut receiver = target(&Arc::new(AtomicUsize::new(0))); + receiver.wake = Arc::new(move || std::panic::panic_any(Payload(callback_drops.clone()))); + let receiver = Arc::new(receiver); + let sender = Arc::new(target(&Arc::new(AtomicUsize::new(0)))); + let (a, b) = (new_link_table(), new_link_table()); + register_link(&receiver, &a, &sender, &b, false); let result = catch_unwind(AssertUnwindSafe(|| { tracing::subscriber::with_default(PanicSubscriber(drops.clone()), || { - deliver(&signal, ActorId::next(), ExitReason::Kill); + propagate_exit(sender.id, &b, &ExitReason::Kill); }); })); assert!(result.is_ok(), "delivery unwound through its caller"); @@ -367,94 +597,168 @@ mod tests { fn trap_exit_defaults_false() { assert!(!new_trap_exit_flag().load(Ordering::Acquire)); } - fn signal(counter: &Arc) -> ExitSignalFn { - let c = counter.clone(); - Arc::new(move |_, _| { - c.fetch_add(1, Ordering::SeqCst); - }) + /// A receiver whose wake and Exit deliveries are counted. + fn target(counter: &Arc) -> SignalTarget { + let (c1, c2, c3) = (counter.clone(), counter.clone(), counter.clone()); + SignalTarget { + id: ActorId::next(), + parent: None, + trap: new_trap_exit_flag(), + request: new_linked_exit_reason(), + wake: Arc::new(move || { + c1.fetch_add(1, Ordering::SeqCst); + }), + send_exit: Arc::new(move |_| { + c2.fetch_add(1, Ordering::SeqCst); + Ok(()) + }), + send_parent_exit: Arc::new(move |_| { + c3.fetch_add(1, Ordering::SeqCst); + Ok(()) + }), + send_held: Arc::new(|_, _| Ok(())), + } + } + fn pair(seen: &Arc) -> (Arc, LinkTable, Arc, LinkTable) { + ( + Arc::new(target(seen)), + new_link_table(), + Arc::new(target(seen)), + new_link_table(), + ) } #[test] fn closure_is_terminal_and_delivers_each_edge_once() { - let a = new_link_table(); - let b = new_link_table(); - let aid = ActorId::next(); - let bid = ActorId::next(); let seen = Arc::new(AtomicUsize::new(0)); - register_link(aid, &a, signal(&seen), bid, &b, signal(&seen)); - propagate_exit(aid, &a, &ExitReason::Kill); - propagate_exit(aid, &a, &ExitReason::Kill); + let (a, at, b, bt) = pair(&seen); + assert_eq!(register_link(&a, &at, &b, &bt, false), LinkOutcome::Linked); + propagate_exit(a.id, &at, &ExitReason::Panic("x".into())); + propagate_exit(a.id, &at, &ExitReason::Panic("x".into())); + assert_eq!(seen.load(Ordering::SeqCst), 1); + assert!(bt.lock().unwrap().entries.is_empty()); + // b links to the dead a: a late NoProc notice, delivered and removed. + assert_eq!(register_link(&b, &bt, &a, &at, false), LinkOutcome::NoProc); + assert_eq!(seen.load(Ordering::SeqCst), 1, "b already exiting: no second wake"); + assert_eq!(lock_request(&b.request).reason(), Some(&ExitReason::NoProc)); + assert!(at.lock().unwrap().entries.is_empty() && bt.lock().unwrap().entries.is_empty()); + } + #[test] + fn try_link_to_a_dead_actor_keeps_no_record() { + let seen = Arc::new(AtomicUsize::new(0)); + let (a, at, b, bt) = pair(&seen); + propagate_exit(b.id, &bt, &ExitReason::Normal); + assert_eq!(register_link(&a, &at, &b, &bt, true), LinkOutcome::NoProc); + assert_eq!(seen.load(Ordering::SeqCst), 0); + assert!(at.lock().unwrap().entries.is_empty()); + assert_eq!(lock_request(&a.request).reason(), None); + } + #[test] + fn unlink_fences_a_notice_taken_at_death() { + let seen = Arc::new(AtomicUsize::new(0)); + let (a, at, b, bt) = pair(&seen); + register_link(&a, &at, &b, &bt, false); + let notices = close_links(a.id, &at, &ExitReason::Panic("x".into())); + unregister_link(b.id, &bt, a.id, &at); + notices.deliver(); + assert_eq!(seen.load(Ordering::SeqCst), 0, "unlink returned before delivery"); + // A relink while the notice is pending is a no-op, as an existing record. + let (c, ct, d, dt) = pair(&seen); + register_link(&c, &ct, &d, &dt, false); + let notices = close_links(c.id, &ct, &ExitReason::Panic("y".into())); + assert_eq!(register_link(&d, &dt, &c, &ct, false), LinkOutcome::Unchanged); + notices.deliver(); assert_eq!(seen.load(Ordering::SeqCst), 1); - assert!(b.lock().unwrap().entries.is_empty()); - register_link(bid, &b, signal(&seen), aid, &a, signal(&seen)); - assert_eq!(seen.load(Ordering::SeqCst), 2); - assert!(a.lock().unwrap().entries.is_empty() && b.lock().unwrap().entries.is_empty()); + } + #[test] + fn signal_rules_follow_exit_signal_2() { + let seen = Arc::new(AtomicUsize::new(0)); + let t = target(&seen); + let from = ActorId::next(); + // Normal is ignored by a non-trapping target. + deliver_signal(&t, from, ExitReason::Normal, Origin::Direct, false); + assert_eq!(lock_request(&t.request).reason(), None); + // Any other reason begins exit without stopped(). + deliver_signal(&t, from, ExitReason::Shutdown, Origin::Direct, false); + assert_eq!(lock_request(&t.request).reason(), Some(&ExitReason::Shutdown)); + assert!(!lock_request(&t.request).begin_hook()); + // Trapping: queued as Exit, nothing committed. + let t = target(&seen); + t.trap.store(true, Ordering::Release); + deliver_signal(&t, from, ExitReason::Shutdown, Origin::Direct, false); + assert_eq!(lock_request(&t.request).reason(), None); + // A Kill signal is untrappable and ends Killed; a Kill link notice is trappable. + deliver_signal(&t, from, ExitReason::Kill, Origin::Link, false); + assert_eq!(lock_request(&t.request).reason(), None); + deliver_signal(&t, from, ExitReason::Kill, Origin::Direct, false); + assert_eq!(lock_request(&t.request).reason(), Some(&ExitReason::Killed)); + } + #[test] + fn reason_precedence_until_death() { + // A late non-kill signal replaces a requested reason (decision 8); + // a kill supersedes it and nothing replaces Killed. + let mut r = ExitRequest::default(); + assert!(r.request(ExitReason::Normal)); + assert!(r.untrapped(ExitReason::Shutdown)); + assert_eq!(r.reason(), Some(&ExitReason::Shutdown)); + assert!(r.begin_hook(), "a late signal does not skip a self-exit's stopped()"); + r.panic(ExitReason::Panic("stopped".into())); + assert_eq!(r.reason(), Some(&ExitReason::Panic("stopped".into()))); + assert!(r.kill()); + r.panic(ExitReason::Panic("after kill".into())); + assert!(!r.untrapped(ExitReason::Shutdown)); + assert!(r.request(ExitReason::Normal)); + assert_eq!(r.die(), ExitReason::Killed); + assert!(!r.kill() && !r.untrapped(ExitReason::Shutdown)); + // Runtime drop: Killed unless a non-Normal reason was requested. + let mut r = ExitRequest::default(); + r.request(ExitReason::Normal); + assert_eq!(r.runtime_drop(), ExitReason::Killed); + let mut r = ExitRequest::default(); + r.request(ExitReason::Shutdown); + assert_eq!(r.runtime_drop(), ExitReason::Shutdown); } #[test] fn self_link_is_noop() { - let a = new_link_table(); - let id = ActorId::next(); let seen = Arc::new(AtomicUsize::new(0)); - register_link(id, &a, signal(&seen), id, &a, signal(&seen)); - assert!(a.lock().unwrap().entries.is_empty()); - assert_eq!(Arc::strong_count(&a), 1); + let a = Arc::new(target(&seen)); + let at = new_link_table(); + register_link(&a, &at, &a, &at, false); + assert!(at.lock().unwrap().entries.is_empty()); + assert_eq!(Arc::strong_count(&at), 1); } #[test] fn weak_edges_do_not_keep_link_tables_alive() { - let a = new_link_table(); - let b = new_link_table(); - let weak_a = Arc::downgrade(&a); - let weak_b = Arc::downgrade(&b); let c = Arc::new(AtomicUsize::new(0)); - register_link( - ActorId::next(), - &a, - signal(&c), - ActorId::next(), - &b, - signal(&c), - ); - drop(a); - drop(b); + let (a, at, b, bt) = pair(&c); + let weak_a = Arc::downgrade(&at); + let weak_b = Arc::downgrade(&bt); + register_link(&a, &at, &b, &bt, false); + drop(at); + drop(bt); assert!(weak_a.upgrade().is_none() && weak_b.upgrade().is_none()); } #[test] fn concurrent_link_unlink_preserves_symmetry() { - let a = new_link_table(); - let b = new_link_table(); - let aid = ActorId::next(); - let bid = ActorId::next(); let c = Arc::new(AtomicUsize::new(0)); + let (a, at, b, bt) = pair(&c); std::thread::scope(|scope| { scope.spawn(|| { for _ in 0..2000 { - register_link(aid, &a, signal(&c), bid, &b, signal(&c)); - unregister_link(aid, &a, bid, &b); + register_link(&a, &at, &b, &bt, false); + unregister_link(a.id, &at, b.id, &bt); } }); scope.spawn(|| { for _ in 0..2000 { - register_link(bid, &b, signal(&c), aid, &a, signal(&c)); - unregister_link(bid, &b, aid, &a); + register_link(&b, &bt, &a, &at, false); + unregister_link(b.id, &bt, a.id, &at); } }); }); assert_eq!( - a.lock().unwrap().entries.len(), - b.lock().unwrap().entries.len() - ); - assert!(a.lock().unwrap().entries.len() <= 1); - } - #[test] - fn normal_stop_cause_is_not_replaced_by_late_failure() { - let reason = new_linked_exit_reason(); - normal_stop(&reason, || {}); - let signal = make_signal( - new_trap_exit_flag(), - Arc::new(|| {}), - Arc::new(|_| Ok(())), - reason.clone(), + at.lock().unwrap().entries.len(), + bt.lock().unwrap().entries.len() ); - signal(ActorId::next(), ExitReason::Panic("late".into())); - assert_eq!(*reason.lock().unwrap(), Some(ExitReason::Normal)); + assert!(at.lock().unwrap().entries.len() <= 1); } } diff --git a/concurrency/src/mailbox.rs b/concurrency/src/mailbox.rs index 2cace291..117faf04 100644 --- a/concurrency/src/mailbox.rs +++ b/concurrency/src/mailbox.rs @@ -13,6 +13,13 @@ impl DropFailures { pub(crate) fn first(&self) -> Option { self.0.lock().unwrap_or_else(|p| p.into_inner()).clone() } + /// Record a cleanup failure; the first one is kept. + pub(crate) fn record(&self, message: String) { + self.0 + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get_or_insert(message); + } pub(crate) fn discard(&self, value: T) { if let Err(payload) = catch_unwind(AssertUnwindSafe(|| drop(value))) { let message = crate::error::panic_message(&*payload); diff --git a/concurrency/src/system.rs b/concurrency/src/system.rs index ccf0db63..2bd2ca27 100644 --- a/concurrency/src/system.rs +++ b/concurrency/src/system.rs @@ -1273,8 +1273,8 @@ impl Drop for Lease { .scope .child() .and_then(|c| c.exit_reason()) - .or_else(|| lock(&self.scope.cause).clone()) - .unwrap_or(ExitReason::Kill); + .or_else(|| crate::link::lock_request(&self.scope.cause).reason().cloned()) + .unwrap_or(ExitReason::Killed); self.scope.ready.send_replace(StartupState::Failed(reason)); } } diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index eed8d055..3b90020c 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -1,8 +1,10 @@ use crate::child_handle::{ActorId, ChildHandle}; use crate::error::{panic_message, ActorError, ExitReason}; +use crate::child_handle::Completion; +use crate::error::NoProc; use crate::link::{ - self, new_link_table, new_linked_exit_reason, new_trap_exit_flag, Exit, LinkTable, - LinkedExitReason, SendExitFn, TrapExitFlag, + self, new_link_table, new_linked_exit_reason, new_trap_exit_flag, Exit, LinkOutcome, + LinkTable, LinkedExitReason, SendExitFn, SendHeldFn, SignalTarget, TrapExitFlag, }; use crate::mailbox::tasks as mpsc; use crate::message::Message; @@ -270,6 +272,95 @@ impl Envelope for ExitEnvelope { } } +/// A parent exit (E3): when dispatched, exit begins with the parent's reason +/// and `stopped()` runs. `exit_received` never sees it. +struct ParentExitEnvelope { + exit: Exit, +} + +impl Envelope for ParentExitEnvelope { + #[cfg(feature = "diagnostics")] + fn message_type(&self) -> &'static str { + "ParentExit" + } + + fn handle<'a>( + self: Box, + _actor: &'a mut A, + ctx: &'a Context, + ) -> Pin + Send + 'a>> { + Box::pin(async move { ctx.stop_with_reason(self.exit.reason) }) + } +} + +/// An exit signal that arrived while `started()` ran, in its mailbox place +/// (phase 0). If the actor trapped exits when `started()` returned it is an +/// `Exit` (or parent exit); otherwise it was resolved then and is inert. +struct HeldEnvelope { + exit: Exit, + parent: bool, +} + +impl Envelope for HeldEnvelope { + #[cfg(feature = "diagnostics")] + fn message_type(&self) -> &'static str { + "Exit" + } + + fn handle<'a>( + self: Box, + actor: &'a mut A, + ctx: &'a Context, + ) -> Pin + Send + 'a>> { + Box::pin(async move { + if !link::lock_request(&ctx.shared.linked_reason).held_trapped() { + return; + } + if self.parent { + ctx.stop_with_reason(self.exit.reason); + } else { + actor.exit_received(self.exit, ctx).await; + } + }) + } +} + +/// Build the signal target of a new actor instance. +fn signal_target( + id: ActorId, + parent: Option, + sender: &mpsc::Sender + Send>>, + trap: TrapExitFlag, + request: LinkedExitReason, + token: CancellationToken, +) -> Arc { + let exit_tx = sender.clone(); + let parent_tx = sender.clone(); + let held_tx = sender.clone(); + let send_exit: SendExitFn = Arc::new(move |exit: Exit| { + let envelope: Box + Send> = Box::new(ExitEnvelope { exit }); + exit_tx.send(envelope).map_err(|_| ActorError::ActorStopped) + }); + let send_parent_exit: SendExitFn = Arc::new(move |exit: Exit| { + let envelope: Box + Send> = Box::new(ParentExitEnvelope { exit }); + parent_tx.send(envelope).map_err(|_| ActorError::ActorStopped) + }); + let send_held: SendHeldFn = Arc::new(move |exit: Exit, parent: bool| { + let envelope: Box + Send> = Box::new(HeldEnvelope { exit, parent }); + held_tx.send(envelope).map_err(|_| ActorError::ActorStopped) + }); + Arc::new(SignalTarget { + id, + parent, + trap, + request, + wake: Arc::new(move || token.cancel()), + send_exit, + send_parent_exit, + send_held, + }) +} + // --------------------------------------------------------------------------- // Context // --------------------------------------------------------------------------- @@ -298,6 +389,8 @@ struct ActorShared { trap_exit: TrapExitFlag, links: LinkTable, linked_reason: LinkedExitReason, + /// This instance as the target of exit signals and link notices. + signal: Arc, mailbox: crate::instrumentation::Mailbox, managed: Option>, /// This instance's reply registry: deferred tokens, credits, turn phase. @@ -351,24 +444,38 @@ impl Context { self.shared.id } - /// Signal the actor to stop. The current handler will finish, then - /// `stopped()` is called and the actor exits. + /// Self-exit: the current handler finishes, then `stopped()` is called + /// and the actor exits `Normal`. pub fn stop(&self) { self.stop_with_reason(ExitReason::Normal); } - /// Request a cooperative exit with an explicit reason, without unwinding. - /// The current handler finishes and `stopped()` runs. The first committed - /// stop/link reason wins; a cleanup panic can still make the exit abnormal. - /// `Shutdown` is non-abnormal; `Kill` requests an abnormal exit and can - /// trigger transient supervision or propagate through links. It does not - /// forcibly interrupt running Rust code. + /// Self-exit with an explicit reason (OTP `{stop, Reason, State}`), without + /// unwinding. The current handler finishes and `stopped()` runs. The first + /// requested self-exit or stop wins; until the death point a kill + /// replaces it with `Killed`, an untrapped exit signal with its reason, and + /// a panic (in the handler or `stopped()`) with `Panic`. `Kill` stays + /// `Kill` and propagates over links as `Kill` (trappable). pub fn stop_with_reason(&self, reason: ExitReason) { link::stop_with_reason(&self.shared.linked_reason, reason, || { self.shared.cancellation_token.cancel() }); } + /// Send an exit signal from this actor to `target` (OTP + /// `exit_signal(Target, Reason)`); see [`ChildHandle::exit`] for the + /// delivery rules. From the target's `start_linked` parent the signal is a + /// parent exit (a trapping target ends normally, through `stopped()`). + /// Works until this actor's death point, including from `stopped()`; + /// inert after it. A signal to this actor's own handle is a signal, not a + /// self-exit: a non-trapping actor ignores `Normal`. + pub fn exit(&self, target: &ChildHandle, reason: ExitReason) { + if link::lock_request(&self.shared.linked_reason).is_dead() { + return; + } + target.exit_from(self.shared.id, reason, false); + } + /// Send a fire-and-forget message to this actor. pub fn send(&self, msg: M) -> Result<(), ActorError> where @@ -467,10 +574,12 @@ impl Context { /// Set up a unidirectional monitor on another actor. /// /// Returns a [`MonitorRef`] that can be used to cancel the monitor via - /// [`Context::demonitor`]. When the monitored actor stops, a [`Down`] - /// message is delivered to this actor's mailbox via `Handler`. + /// [`Context::demonitor`]. When the monitored actor's exit is published, + /// a [`Down`] with its final reason is delivered via `Handler`. /// - /// If the target is already dead, a `Down` message is delivered immediately. + /// If the target is past its death point, exactly one + /// `Down { reason: NoProc }` is delivered at once (OTP `noproc`; use + /// [`ChildHandle::exit_reason`] for the real reason). /// /// Multiple independent monitors are allowed on the same target — each /// call returns a distinct `MonitorRef`. @@ -502,6 +611,15 @@ impl Context { let Some(monitors) = monitors.as_mut() else { return monitor_ref; }; + // Linearised against the target's death point (E4): past it the + // only notice is NoProc, delivered under the observer's lock. + if target.is_past_death() { + let _ = self.send(Down { + monitor_ref, + reason: ExitReason::NoProc, + }); + return monitor_ref; + } monitors.insert(monitor_ref, active.clone()); } @@ -521,21 +639,21 @@ impl Context { futures::future::Either::Left(_) => return, futures::future::Either::Right((reason, _)) => reason, }; - // Remove the entry from the monitor table so it doesn't accumulate - // stale entries over the actor's lifetime. Done before delivery - // since `demonitor` is now a no-op for this monitor anyway. - let deliver = monitors - .lock() - .unwrap_or_else(|p| p.into_inner()) + // Receiver-side fence (E6): the entry is removed and the Down + // enqueued in one critical section, so once `demonitor` returns + // no Down of this monitor is enqueued. + let mut entries = monitors.lock().unwrap_or_else(|p| p.into_inner()); + if entries .as_mut() .and_then(|entries| entries.remove(&monitor_ref)) - .is_some(); - if deliver { + .is_some() + { let _ = actor_ref.send(Down { monitor_ref, reason, }); } + drop(entries); }); monitor_ref @@ -543,10 +661,8 @@ impl Context { /// Cancel a previously-set monitor. /// - /// The monitor waiter is released promptly. - /// If the target hasn't yet died, no `Down` message will be delivered. - /// If a `Down` message has already been delivered (or queued), this is - /// a best-effort cancellation — the message may still arrive. + /// A fence (OTP `demonitor/1`): once it returns, no `Down` of this + /// monitor is enqueued; one already queued stays. pub fn demonitor(&self, monitor_ref: MonitorRef) { let active = self .shared @@ -561,44 +677,49 @@ impl Context { } } - /// Set up a bidirectional link with another actor. + /// Set up a bidirectional link with another actor (OTP `link/1`). /// - /// When either side dies abnormally, the other receives an exit signal. - /// By default the receiver is terminated by the signal. Call - /// [`trap_exit(true)`] to convert signals into `Exit` messages delivered - /// via [`Actor::exit_received`] instead. + /// When either side's exit is published, the other receives a link + /// notice with the final reason: a non-trapping peer ignores `Normal` and + /// otherwise exits with that reason without running `stopped()`; a + /// trapping peer gets an `Exit` via [`Actor::exit_received`]. /// - /// If the target is already dead with an abnormal reason, the exit - /// signal is delivered immediately without retaining a link. Repeating an - /// existing live link is a no-op; each attempt to link a closed peer reports - /// that peer's exit for the new attempt. + /// If the target is past its death point, this actor receives a `NoProc` + /// notice: an `Exit { reason: NoProc }` when trapping, otherwise it exits + /// `NoProc` (decision 1). Use [`try_link`](Self::try_link) to get + /// `Err(NoProc)` instead. An existing link, or a pending notice from that + /// peer, makes this a no-op. /// - /// [`trap_exit(true)`]: Self::trap_exit /// [`Actor::exit_received`]: Actor::exit_received pub fn link(&self, target: &ChildHandle) { - let own_signal = link::make_signal( - self.shared.trap_exit.clone(), - self.own_cancel_fn(), - self.own_send_exit_fn(), - self.shared.linked_reason.clone(), - ); - let peer_signal = link::make_signal( - target.trap_exit_flag().clone(), - target.cancel_fn().clone(), - target.send_exit_fn().clone(), - target.linked_reason().clone(), - ); link::register_link( - self.shared.id, + &self.shared.signal, &self.shared.links, - own_signal, - target.id(), + target.signal_target(), target.links(), - peer_signal, + false, ); } - /// Remove a previously-set bidirectional link. + /// Link as [`link`](Self::link), but report a target past its death + /// point as `Err(NoProc)` and keep no record, whether or not this actor + /// traps exits. + pub fn try_link(&self, target: &ChildHandle) -> Result<(), NoProc> { + match link::register_link( + &self.shared.signal, + &self.shared.links, + target.signal_target(), + target.links(), + true, + ) { + LinkOutcome::NoProc => Err(NoProc), + LinkOutcome::Linked | LinkOutcome::Unchanged => Ok(()), + } + } + + /// Remove a link. A fence (OTP `unlink/1`): once it returns, the link has + /// no further effect on this actor, including a notice already taken at + /// the peer's death; an `Exit` already queued stays. pub fn unlink(&self, target: &ChildHandle) { link::unregister_link( self.shared.id, @@ -608,33 +729,23 @@ impl Context { ); } - /// Control how exit signals from linked actors are handled. + /// Control how exit signals (link notices and `exit` signals) are + /// handled; read when each signal arrives (OTP `trap_exit`). /// - /// - `false` (default): an abnormal exit signal from a linked actor cancels - /// this actor (propagating the death). - /// - `true`: the signal is converted to an `Exit` message and delivered to + /// - `false` (default): `Normal` is ignored; any other reason ends this + /// actor after its current handler, without `stopped()`. + /// - `true`: the signal is queued as an `Exit` for /// [`Actor::exit_received`]. /// - /// A direct `stop_with_reason(Kill)` cannot be trapped. Propagated link - /// notices, including Kill, are delivered as Exit when trapping is enabled. - /// Cancellation remains cooperative: an in-flight handler is not preempted. + /// Only a kill is untrappable. A parent's exit (the `start_linked` parent, + /// the supervisor or the system) ends a trapping actor normally, through + /// `stopped()`, and never reaches `exit_received`. Set it first thing in + /// `started()`: signals that arrive while `started()` runs are resolved + /// when it returns. Handlers are never preempted. pub fn trap_exit(&self, enabled: bool) { self.shared.trap_exit.store(enabled, Ordering::Release); } - /// Build a type-erased cancel closure for this actor. - fn own_cancel_fn(&self) -> Arc { - let token = self.shared.cancellation_token.clone(); - let reason = self.shared.linked_reason.clone(); - Arc::new(move || link::normal_stop(&reason, || token.cancel())) - } - - /// Build a type-erased `SendExitFn` that enqueues an `ExitEnvelope` - /// onto this actor's mailbox. - fn own_send_exit_fn(&self) -> SendExitFn { - self.shared.send_exit_fn() - } - /// Type-erased handle to this actor, without an intermediate `ActorRef`. pub(crate) fn child_handle(&self) -> ChildHandle { self.shared.child_handle() @@ -944,24 +1055,17 @@ impl ActorRef { } } - /// Poll the exit reason. Returns `None` if the actor is still running. - /// A completion channel closed without a reason reports [`ExitReason::Kill`], - /// matching the wait methods (for example, after runtime shutdown). + /// Poll the exit reason. Returns `None` until the exit is published. + /// A completion channel closed without a reason reports + /// [`ExitReason::Killed`], matching the wait methods (for example, after + /// runtime shutdown). pub fn exit_reason(&self) -> Option { crate::child_handle::poll_exit_reason(&self.shared.completion_rx) } - /// Wait until the actor stops and return the exit reason. + /// Wait until the actor's exit is published and return its final reason. pub async fn wait_exit(&self) -> ExitReason { - let mut rx = self.shared.completion_rx.clone(); - loop { - if let Some(reason) = rx.borrow_and_update().clone() { - return reason; - } - if rx.changed().await.is_err() { - return ExitReason::Kill; - } - } + crate::child_handle::wait_loop(self.shared.completion_rx.clone()).await } /// Waiting messages, including in-flight enqueue reservations, excluding @@ -990,25 +1094,11 @@ impl ActorRef { } impl ActorShared { - /// Build a type-erased `SendExitFn` that enqueues an `ExitEnvelope` - /// onto this actor's mailbox. - fn send_exit_fn(&self) -> SendExitFn { - let sender = self.sender.clone(); - Arc::new(move |exit: Exit| { - let envelope: Box + Send> = Box::new(ExitEnvelope { exit }); - sender.send(envelope).map_err(|_| ActorError::ActorStopped) - }) - } - fn child_handle(&self) -> ChildHandle { - ChildHandle::from_tasks( - self.id, - self.cancellation_token.clone(), - self.completion_rx.clone(), - self.trap_exit.clone(), + ChildHandle::new( + self.signal.clone(), + Completion::Tasks(self.completion_rx.clone()), self.links.clone(), - self.linked_reason.clone(), - self.send_exit_fn(), ) .with_mailbox_failures(self.sender.failures()) } @@ -1082,6 +1172,16 @@ impl ActorRef { .unwrap_or_else(new_linked_exit_reason); let id = ActorId::next(); + // Phase 0: signals that arrive before started() returns are held. + link::lock_request(&linked_reason).begin_starting(); + let signal = signal_target::( + id, + parent.as_ref().map(ChildHandle::id), + &tx, + trap_exit.clone(), + linked_reason.clone(), + cancellation_token.clone(), + ); let (cleaned_tx, cleaned_rx) = watch::channel(false); let actor_ref = ActorRef { shared: Arc::new(ActorShared { @@ -1099,6 +1199,7 @@ impl ActorRef { trap_exit, links: links.clone(), linked_reason: linked_reason.clone(), + signal, mailbox: crate::instrumentation::Mailbox::default(), managed: managed.clone(), }), @@ -1153,7 +1254,6 @@ impl ActorRef { linked_reason, actor_ref.shared.monitors.clone(), completion_tx, - ExitReason::Kill, move || cleanup_token.cancel(), ) .with_mailbox_failures(actor_ref.shared.sender.failures()); @@ -1175,30 +1275,22 @@ impl ActorRef { let _borrowed_runtime = borrowed_runtime; #[cfg(feature = "backend-experiments")] let _borrowed_dispatcher = borrowed_dispatcher; - let reason = run_actor( + run_actor( actor, ctx, rx, cancellation_token, + &mut cleanup, #[cfg(feature = "backend-experiments")] turn, ) .await; - let reason = match reason { - Ok(reason) => { - cleanup.complete(reason.clone()); - reason - } - Err(reason) => { - cleanup.complete_secondary(reason.clone()); - reason - } - }; + let reason = cleanup.die(); if let Some(scope) = &managed { - let actual = crate::system::lock(&scope.cause).clone().unwrap_or(reason); - scope.exited(actual); + scope.exited(reason); } - // Publish the exit (monitors and links fire) before the job join. + // Publication (wait_exit, monitors, link notices) after the + // state drop, before the job join. drop(cleanup); // Job cleanup (deferred-replies §5 step 4), after death: join the // cancelled jobs, bounded by the cleanup deadline; a job still @@ -1364,21 +1456,23 @@ impl Drop for ExitOnDrop { } } -async fn run_actor( +async fn run_actor( actor: A, ctx: Context, mut rx: mpsc::Receiver + Send>>, cancellation_token: CancellationToken, + cleanup: &mut crate::actor_cleanup::ActorCleanup, #[cfg(feature = "backend-experiments")] turn: Option, -) -> Result { +) { let replies = ctx.shared.replies.clone(); let jobs = ctx.shared.jobs.clone(); + let request = ctx.shared.linked_reason.clone(); let mut owned = ExitOnDrop { actor: Some(actor), replies: replies.clone(), jobs: jobs.clone(), }; - let primary = match AssertUnwindSafe(run_actor_lifecycle( + if let Err(payload) = AssertUnwindSafe(run_actor_lifecycle( owned.actor.as_mut().expect("the state lives until death"), ctx, &mut rx, @@ -1389,26 +1483,28 @@ async fn run_actor( .catch_unwind() .await { - Ok(reason) => Ok(reason), - Err(payload) => { - let message = panic_message(&*payload); - crate::mailbox::dispose_panic(payload); - Err(ExitReason::Panic(format!( - "actor lifecycle unwound: {message}" - ))) - } - }; - // Death (deferred-replies §5 step 3), only now that `stopped()` returned: - // close the registry (every open token answers ActorStopped), dispose - // the mailbox (queued requests answer ActorStopped), drop the state. - // Early returns (startup panic, closed scope) and an unwound lifecycle - // also begin exit here, so jobs never outlive their owner. + let message = panic_message(&*payload); + crate::mailbox::dispose_panic(payload); + link::lock_request(&request).panic(ExitReason::Panic(format!( + "actor lifecycle unwound: {message}" + ))); + } + // The death point D, only now that `stopped()` returned (or was + // skipped): the reason becomes final, the link table closes, the + // registry closes (every open token is answered) and the mailbox refuses + // admission (queued requests are answered). Early returns (startup + // panic, closed scope) and an unwound lifecycle also come here, so jobs + // never outlive their owner. replies.exiting(); jobs.cancel_all(); + cleanup.die(); replies.close(); drop(rx); + // Destruction: a destructor panic is a cleanup failure, never the reason. let actor = owned.actor.take().expect("the state lives until death"); - crate::actor_cleanup::finish_actor_state(actor, primary) + if let Some(failure) = crate::actor_cleanup::finish_actor_state(actor) { + cleanup.cleanup_failed(failure); + } } async fn run_actor_lifecycle( @@ -1417,12 +1513,13 @@ async fn run_actor_lifecycle( rx: &mut mpsc::Receiver + Send>>, cancellation_token: CancellationToken, #[cfg(feature = "backend-experiments")] mut turn: Option, -) -> ExitReason { +) { let _monitor_cleanup = MonitorCleanup(ctx.shared.monitors.clone()); + let request = ctx.shared.linked_reason.clone(); if let Some(scope) = &ctx.shared.managed { if !scope.can_start() { scope.stop(ExitReason::Shutdown); - return ExitReason::Normal; + return; } } let start_result = AssertUnwindSafe(async { @@ -1436,8 +1533,15 @@ async fn run_actor_lifecycle( crate::actor_cleanup::log_safely(|| { tracing::error!("Panic in started() callback: {msg}"); }); + // No stopped() after a failed start (L5). + link::lock_request(&request).panic(ExitReason::Panic(format!("panic in started(): {msg}"))); + cancellation_token.cancel(); + return; + } + // Phase 0 ends: resolve signals held during started() with the trap + // flag read now. + if link::lock_request(&request).end_starting(ctx.shared.trap_exit.load(Ordering::Acquire)) { cancellation_token.cancel(); - return ExitReason::Panic(format!("panic in started(): {msg}")); } fail_point!("actor.after_started_before_ready", async); @@ -1447,31 +1551,7 @@ async fn run_actor_lifecycle( } } - if cancellation_token.is_cancelled() { - if let Some(scope) = &ctx.shared.managed { - scope.end_running(); - } - ctx.shared.replies.exiting(); - ctx.shared.jobs.cancel_all(); - if let Err(panic) = AssertUnwindSafe(async { - fail_point!("actor.before_stopped", async); - actor.stopped(&ctx).await - }) - .catch_unwind() - .await - { - let msg = panic_message(&*panic); - crate::actor_cleanup::log_safely(|| { - tracing::error!("Panic in stopped() callback: {msg}"); - }); - return ExitReason::Panic(format!("panic in stopped(): {msg}")); - } - return ExitReason::Normal; - } - - let mut exit_reason = ExitReason::Normal; - - loop { + while !cancellation_token.is_cancelled() { let msg = { let recv = pin!(future::poll_fn(|cx| { // Stop and closed-mailbox both take the normal lifecycle exit, @@ -1528,7 +1608,10 @@ async fn run_actor_lifecycle( crate::actor_cleanup::log_safely(|| { tracing::error!("Panic in message handler: {msg}"); }); - exit_reason = ExitReason::Panic(format!("panic in handler: {msg}")); + // A panic begins exit, or replaces the requested reason + // unless a kill was received. + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in handler: {msg}"))); break; } #[cfg(feature = "backend-experiments")] @@ -1551,6 +1634,11 @@ async fn run_actor_lifecycle( cancellation_token.cancel(); ctx.shared.replies.exiting(); ctx.shared.jobs.cancel_all(); + // stopped() is skipped when a kill, an untrapped exit signal or link + // notice began the exit (decision 2); one that already began finishes. + if !link::lock_request(&request).begin_hook() { + return; + } let stop_result = AssertUnwindSafe(async { fail_point!("actor.before_stopped", async); actor.stopped(&ctx).await @@ -1562,12 +1650,8 @@ async fn run_actor_lifecycle( crate::actor_cleanup::log_safely(|| { tracing::error!("Panic in stopped() callback: {msg}"); }); - if !exit_reason.is_abnormal() { - exit_reason = ExitReason::Panic(format!("panic in stopped(): {msg}")); - } + link::lock_request(&request).panic(ExitReason::Panic(format!("panic in stopped(): {msg}"))); } - - exit_reason } // --------------------------------------------------------------------------- @@ -1594,9 +1678,12 @@ pub trait ActorStart: Actor { ActorRef::spawn(self, backend, None) } - /// Register the bidirectional link before the child can run `started()`. - /// A closed parent is observed under its link-state mutex; an abnormal - /// parent exit cancels the child cooperatively before message dispatch. + /// Start linked to `parent_ctx`'s actor, which becomes this instance's + /// parent (E3). The link is registered before `started()`. The parent's + /// exit ends a trapping child normally (its `stopped()` runs; it never + /// reaches `exit_received`), a non-trapping child as any link notice. A + /// parent already past its death point gives the child a `NoProc` parent + /// exit. fn start_linked(self, parent_ctx: &Context

) -> ActorRef { ActorRef::spawn(self, Backend::Async, Some(parent_ctx.child_handle())) } diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 801b5bae..4380b80d 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -8,11 +8,11 @@ use std::{ time::Duration, }; -use crate::child_handle::{ActorId, ChildHandle}; -use crate::error::{panic_message, ActorError, ExitReason}; +use crate::child_handle::{ActorId, ChildHandle, Completion}; +use crate::error::{panic_message, ActorError, ExitReason, NoProc}; use crate::link::{ - self, new_link_table, new_linked_exit_reason, new_trap_exit_flag, Exit, LinkTable, - LinkedExitReason, SendExitFn, TrapExitFlag, + self, new_link_table, new_linked_exit_reason, new_trap_exit_flag, Exit, LinkOutcome, + LinkTable, LinkedExitReason, SendExitFn, SendHeldFn, SignalTarget, TrapExitFlag, }; use crate::message::Message; use crate::monitor::{Down, MonitorCleanup, MonitorRef, MonitorTable}; @@ -164,6 +164,81 @@ impl Envelope for ExitEnvelope { } } +/// A parent exit (E3): when dispatched, exit begins with the parent's reason +/// and `stopped()` runs. `exit_received` never sees it. +struct ParentExitEnvelope { + exit: Exit, +} + +impl Envelope for ParentExitEnvelope { + fn handle(self: Box, _actor: &mut A, ctx: &Context) { + ctx.stop_with_reason(self.exit.reason); + } +} + +/// An exit signal that arrived while `started()` ran, in its mailbox place +/// (phase 0). If the actor trapped exits when `started()` returned it is an +/// `Exit` (or parent exit); otherwise it was resolved then and is inert. +struct HeldEnvelope { + exit: Exit, + parent: bool, +} + +impl Envelope for HeldEnvelope { + fn handle(self: Box, actor: &mut A, ctx: &Context) { + if !link::lock_request(&ctx.linked_reason).held_trapped() { + return; + } + if self.parent { + ctx.stop_with_reason(self.exit.reason); + } else { + actor.exit_received(self.exit, ctx); + } + } +} + +/// Build the signal target of a new actor instance. +fn signal_target( + id: ActorId, + parent: Option, + sender: &mpsc::Sender>, + trap: TrapExitFlag, + request: LinkedExitReason, + token: CancellationToken, +) -> Arc { + let exit_tx = sender.clone(); + let parent_tx = sender.clone(); + let held_tx = sender.clone(); + let send_exit: SendExitFn = Arc::new(move |exit: Exit| { + let envelope: Box + Send> = Box::new(ExitEnvelope { exit }); + exit_tx + .send(MailboxItem::Message(envelope)) + .map_err(|_| ActorError::ActorStopped) + }); + let send_parent_exit: SendExitFn = Arc::new(move |exit: Exit| { + let envelope: Box + Send> = Box::new(ParentExitEnvelope { exit }); + parent_tx + .send(MailboxItem::Message(envelope)) + .map_err(|_| ActorError::ActorStopped) + }); + let send_held: SendHeldFn = Arc::new(move |exit: Exit, parent: bool| { + let envelope: Box + Send> = Box::new(HeldEnvelope { exit, parent }); + held_tx + .send(MailboxItem::Message(envelope)) + .map_err(|_| ActorError::ActorStopped) + }); + Arc::new(SignalTarget { + id, + parent, + trap, + request, + wake: Arc::new(move || token.cancel()), + send_exit, + send_parent_exit, + send_held, + }) +} + // A private wakeup is distinct from user envelopes and never invokes a handler. enum MailboxItem { Message(Box + Send>), @@ -196,6 +271,7 @@ pub struct Context { trap_exit: TrapExitFlag, links: LinkTable, linked_reason: LinkedExitReason, + signal: Arc, mailbox: crate::instrumentation::Mailbox, replies: Arc, jobs: Arc, @@ -213,6 +289,7 @@ impl Clone for Context { trap_exit: self.trap_exit.clone(), links: self.links.clone(), linked_reason: self.linked_reason.clone(), + signal: self.signal.clone(), mailbox: self.mailbox.clone(), replies: self.replies.clone(), jobs: self.jobs.clone(), @@ -240,6 +317,7 @@ impl Context { trap_exit: actor_ref.trap_exit.clone(), links: actor_ref.links.clone(), linked_reason: actor_ref.linked_reason.clone(), + signal: actor_ref.signal.clone(), mailbox: actor_ref.mailbox.clone(), replies: actor_ref.replies.clone(), jobs: actor_ref.jobs.clone(), @@ -266,24 +344,37 @@ impl Context { self.id } - /// Signal the actor to stop. The current handler will finish, then - /// `stopped()` is called and the actor exits. + /// Self-exit: the current handler finishes, then `stopped()` is called + /// and the actor exits `Normal`. pub fn stop(&self) { self.stop_with_reason(ExitReason::Normal); } - /// Request a cooperative exit with an explicit reason, without unwinding. - /// The current handler finishes and `stopped()` runs. The first committed - /// stop/link reason wins; a cleanup panic can still make the exit abnormal. - /// `Shutdown` is non-abnormal; `Kill` requests an abnormal exit and can - /// trigger transient supervision or propagate through links. It does not - /// forcibly interrupt running Rust code. + /// Self-exit with an explicit reason (OTP `{stop, Reason, State}`), without + /// unwinding. The current handler finishes and `stopped()` runs. The first + /// requested self-exit or stop wins; until the death point a kill + /// replaces it with `Killed`, an untrapped exit signal with its reason, and + /// a panic (in the handler or `stopped()`) with `Panic`. `Kill` stays + /// `Kill` and propagates over links as `Kill` (trappable). pub fn stop_with_reason(&self, reason: ExitReason) { link::stop_with_reason(&self.linked_reason, reason, || { self.cancellation_token.cancel() }); } + /// Send an exit signal from this actor to `target` (OTP + /// `exit_signal(Target, Reason)`); see [`ChildHandle::exit`] for the + /// delivery rules. From the target's `start_linked` parent the signal is a + /// parent exit (a trapping target ends normally, through `stopped()`). + /// Works until this actor's death point, including from `stopped()`; + /// inert after it. + pub fn exit(&self, target: &ChildHandle, reason: ExitReason) { + if link::lock_request(&self.linked_reason).is_dead() { + return; + } + target.exit_from(self.id, reason, false); + } + /// Send a fire-and-forget message to this actor. pub fn send(&self, msg: M) -> Result<(), ActorError> where @@ -368,6 +459,7 @@ impl Context { trap_exit: self.trap_exit.clone(), links: self.links.clone(), linked_reason: self.linked_reason.clone(), + signal: self.signal.clone(), mailbox: self.mailbox.clone(), replies: self.replies.clone(), jobs: self.jobs.clone(), @@ -381,7 +473,8 @@ impl Context { /// [`Context::demonitor`]. When the monitored actor stops, a [`Down`] /// message is delivered to this actor's mailbox via `Handler`. /// - /// If the target is already dead, a `Down` message is delivered immediately. + /// If the target is past its death point, exactly one + /// `Down { reason: NoProc }` is delivered at once (OTP `noproc`). /// /// Multiple independent monitors are allowed on the same target — each /// call returns a distinct `MonitorRef`. @@ -414,6 +507,15 @@ impl Context { let Some(monitors) = monitors.as_mut() else { return monitor_ref; }; + // Linearised against the target's death point (E4): past it the + // only notice is NoProc, delivered under the observer's lock. + if target.is_past_death() { + let _ = self.send(Down { + monitor_ref, + reason: ExitReason::NoProc, + }); + return monitor_ref; + } monitors.insert(monitor_ref, active.clone()); } @@ -436,21 +538,21 @@ impl Context { } }); let Some(reason) = reason else { return }; - // Remove the entry from the monitor table so it doesn't accumulate - // stale entries over the actor's lifetime. Done before delivery - // since `demonitor` is now a no-op for this monitor anyway. - let deliver = monitors - .lock() - .unwrap_or_else(|p| p.into_inner()) + // Receiver-side fence (E6): the entry is removed and the Down + // enqueued in one critical section, so once `demonitor` returns + // no Down of this monitor is enqueued. + let mut entries = monitors.lock().unwrap_or_else(|p| p.into_inner()); + if entries .as_mut() .and_then(|entries| entries.remove(&monitor_ref)) - .is_some(); - if deliver { + .is_some() + { let _ = actor_ref.send(Down { monitor_ref, reason, }); } + drop(entries); }); monitor_ref @@ -458,10 +560,8 @@ impl Context { /// Cancel a previously-set monitor. /// - /// The monitor waiter is released promptly. - /// If the target hasn't yet died, no `Down` message will be delivered. - /// If a `Down` message has already been delivered (or queued), this is - /// a best-effort cancellation — the message may still arrive. + /// A fence (OTP `demonitor/1`): once it returns, no `Down` of this + /// monitor is enqueued; one already queued stays. pub fn demonitor(&self, monitor_ref: MonitorRef) { let active = self .monitors @@ -475,79 +575,66 @@ impl Context { } } - /// Set up a bidirectional link with another actor. + /// Set up a bidirectional link with another actor (OTP `link/1`). /// - /// When either side dies abnormally, the other receives an exit signal. - /// By default the receiver is terminated by the signal. Call - /// [`trap_exit(true)`] to convert signals into `Exit` messages delivered - /// via [`Actor::exit_received`] instead. + /// When either side's exit is published, the other receives a link + /// notice with the final reason: a non-trapping peer ignores `Normal` and + /// otherwise exits with that reason without running `stopped()`; a + /// trapping peer gets an `Exit` via [`Actor::exit_received`]. /// - /// If the target is already dead with an abnormal reason, the exit - /// signal is delivered immediately without retaining a link. Repeating an - /// existing live link is a no-op; each attempt to link a closed peer reports - /// that peer's exit for the new attempt. + /// If the target is past its death point, this actor receives a `NoProc` + /// notice: an `Exit { reason: NoProc }` when trapping, otherwise it exits + /// `NoProc`. Use [`try_link`](Self::try_link) to get `Err(NoProc)` + /// instead. An existing link, or a pending notice from that peer, makes + /// this a no-op. /// - /// [`trap_exit(true)`]: Self::trap_exit /// [`Actor::exit_received`]: Actor::exit_received pub fn link(&self, target: &ChildHandle) { - let own_signal = link::make_signal( - self.trap_exit.clone(), - self.own_cancel_fn(), - self.own_send_exit_fn(), - self.linked_reason.clone(), - ); - let peer_signal = link::make_signal( - target.trap_exit_flag().clone(), - target.cancel_fn().clone(), - target.send_exit_fn().clone(), - target.linked_reason().clone(), - ); link::register_link( - self.id, + &self.signal, &self.links, - own_signal, - target.id(), + target.signal_target(), target.links(), - peer_signal, + false, ); } - /// Remove a previously-set bidirectional link. + /// Link as [`link`](Self::link), but report a target past its death + /// point as `Err(NoProc)` and keep no record. + pub fn try_link(&self, target: &ChildHandle) -> Result<(), NoProc> { + match link::register_link( + &self.signal, + &self.links, + target.signal_target(), + target.links(), + true, + ) { + LinkOutcome::NoProc => Err(NoProc), + LinkOutcome::Linked | LinkOutcome::Unchanged => Ok(()), + } + } + + /// Remove a link. A fence (OTP `unlink/1`): once it returns, the link has + /// no further effect on this actor; an `Exit` already queued stays. pub fn unlink(&self, target: &ChildHandle) { link::unregister_link(self.id, &self.links, target.id(), target.links()); } - /// Control how exit signals from linked actors are handled. + /// Control how exit signals (link notices and `exit` signals) are + /// handled; read when each signal arrives (OTP `trap_exit`). /// - /// - `false` (default): an abnormal exit signal from a linked actor cancels - /// this actor. - /// - `true`: the signal is converted to an `Exit` message and delivered to + /// - `false` (default): `Normal` is ignored; any other reason ends this + /// actor after its current handler, without `stopped()`. + /// - `true`: the signal is queued as an `Exit` for /// [`Actor::exit_received`]. /// - /// Direct Kill requests cannot be trapped; propagated Kill notices can. + /// Only a kill is untrappable. A `start_linked` parent's exit ends a + /// trapping actor normally, through `stopped()`. Signals that arrive while + /// `started()` runs are resolved when it returns. pub fn trap_exit(&self, enabled: bool) { self.trap_exit.store(enabled, Ordering::Release); } - /// Build a type-erased cancel closure for this actor. - fn own_cancel_fn(&self) -> Arc { - let token = self.cancellation_token.clone(); - let reason = self.linked_reason.clone(); - Arc::new(move || link::normal_stop(&reason, || token.cancel())) - } - - /// Build a type-erased `SendExitFn` that enqueues an `ExitEnvelope` - /// onto this actor's mailbox. - fn own_send_exit_fn(&self) -> SendExitFn { - let sender = self.sender.clone(); - Arc::new(move |exit: Exit| { - let envelope: Box + Send> = Box::new(ExitEnvelope { exit }); - sender - .send(MailboxItem::Message(envelope)) - .map_err(|_| ActorError::ActorStopped) - }) - } - pub(crate) fn cancellation_token(&self) -> CancellationToken { self.cancellation_token.clone() } @@ -631,6 +718,7 @@ pub struct ActorRef { trap_exit: TrapExitFlag, links: LinkTable, linked_reason: LinkedExitReason, + signal: Arc, mailbox: crate::instrumentation::Mailbox, replies: Arc, jobs: Arc, @@ -654,6 +742,7 @@ impl Clone for ActorRef { trap_exit: self.trap_exit.clone(), links: self.links.clone(), linked_reason: self.linked_reason.clone(), + signal: self.signal.clone(), mailbox: self.mailbox.clone(), replies: self.replies.clone(), jobs: self.jobs.clone(), @@ -757,9 +846,9 @@ impl ActorRef { self.cleaned.wait(); } - /// Poll the exit reason. Returns `None` if the actor is still running. - /// A completion channel closed without a reason reports [`ExitReason::Kill`], - /// matching the wait methods (for example, after runtime shutdown). + /// Poll the exit reason. Returns `None` until the exit is published. + /// A completion channel closed without a reason reports + /// [`ExitReason::Killed`], matching the wait methods. pub fn exit_reason(&self) -> Option { crate::child_handle::poll_exit_reason(&self.completion) } @@ -791,21 +880,10 @@ impl ActorRef { impl From> for ChildHandle { fn from(actor_ref: ActorRef) -> Self { let failures = actor_ref.sender.failures(); - let sender = actor_ref.sender.clone(); - let send_exit: SendExitFn = Arc::new(move |exit: Exit| { - let envelope: Box + Send> = Box::new(ExitEnvelope { exit }); - sender - .send(MailboxItem::Message(envelope)) - .map_err(|_| ActorError::ActorStopped) - }); - ChildHandle::from_threads( - actor_ref.id, - actor_ref.cancellation_token, - actor_ref.completion, - actor_ref.trap_exit, + ChildHandle::new( + actor_ref.signal, + Completion::Threads(actor_ref.completion), actor_ref.links, - actor_ref.linked_reason, - send_exit, ) .with_mailbox_failures(failures) } @@ -845,6 +923,16 @@ impl ActorRef { let trap_exit = new_trap_exit_flag(); let links = new_link_table(); let linked_reason = new_linked_exit_reason(); + // Phase 0: signals that arrive before started() returns are held. + link::lock_request(&linked_reason).begin_starting(); + let signal = signal_target::( + id, + parent.as_ref().map(ChildHandle::id), + &tx, + trap_exit.clone(), + linked_reason.clone(), + cancellation_token.clone(), + ); let actor_ref = ActorRef { id, @@ -855,6 +943,7 @@ impl ActorRef { trap_exit: trap_exit.clone(), links: links.clone(), linked_reason: linked_reason.clone(), + signal: signal.clone(), mailbox: crate::instrumentation::Mailbox::default(), replies: Registry::new(id, A::MAX_OUTSTANDING), jobs: Arc::new(crate::reply::jobs::JobSet::new( @@ -873,6 +962,7 @@ impl ActorRef { trap_exit, links: links.clone(), linked_reason: linked_reason.clone(), + signal, mailbox: actor_ref.mailbox.clone(), replies: actor_ref.replies.clone(), jobs: actor_ref.jobs.clone(), @@ -890,7 +980,6 @@ impl ActorRef { linked_reason, actor_ref.monitors.clone(), completion_tx, - ExitReason::Panic("unexpected framework panic".into()), move || cleanup_token.cancel(), ) .with_mailbox_failures(actor_ref.sender.failures()); @@ -902,33 +991,30 @@ impl ActorRef { let _thread_handle = rt::spawn(move || { let _cleaned = cleaned; let ((mut actor, ctx, mut rx, cancellation_token), mut cleanup) = startup.into_parts(); - let primary = match catch_unwind(AssertUnwindSafe(|| { + let request = ctx.linked_reason.clone(); + if let Err(payload) = catch_unwind(AssertUnwindSafe(|| { run_actor(&mut actor, ctx, &mut rx, cancellation_token) })) { - Ok(reason) => Ok(reason), - Err(payload) => { - let message = panic_message(&*payload); - crate::mailbox::dispose_panic(payload); - Err(ExitReason::Panic(format!( - "actor lifecycle unwound: {message}" - ))) - } - }; - // Death (deferred-replies §5 step 3), only now that `stopped()` - // returned: close the registry (every open token answers - // ActorStopped), dispose the mailbox (queued requests answer - // ActorStopped; later sends from state destructors fail), then drop - // the state, all before the exit is published. + let message = panic_message(&*payload); + crate::mailbox::dispose_panic(payload); + link::lock_request(&request).panic(ExitReason::Panic(format!( + "actor lifecycle unwound: {message}" + ))); + } + // The death point D, only now that `stopped()` returned (or was + // skipped): the reason becomes final, the link table closes, the + // registry closes (every open token is answered) and the mailbox + // refuses admission (queued requests are answered). replies.exiting(); jobs.cancel_all(); + cleanup.die(); replies.close(); drop(rx); - let outcome = crate::actor_cleanup::finish_actor_state(actor, primary); - match outcome { - Ok(reason) => cleanup.complete(reason), - Err(reason) => cleanup.complete_secondary(reason), + // Destruction: a destructor panic is a cleanup failure only. + if let Some(failure) = crate::actor_cleanup::finish_actor_state(actor) { + cleanup.cleanup_failed(failure); } - // Publish the exit (monitors and links fire) before the job join. + // Publication (wait_exit, monitors, link notices) before the job join. drop(cleanup); // Job cleanup (§5 step 4), after death: join the cancelled jobs // that signal completion, bounded by the cleanup deadline. @@ -948,7 +1034,7 @@ fn run_actor( ctx: Context, rx: &mut mpsc::Receiver>, cancellation_token: CancellationToken, -) -> ExitReason { +) { run_actor_receiving( actor, ctx, @@ -965,8 +1051,9 @@ fn run_actor_receiving( ctx: Context, mut receive: impl FnMut() -> Result, std::sync::mpsc::RecvTimeoutError>, cancellation_token: CancellationToken, -) -> ExitReason { +) { let mut discarded = None; + let request = ctx.linked_reason.clone(); let _monitor_cleanup = MonitorCleanup(ctx.monitors.clone()); let start_result = catch_unwind(AssertUnwindSafe(|| { fail_point!("actor.before_started"); @@ -977,38 +1064,22 @@ fn run_actor_receiving( crate::actor_cleanup::log_safely(|| { tracing::error!("Panic in started() callback: {msg}"); }); + // No stopped() after a failed start (L5). + link::lock_request(&request).panic(ExitReason::Panic(format!("panic in started(): {msg}"))); cancellation_token.cancel(); - return ExitReason::Panic(format!("panic in started(): {msg}")); + return; } - - if cancellation_token.is_cancelled() { - ctx.replies.exiting(); - ctx.jobs.cancel_all(); - if let Err(panic) = catch_unwind(AssertUnwindSafe(|| { - fail_point!("actor.before_stopped"); - actor.stopped(&ctx) - })) { - let msg = panic_message(&*panic); - crate::actor_cleanup::log_safely(|| { - tracing::error!("Panic in stopped() callback: {msg}"); - }); - return ExitReason::Panic(format!("panic in stopped(): {msg}")); - } - return ExitReason::Normal; + // Phase 0 ends: resolve signals held during started() with the trap + // flag read now. + if link::lock_request(&request).end_starting(ctx.trap_exit.load(Ordering::Acquire)) { + cancellation_token.cancel(); } - let mut exit_reason = ExitReason::Normal; - - loop { + while !cancellation_token.is_cancelled() { let msg = match receive() { Ok(MailboxItem::Message(msg)) => Some(msg), Ok(MailboxItem::Shutdown) => None, - Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { - if cancellation_token.is_cancelled() { - break; - } - continue; - } + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => continue, Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => None, }; match msg { @@ -1040,10 +1111,10 @@ fn run_actor_receiving( crate::actor_cleanup::log_safely(|| { tracing::error!("Panic in message handler: {msg}"); }); - exit_reason = ExitReason::Panic(format!("panic in handler: {msg}")); - break; - } - if cancellation_token.is_cancelled() { + // A panic begins exit, or replaces the requested reason + // unless a kill was received. + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in handler: {msg}"))); break; } } @@ -1054,17 +1125,20 @@ fn run_actor_receiving( cancellation_token.cancel(); ctx.replies.exiting(); ctx.jobs.cancel_all(); - let stop_result = catch_unwind(AssertUnwindSafe(|| { - fail_point!("actor.before_stopped"); - actor.stopped(&ctx); - })); - if let Err(panic) = stop_result { - let msg = panic_message(&*panic); - crate::actor_cleanup::log_safely(|| { - tracing::error!("Panic in stopped() callback: {msg}"); - }); - if !exit_reason.is_abnormal() { - exit_reason = ExitReason::Panic(format!("panic in stopped(): {msg}")); + // stopped() is skipped when a kill, an untrapped exit signal or link + // notice began the exit (decision 2); one that already began finishes. + if link::lock_request(&request).begin_hook() { + let stop_result = catch_unwind(AssertUnwindSafe(|| { + fail_point!("actor.before_stopped"); + actor.stopped(&ctx); + })); + if let Err(panic) = stop_result { + let msg = panic_message(&*panic); + crate::actor_cleanup::log_safely(|| { + tracing::error!("Panic in stopped() callback: {msg}"); + }); + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in stopped(): {msg}"))); } } @@ -1072,7 +1146,6 @@ fn run_actor_receiving( // Preserve stopped() -> monitor cleanup -> undelivered-message drop. drop(_monitor_cleanup); ctx.sender.failures().discard(discarded); - exit_reason } // --------------------------------------------------------------------------- @@ -1086,9 +1159,10 @@ pub trait ActorStart: Actor { ActorRef::spawn(self, None) } - /// Register the bidirectional link before the child can run `started()`. - /// A closed parent is observed under its link-state mutex; an abnormal - /// parent exit cancels the child cooperatively before message dispatch. + /// Start linked to `parent_ctx`'s actor, which becomes this instance's + /// parent (E3). The link is registered before `started()`. The parent's + /// exit ends a trapping child normally (its `stopped()` runs; it never + /// reaches `exit_received`), a non-trapping child as any link notice. fn start_linked(self, parent_ctx: &Context

) -> ActorRef { ActorRef::spawn(self, Some(parent_ctx.actor_ref().child_handle())) } From 8ca2940c354ef9e91fcd6d63080c4040077c17a1 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:22:33 -0300 Subject: [PATCH 062/208] wip --- concurrency/src/lifecycle_trace.rs | 3 +++ concurrency/src/metrics.rs | 6 +++++- concurrency/tests/metrics.rs | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/concurrency/src/lifecycle_trace.rs b/concurrency/src/lifecycle_trace.rs index c8166f0b..3f4dd745 100644 --- a/concurrency/src/lifecycle_trace.rs +++ b/concurrency/src/lifecycle_trace.rs @@ -181,6 +181,9 @@ pub(crate) fn reason(reason: Option<&crate::ExitReason>) -> &'static str { Some(ExitReason::Shutdown) => "Shutdown", Some(ExitReason::Panic(_)) => "Panic", Some(ExitReason::Kill) => "Kill", + // Supervision.tla has one class per restart behaviour: Killed and + // NoProc are abnormal, as Kill. + Some(_) => "Kill", } } diff --git a/concurrency/src/metrics.rs b/concurrency/src/metrics.rs index f6c73ebe..b2b5d40f 100644 --- a/concurrency/src/metrics.rs +++ b/concurrency/src/metrics.rs @@ -177,6 +177,8 @@ counters!( actors_exited_shutdown_total, actors_exited_panic_total, actors_exited_kill_total, + actors_exited_killed_total, + actors_exited_noproc_total, messages_enqueued_total, messages_rejected_total, mailbox_depth, @@ -466,6 +468,8 @@ impl ActorObservation { ExitReason::Shutdown => &COUNTERS.actors_exited_shutdown_total, ExitReason::Panic(_) => &COUNTERS.actors_exited_panic_total, ExitReason::Kill => &COUNTERS.actors_exited_kill_total, + ExitReason::Killed => &COUNTERS.actors_exited_killed_total, + ExitReason::NoProc => &COUNTERS.actors_exited_noproc_total, } .fetch_add(1, Relaxed); } @@ -473,7 +477,7 @@ impl ActorObservation { impl Drop for ActorObservation { fn drop(&mut self) { if !self.finished { - self.finish(&ExitReason::Kill); + self.finish(&ExitReason::Killed); } } } diff --git a/concurrency/tests/metrics.rs b/concurrency/tests/metrics.rs index dd69c161..71367020 100644 --- a/concurrency/tests/metrics.rs +++ b/concurrency/tests/metrics.rs @@ -298,6 +298,8 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { + s.actors_exited_shutdown_total + s.actors_exited_panic_total + s.actors_exited_kill_total + + s.actors_exited_killed_total + + s.actors_exited_noproc_total ); assert_eq!( s.messages_enqueued_total, From b18231b4a77bd3e3cd84e6a544cd852af081bcd0 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:23:07 -0300 Subject: [PATCH 063/208] feat(test): faults::exit sends an exact exit signal --- testing/src/faults.rs | 124 +++++++----------------------------------- 1 file changed, 21 insertions(+), 103 deletions(-) diff --git a/testing/src/faults.rs b/testing/src/faults.rs index 762389d8..d94ff025 100644 --- a/testing/src/faults.rs +++ b/testing/src/faults.rs @@ -1,116 +1,34 @@ //! Fault injection through public lifecycle APIs. //! -//! [`exit`] makes an actor exit with a chosen [`ExitReason`]. It resembles -//! Erlang's `exit(Pid, Reason)` for `Panic`, delivered through a dying linked -//! peer. `Normal`, `Shutdown` and `Kill` are direct requests, -//! which always stop the target (see [`exit`]). Stopping is cooperative: a -//! running handler finishes first, then `stopped()` runs. +//! [`exit`] sends an exit signal with a chosen [`ExitReason`], exactly as +//! Erlang's `exit_signal(Pid, Reason)` (`ChildHandle::exit`). Stopping is +//! cooperative: a running handler finishes first. //! //! Message-level faults (delay, discard, crash on a message, a reply slower //! than the caller's timeout) are configured on [`Mock`](crate::tasks::Mock) //! and [`Probe`](crate::tasks::Probe). -//! -//! Panic injection uses a short-lived linked actor. Kill uses the core's -//! direct `ChildHandle::stop_with_reason`, including in Tokio-free builds. -use spawned_concurrency::{message::Message, threads, ChildHandle, ExitReason}; +use spawned_concurrency::{ChildHandle, ExitReason}; -/// Make `target` exit with `reason`. +/// Send `target` an exit signal with `reason` from outside any actor +/// ([`ChildHandle::exit`], sender `ActorId::EXTERNAL`): /// -/// - `Normal`: [`ChildHandle::stop`]. The target stops even if it traps -/// exits; this differs from Erlang's `exit(Pid, normal)`, which a -/// non-trapping process ignores and a trapping one receives as a message. -/// - `Shutdown`: [`ChildHandle::shutdown`]. Also a direct stop that trapping -/// cannot intercept. -/// - `Panic(msg)`: a short-lived linked actor panics with `msg`. A -/// non-trapping target exits with `Panic("panic in handler: {msg}")` (the -/// wording the core uses for a handler panic), not `Panic(msg)`; a target -/// that traps exits receives an [`Exit`](spawned_concurrency::Exit) carrying -/// that reason instead of stopping. The panic is real, so the default panic -/// hook prints `msg` to stderr. If the target already exited abnormally, -/// its link stops the fuse first; that is logged and `exit` returns (the -/// target is gone, with its own reason). Any other failure to deliver the -/// panic panics here rather than dropping the fault silently. -/// - `Kill`: direct `ChildHandle::stop_with_reason(Kill)`. The target stops -/// even when trapping exits; linked trapping peers receive an Exit notice. -/// Available in both runtime modes without constructing a private runtime. +/// - `Kill`: untrappable; the target ends `Killed` ([`ChildHandle::kill`]). +/// - `Normal`: ignored by a non-trapping target; a trapping one receives an +/// [`Exit`](spawned_concurrency::Exit). +/// - any other reason: a non-trapping target exits with exactly that reason +/// after its current handler, without `stopped()`; a trapping one receives +/// an `Exit` carrying it. /// -/// A target that has already exited is left alone (logged at debug level). A -/// target that exits `Normal` or `Shutdown` concurrently with a `Panic` -/// injection does not propagate that exit over the link, so the fuse still -/// panics on its own and the default hook prints it; the target is unaffected. +/// Use [`ChildHandle::stop`] or [`ChildHandle::stop_with_reason`] for an +/// administrative stop that `trap_exit` cannot intercept and that runs +/// `stopped()`. /// -/// Never waits for the target and never blocks: it returns once the exit is -/// requested, so it is safe to call from async code, including a -/// `current_thread` runtime. Wait for the exit with `ChildHandle::wait_exit_async`, -/// [`tasks::exit_within`](crate::tasks::exit_within) or -/// [`threads::exit_within`](crate::threads::exit_within). +/// A target past its death point is left alone. Never waits and never +/// blocks, so it is safe to call from async code, including a +/// `current_thread` runtime. Wait for the exit with +/// `ChildHandle::wait_exit_async`, [`tasks::exit_within`](crate::tasks::exit_within) +/// or [`threads::exit_within`](crate::threads::exit_within). pub fn exit(target: &ChildHandle, reason: ExitReason) { tracing::debug!(target = ?target.id(), %reason, "spawned-test injecting exit"); - match reason { - ExitReason::Normal => target.stop(), - ExitReason::Shutdown => target.shutdown(), - ExitReason::Panic(msg) => panic_linked(target, msg), - ExitReason::Kill => target.stop_with_reason(ExitReason::Kill), - } -} - -struct Fuse; -impl threads::Actor for Fuse {} -struct Detonate(String); -impl Message for Detonate { - type Result = (); -} -impl threads::Handler for Fuse { - fn handle( - &mut self, - msg: Detonate, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, - _: &threads::Context, - ) { - panic!("{}", msg.0); - } -} - -fn panic_linked(target: &ChildHandle, msg: String) { - use threads::ActorStart as _; - if already_exited(target) { - return; - } - let fuse = Fuse.start(); - // Registers both link edges synchronously, or, if the target has already - // exited abnormally, signals that exit to the fuse right away. - fuse.context().link(target); - if let Err(error) = fuse.send(Detonate(msg)) { - // Nothing else knows the fuse, so only the target's link can have - // stopped it: the target exited abnormally before the fault landed. - // Read the outcome without waiting (the fuse's mailbox can close - // before its exit reason is published), so async callers never block. - match fuse.child_handle().exit_reason() { - Some(ExitReason::Normal) => panic!( - "spawned-test: could not inject a panic into {:?}: the fuse stopped \ - ({error}) without a linked exit", - target.id() - ), - reason => tracing::debug!( - target = ?target.id(), - fuse_reason = ?reason, - "spawned-test: target exited before the injected panic; not needed" - ), - } - } -} - -/// `true` (and logged) if `target` already exited, so there is nothing to fault. -fn already_exited(target: &ChildHandle) -> bool { - match target.exit_reason() { - Some(reason) => { - tracing::debug!( - target = ?target.id(), - %reason, - "spawned-test: target already exited; fault not injected" - ); - true - } - None => false, - } + target.exit(reason); } From 324f6294a7dc6273e5002778799adc863cc0e6cc Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:24:26 -0300 Subject: [PATCH 064/208] test: adapt unit tests to the death point and Killed; invert the late-link regression (decision 8) --- concurrency/src/child_handle.rs | 2 +- concurrency/src/link_regression_tests.rs | 55 +++++++------- concurrency/src/tasks/actor.rs | 91 ++++++++++++++---------- concurrency/src/threads/actor.rs | 37 ++++++++-- 4 files changed, 114 insertions(+), 71 deletions(-) diff --git a/concurrency/src/child_handle.rs b/concurrency/src/child_handle.rs index c24632f9..fcf8ce8b 100644 --- a/concurrency/src/child_handle.rs +++ b/concurrency/src/child_handle.rs @@ -365,7 +365,7 @@ impl std::hash::Hash for ChildHandle { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; use crate::link::{new_link_table, new_linked_exit_reason, new_trap_exit_flag}; diff --git a/concurrency/src/link_regression_tests.rs b/concurrency/src/link_regression_tests.rs index f6bbeebc..4708b999 100644 --- a/concurrency/src/link_regression_tests.rs +++ b/concurrency/src/link_regression_tests.rs @@ -24,7 +24,7 @@ fn runtime() -> tokio::runtime::Runtime { } #[test] -fn linking_in_post_drain_pre_completion_window_delivers_exit() { +fn linking_between_death_point_and_publication_delivers_noproc() { let rt = runtime(); let events = Arc::new(Mutex::new(Vec::new())); let observer = { @@ -35,17 +35,18 @@ fn linking_in_post_drain_pre_completion_window_delivers_exit() { let peer_id = ActorId::next(); let peer_links = link::new_link_table(); let (_completion_tx, completion_rx) = spawned_rt::tasks::watch::channel(None); - // This is the actual state between target propagation and publication. - let reason = ExitReason::Panic("closing before completion".into()); - link::propagate_exit(peer_id, &peer_links, &reason); - let peer = ChildHandle::from_tasks( - peer_id, - spawned_rt::tasks::CancellationToken::new(), - completion_rx, - link::new_trap_exit_flag(), + // This is the actual state between the death point D and publication: + // a registration after D reports NoProc, not the real reason (E4). + let real = ExitReason::Panic("closing before completion".into()); + link::propagate_exit(peer_id, &peer_links, &real); + let reason = ExitReason::NoProc; + let peer = ChildHandle::new( + crate::child_handle::tests::test_target( + peer_id, + spawned_rt::threads::CancellationToken::new(), + ), + crate::child_handle::Completion::Tasks(completion_rx), peer_links, - link::new_linked_exit_reason(), - Arc::new(|_| Ok(())), ); observer.context().link(&peer); let delivered = rt.block_on(async { @@ -109,7 +110,7 @@ fn never_polled_actor_propagates_runtime_shutdown_to_peer() { survivor.block_on(peer.wait_exit()); assert_eq!( result, - Ok(ExitReason::Kill), + Ok(ExitReason::Killed), "aborted actor failed to signal its live peer" ); } @@ -158,7 +159,7 @@ fn polled_actor_propagates_runtime_shutdown_to_peer() { .block_on(async { tokio::time::timeout(Duration::from_secs(1), peer.wait_exit()).await }); peer.child_handle().stop(); survivor.block_on(peer.wait_exit()); - assert_eq!(result, Ok(ExitReason::Kill)); + assert_eq!(result, Ok(ExitReason::Killed)); } struct MonitorObserver(Option>); @@ -206,7 +207,7 @@ fn never_polled_actor_notifies_monitor_on_runtime_shutdown() { survivor.block_on(async { tokio::time::timeout(Duration::from_secs(1), rx).await }); observer.child_handle().stop(); survivor.block_on(observer.wait_exit()); - assert_eq!(result.unwrap().unwrap().reason, ExitReason::Kill); + assert_eq!(result.unwrap().unwrap().reason, ExitReason::Killed); } struct Stop; @@ -250,8 +251,12 @@ impl crate::tasks::Handler for Stopping { reply.reply(()); } } +/// Inverted in v0.8 (decision 8, as OTP: a process in `terminate` dies with +/// a later exit signal): the abnormal link notice that arrives while +/// `stopped()` runs replaces the requested `Normal`; the running `stopped()` +/// still finishes, and the linked survivor receives the new reason. #[test] -fn late_link_failure_cannot_relabel_an_already_requested_normal_stop() { +fn late_link_failure_relabels_an_already_requested_normal_stop() { let rt = runtime(); let (entered_tx, entered_rx) = spawned_rt::tasks::oneshot::channel(); let (release_tx, release_rx) = spawned_rt::tasks::oneshot::channel(); @@ -277,10 +282,12 @@ fn late_link_failure_cannot_relabel_an_already_requested_normal_stop() { ExitReason::Panic(_) )); release_tx.send(()).unwrap(); - assert_eq!(rt.block_on(stopping.wait_exit()), ExitReason::Normal); - assert!(survivor.child_handle().is_alive()); - survivor.child_handle().stop(); - rt.block_on(survivor.wait_exit()); + let relabelled = rt.block_on(stopping.wait_exit()); + assert!( + matches!(&relabelled, ExitReason::Panic(m) if m.contains("linked failure")), + "got {relabelled:?}" + ); + assert_eq!(rt.block_on(survivor.wait_exit()), relabelled); } struct LinkedAtStart { @@ -379,7 +386,7 @@ fn never_polled_monitor_owner_releases_waiter_and_refuses_late_registration() { ); target.child_handle().stop(); survivor.block_on(target.wait_exit()); - assert_eq!(owner.child_handle().exit_reason(), Some(ExitReason::Kill)); + assert_eq!(owner.child_handle().exit_reason(), Some(ExitReason::Killed)); } #[test] @@ -422,7 +429,7 @@ fn linked_runtime_abort_is_trappable() { *events.lock().unwrap(), vec![Exit { from: target.id(), - reason: ExitReason::Kill + reason: ExitReason::Killed }] ); peer.child_handle().stop(); @@ -473,7 +480,7 @@ fn never_polled_completion_waits_for_actor_payload_destruction() { before, None, "completion published while the actor's destructor was still running" ); - assert_eq!(actor.child_handle().exit_reason(), Some(ExitReason::Kill)); + assert_eq!(actor.child_handle().exit_reason(), Some(ExitReason::Killed)); } #[test] @@ -522,7 +529,7 @@ fn linked_runtime_abort_cancels_an_idle_threads_actor() { let _ = peer.send(Ping); // Wake cleanup even if the cancellation callback regresses. } peer.child_handle().wait_exit_blocking(); - assert_eq!(result, Ok(ExitReason::Kill)); + assert_eq!(result, Ok(ExitReason::Killed)); } #[test] @@ -575,5 +582,5 @@ fn dropping_runtime_with_large_polled_link_cycle_reclaims_every_actor() { assert_eq!(dropped.load(Ordering::SeqCst), actors.len()); assert!(actors .iter() - .all(|a| a.child_handle().exit_reason() == Some(ExitReason::Kill))); + .all(|a| a.child_handle().exit_reason() == Some(ExitReason::Killed))); } diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 3b90020c..ea7188b2 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -1654,6 +1654,49 @@ async fn run_actor_lifecycle( } } +/// A context and its cleanup without spawning, for run-loop unit tests. +#[cfg(test)] +fn test_context( + tx: &mpsc::Sender + Send>>, + token: &CancellationToken, +) -> (Context, crate::actor_cleanup::ActorCleanup) { + let id = ActorId::next(); + let (completion_tx, completion_rx) = watch::channel(None); + let trap = new_trap_exit_flag(); + let links = new_link_table(); + let request = new_linked_exit_reason(); + let monitors: MonitorTable = Arc::new(Mutex::new(Some(HashMap::new()))); + let signal = signal_target::(id, None, tx, trap.clone(), request.clone(), token.clone()); + let ctx = Context { + shared: Arc::new(ActorShared { + id, + sender: tx.clone(), + cancellation_token: token.clone(), + completion_rx, + monitors: monitors.clone(), + trap_exit: trap, + links: links.clone(), + linked_reason: request.clone(), + signal, + mailbox: crate::instrumentation::Mailbox::default(), + managed: None, + replies: Registry::new(id, DEFAULT_MAX_OUTSTANDING), + jobs: Arc::new(crate::reply::jobs::JobSet::new(id, 1)), + cleaned: watch::channel(false).1, + }), + }; + let noop: fn() = || {}; + let cleanup = crate::actor_cleanup::ActorCleanup::new( + id, + links, + request, + monitors, + completion_tx, + noop, + ); + (ctx, cleanup) +} + // --------------------------------------------------------------------------- // Actor::start // --------------------------------------------------------------------------- @@ -3484,29 +3527,12 @@ mod turn_regressions { rt::Runtime::new().unwrap().block_on(async { let (tx, rx) = mpsc::channel(); let token = CancellationToken::new(); - let (_, completion_rx) = watch::channel(None); - let ctx = Context { - shared: Arc::new(ActorShared { - id: ActorId::next(), - sender: tx.clone(), - cancellation_token: token.clone(), - completion_rx, - monitors: Arc::new(Mutex::new(Some(HashMap::new()))), - trap_exit: new_trap_exit_flag(), - links: new_link_table(), - linked_reason: new_linked_exit_reason(), - mailbox: crate::instrumentation::Mailbox::default(), - managed: None, - replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), - jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), - cleaned: watch::channel(false).1, - }), - }; + let (ctx, mut cleanup) = test_context(&tx, &token); let count = Arc::new(AtomicUsize::new(0)); for _ in 0..2 { tx.send(Box::new(MessageEnvelope { msg: Inc, - tx: None, + slot: SlotHandle::detached(), #[cfg(feature = "metrics")] observation: None, }) as Box + Send>) @@ -3524,12 +3550,14 @@ mod turn_regressions { ctx, rx, token.clone(), + &mut cleanup, Some(turn), )); assert!(futures::poll!(job.as_mut()).is_pending()); assert_eq!(count.load(Ordering::SeqCst), 1); token.cancel(); - assert_eq!(job.await, Ok(ExitReason::Normal)); + job.await; + assert_eq!(cleanup.die(), ExitReason::Normal); assert_eq!(count.load(Ordering::SeqCst), 1); }); } @@ -3561,30 +3589,14 @@ mod idle_stop_regression { rt::Runtime::new().unwrap().block_on(async { let (tx, rx) = mpsc::channel(); let token = CancellationToken::new(); - let (_, completion_rx) = watch::channel(None); - let ctx = Context { - shared: Arc::new(ActorShared { - id: ActorId::next(), - sender: tx.clone(), - cancellation_token: token.clone(), - completion_rx, - monitors: Arc::new(Mutex::new(Some(HashMap::new()))), - trap_exit: new_trap_exit_flag(), - links: new_link_table(), - linked_reason: new_linked_exit_reason(), - mailbox: crate::instrumentation::Mailbox::default(), - managed: None, - replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), - jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), - cleaned: watch::channel(false).1, - }), - }; + let (ctx, mut cleanup) = test_context(&tx, &token); let count = Arc::new(AtomicUsize::new(0)); let mut actor = Box::pin(run_actor( Counter(count.clone()), ctx, rx, token.clone(), + &mut cleanup, #[cfg(feature = "backend-experiments")] None, )); @@ -3600,7 +3612,8 @@ mod idle_stop_regression { }) as Box + Send>) .unwrap(); token.cancel(); - assert_eq!(actor.await, Ok(ExitReason::Normal)); + actor.await; + assert_eq!(cleanup.die(), ExitReason::Normal); assert_eq!( count.load(Ordering::SeqCst), 0, diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 4380b80d..250142d6 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -2027,15 +2027,28 @@ mod idle_stop_regression { let (_, completion_rx) = spawned_rt::sync::watch::channel(None); let monitors = Arc::new(Mutex::new(Some(HashMap::new()))); let events = Arc::new(Mutex::new(Vec::new())); + let (id, trap, request) = ( + ActorId::next(), + new_trap_exit_flag(), + new_linked_exit_reason(), + ); let ctx = Context { - id: ActorId::next(), + id, sender: tx.clone(), cancellation_token: token.clone(), completion: completion_rx, monitors: monitors.clone(), - trap_exit: new_trap_exit_flag(), + trap_exit: trap.clone(), links: new_link_table(), - linked_reason: new_linked_exit_reason(), + linked_reason: request.clone(), + signal: signal_target::<_>( + id, + None, + &tx, + trap.clone(), + request.clone(), + token.clone(), + ), mailbox: crate::instrumentation::Mailbox::default(), replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), @@ -2077,7 +2090,8 @@ mod idle_stop_regression { .unwrap(); token.cancel(); release.wait(); - assert_eq!(actor.join().unwrap(), ExitReason::Normal); + actor.join().unwrap(); + assert_eq!(link::lock_request(&request).reason(), None, "a plain stop request"); assert_eq!( count.load(Ordering::SeqCst), 0, @@ -2157,9 +2171,17 @@ mod stop_wakeup_regression { cancellation_token: token.clone(), completion, monitors: Arc::new(Mutex::new(Some(HashMap::new()))), - trap_exit: new_trap_exit_flag(), + trap_exit: trap.clone(), links: new_link_table(), - linked_reason: new_linked_exit_reason(), + linked_reason: request.clone(), + signal: signal_target::<_>( + id, + None, + &tx, + trap.clone(), + request.clone(), + token.clone(), + ), mailbox: crate::instrumentation::Mailbox::default(), replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), @@ -2185,7 +2207,8 @@ mod stop_wakeup_regression { stopped .recv_timeout(Duration::from_secs(2)) .expect("cancel did not wake recv"); - assert_eq!(actor.join().unwrap(), ExitReason::Normal); + actor.join().unwrap(); + assert_eq!(link::lock_request(&request).reason(), None, "a plain stop request"); } struct Crash; From ad60bbf93e6320a094f658cca728de96175df469 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:24:50 -0300 Subject: [PATCH 065/208] wip --- concurrency/src/threads/actor.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 250142d6..1fb2ddaa 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -2165,9 +2165,14 @@ mod stop_wakeup_regression { let token = CancellationToken::new(); register_stop_wakeup(&token, tx.clone()); let (_, completion) = spawned_rt::sync::watch::channel(None); + let (id, trap, request) = ( + ActorId::next(), + new_trap_exit_flag(), + new_linked_exit_reason(), + ); let ctx = Context { - id: ActorId::next(), - sender: tx, + id, + sender: tx.clone(), cancellation_token: token.clone(), completion, monitors: Arc::new(Mutex::new(Some(HashMap::new()))), From 79e1951d828d8e5245cd7cfa77af511eaa5c8364 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:25:09 -0300 Subject: [PATCH 066/208] test(shards): ReplyReceiver::try_recv returns Option (rank 11 API) --- concurrency/tests/full_actor_shards.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/concurrency/tests/full_actor_shards.rs b/concurrency/tests/full_actor_shards.rs index cf6acbd7..60e90467 100644 --- a/concurrency/tests/full_actor_shards.rs +++ b/concurrency/tests/full_actor_shards.rs @@ -309,10 +309,7 @@ fn blocking_handler_stalls_its_shard_while_another_shard_progresses() { .unwrap(); tokio::time::timeout(LIMIT, rx).await.unwrap().unwrap(); let mut queued = actors[b].request_raw(Thread).unwrap(); - assert!(matches!( - queued.try_recv(), - Err(tokio::sync::oneshot::error::TryRecvError::Empty) - )); + assert!(queued.try_recv().is_none()); if let Some(other) = (0..actors.len()).find(|&id| names[id] != names[a]) { assert_eq!( actors[other] From 3958d90d6dae6fd243dade063050946bebec21eb Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:30:10 -0300 Subject: [PATCH 067/208] test: update lifecycle tests for Killed, NoProc, cleanup failures and decision 8; Transient restart on cleanup failure --- concurrency/src/link.rs | 4 ++- concurrency/src/supervision.rs | 28 +++++++++++++++++--- concurrency/src/tasks/actor.rs | 8 +++--- concurrency/src/threads/actor.rs | 9 +++++++ concurrency/tests/actor_scenarios.rs | 9 +++++-- concurrency/tests/backend_resource_limits.rs | 8 +++--- concurrency/tests/cross_backend.rs | 4 +-- concurrency/tests/execution_groups.rs | 2 +- concurrency/tests/failure_windows.rs | 6 +++-- concurrency/tests/mailbox_drop_panic.rs | 19 +++++++------ concurrency/tests/metrics.rs | 2 +- concurrency/tests/native_mailbox_drop.rs | 19 ++++--------- concurrency/tests/panic_safety.rs | 18 ++++++++++--- concurrency/tests/startup_admission.rs | 2 +- concurrency/tests/supervision_trace.rs | 25 +++++++++++------ verification/tla/README.md | 7 +++-- verification/tla/negative_checks.py | 14 +++++----- 17 files changed, 122 insertions(+), 62 deletions(-) diff --git a/concurrency/src/link.rs b/concurrency/src/link.rs index ba17c049..35806b89 100644 --- a/concurrency/src/link.rs +++ b/concurrency/src/link.rs @@ -637,8 +637,10 @@ mod tests { assert_eq!(seen.load(Ordering::SeqCst), 1); assert!(bt.lock().unwrap().entries.is_empty()); // b links to the dead a: a late NoProc notice, delivered and removed. + // b, already exiting, gets the late NoProc signal, which replaces + // its reason (decision 8). assert_eq!(register_link(&b, &bt, &a, &at, false), LinkOutcome::NoProc); - assert_eq!(seen.load(Ordering::SeqCst), 1, "b already exiting: no second wake"); + assert_eq!(seen.load(Ordering::SeqCst), 2); assert_eq!(lock_request(&b.request).reason(), Some(&ExitReason::NoProc)); assert!(at.lock().unwrap().entries.is_empty() && bt.lock().unwrap().entries.is_empty()); } diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs index 618316e7..a43a1b8c 100644 --- a/concurrency/src/supervision.rs +++ b/concurrency/src/supervision.rs @@ -38,15 +38,29 @@ pub enum RestartMode { Temporary, } impl RestartMode { - fn restarts(self, reason: Option<&ExitReason>) -> bool { + /// The restart table. A Transient child restarts if its final reason is + /// abnormal or a cleanup failure (a destructor panic after its death + /// point) was recorded: `is_abnormal()` alone no longer predicts it. + fn restarts(self, reason: Option<&ExitReason>, cleanup_failed: bool) -> bool { match self { Self::Permanent => true, - Self::Transient => reason.map(ExitReason::is_abnormal).unwrap_or(true), + Self::Transient => cleanup_failed || reason.map(ExitReason::is_abnormal).unwrap_or(true), Self::Temporary => false, } } } +/// Trace projection of a child's exit: the raw reason class, or the abnormal +/// class when a cleanup failure was recorded, so `Supervision.tla`'s restart +/// decision (`RestartFollowsPolicy`) stays valid. +fn exit_class(reason: &ExitReason, cleanup_failed: bool) -> &'static str { + if cleanup_failed && !reason.is_abnormal() { + "Panic" + } else { + trace::reason(Some(reason)) + } +} + #[derive(Debug, Clone)] pub struct Backoff { /// First retry's delay cap, not its minimum. With jitter enabled the delay @@ -2051,8 +2065,9 @@ async fn run_child(spec: ChildSpec, view: Arc>, core: Arc { if !stop_turn { + let failed = child.cleanup_failure().is_some(); trace::emit(core.trace, || { - Event::new("SelfExit", c, trace::reason(Some(&reason))) + Event::new("SelfExit", c, exit_class(&reason, failed)) }); } reason @@ -2141,8 +2156,12 @@ async fn run_child(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc actor.cancellation_token.cancel(), _ => failing.as_ref().unwrap().send(Crash).unwrap(), } + if route == 3 { + // An untrapped link notice ends the idle actor without + // stopped() (decision 2): the wakeup is observed as its exit. + let reason = actor.child_handle().wait_exit_blocking(); + assert!(matches!(reason, ExitReason::Panic(_)), "got {reason:?}"); + assert!(stopped.try_recv().is_err(), "stopped() ran"); + failing.unwrap().join(); + continue; + } let result = stopped.recv_timeout(Duration::from_secs(2)); // A failed test must not hang cleanup on a broken wakeup callback. if result.is_err() { diff --git a/concurrency/tests/actor_scenarios.rs b/concurrency/tests/actor_scenarios.rs index b5a42f9b..482500bd 100644 --- a/concurrency/tests/actor_scenarios.rs +++ b/concurrency/tests/actor_scenarios.rs @@ -264,7 +264,8 @@ fn scenario(backend: u8, count: usize, operations: &[(u8, u8)], endings: &[u8]) let child = actor.child(); let ending = endings[i % endings.len()] % 3; // Compete monitor registration and a request with termination. A - // monitor must observe termination even if registration loses the race. + // monitor must observe termination even if registration loses the + // race: then its single Down carries NoProc (E4). let gate = Arc::new(Barrier::new(2)); let racer = actor.clone(); let gate2 = gate.clone(); @@ -323,7 +324,11 @@ fn scenario(backend: u8, count: usize, operations: &[(u8, u8)], endings: &[u8]) .position(|(id, _)| *id == event.monitor_ref) .expect("duplicate or unknown monitor"); let (_, reason) = expected_down.remove(index); - assert_eq!(event.reason, reason); + assert!( + event.reason == reason || event.reason == ExitReason::NoProc, + "Down {:?}, final reason {reason:?}", + event.reason + ); } assert!(expected_down.is_empty()); observer.child().stop(); diff --git a/concurrency/tests/backend_resource_limits.rs b/concurrency/tests/backend_resource_limits.rs index 97f926a1..4c1602a4 100644 --- a/concurrency/tests/backend_resource_limits.rs +++ b/concurrency/tests/backend_resource_limits.rs @@ -185,21 +185,21 @@ fn descriptor_limit_child() { tokio::time::timeout(LIMIT, actor.wait_exit()) .await .unwrap(), - spawned_concurrency::ExitReason::Kill + spawned_concurrency::ExitReason::Killed ); assert_eq!( actor.exit_reason(), - Some(spawned_concurrency::ExitReason::Kill) + Some(spawned_concurrency::ExitReason::Killed) ); let child = actor.child_handle(); assert!(!child.is_alive()); assert_eq!( child.exit_reason(), - Some(spawned_concurrency::ExitReason::Kill) + Some(spawned_concurrency::ExitReason::Killed) ); assert_eq!( child.wait_exit_blocking(), - spawned_concurrency::ExitReason::Kill + spawned_concurrency::ExitReason::Killed ); } } diff --git a/concurrency/tests/cross_backend.rs b/concurrency/tests/cross_backend.rs index fb335bf2..a5237545 100644 --- a/concurrency/tests/cross_backend.rs +++ b/concurrency/tests/cross_backend.rs @@ -900,7 +900,7 @@ fn monitor_workload(env: &mut Env, endings: &[Ending], double: &[bool]) -> Monit for (target, &ending) in targets.iter().zip(endings) { end(env, target, ending); } - // Watching an already-dead target delivers Down immediately. + // Watching an already-dead target delivers one Down { NoProc } at once (E4). monitors.push(vec![watch(&targets[0])]); let expected: usize = monitors.iter().map(Vec::len).sum(); let count = |events: &[Obs]| { @@ -1232,7 +1232,7 @@ proptest! { for (i, &ending) in endings.iter().enumerate() { prop_assert!(outcome.downs[i].iter().all(|r| *r == expected_reason(ending))); } - prop_assert_eq!(&outcome.downs[endings.len()], &vec![expected_reason(endings[0])]); + prop_assert_eq!(&outcome.downs[endings.len()], &vec![ExitReason::NoProc]); } #[test] diff --git a/concurrency/tests/execution_groups.rs b/concurrency/tests/execution_groups.rs index 2f1e5f61..7bd3c476 100644 --- a/concurrency/tests/execution_groups.rs +++ b/concurrency/tests/execution_groups.rs @@ -327,7 +327,7 @@ async fn ambient_actor_started_inside_group_is_unmanaged_and_aborted_at_teardown .unwrap() .starts_with("spawned-ambient-child")); assert!(system.shutdown(LIMIT).await.complete); - assert_eq!(ambient.wait_exit().await, ExitReason::Kill); + assert_eq!(ambient.wait_exit().await, ExitReason::Killed); assert_eq!(managed.wait_exit().await, ExitReason::Shutdown); } diff --git a/concurrency/tests/failure_windows.rs b/concurrency/tests/failure_windows.rs index 0b5443ea..bd5d907a 100644 --- a/concurrency/tests/failure_windows.rs +++ b/concurrency/tests/failure_windows.rs @@ -148,11 +148,13 @@ fn controlled_failures_reach_every_required_window_on_every_backend() { cleanup.track(&target); peer.link(&target.child()); peer.stop(); - control.entered(); // Normal stop is committed while cleanup is pending. + control.entered(); // Normal stop is requested while stopped() runs. assert!(target.request(&rt, Command::Panic).is_err()); assert!(matches!(target.wait(&rt), ExitReason::Panic(_))); control.release(); - assert_eq!(peer.wait(&rt), ExitReason::Normal); + // The late abnormal link notice replaces the requested Normal + // before the death point (decision 8); stopped() still finished. + assert!(matches!(peer.wait(&rt), ExitReason::Panic(_))); reached.insert(REQUIRED[5]); } { diff --git a/concurrency/tests/mailbox_drop_panic.rs b/concurrency/tests/mailbox_drop_panic.rs index 961804f1..eab880be 100644 --- a/concurrency/tests/mailbox_drop_panic.rs +++ b/concurrency/tests/mailbox_drop_panic.rs @@ -164,11 +164,12 @@ fn mailbox_drop_child() { ExitReason::Panic("panic in handler: original handler failure".into()) ); } else { + // A destructor panic after the death point is a + // cleanup failure; the final reason stays Normal. + assert_eq!(reason, ExitReason::Normal); assert_eq!( - reason, - ExitReason::Panic( - "queued message destructor: queued message destructor".into() - ) + victim.child_handle().cleanup_failure().as_deref(), + Some("queued message destructor") ); } let first = tokio::time::timeout(LIMIT, events.recv()) @@ -259,10 +260,12 @@ fn abandoned_receiver_child() { let replies: Vec<_> = (0..2).map(|_| actor.request_raw(Bomb(dropped.clone())).unwrap()).collect(); shutdown.send(()).unwrap(); let reason = tokio::time::timeout(LIMIT, actor.wait_exit()).await.unwrap(); - if state_panics && !polled { assert!(matches!(reason, ExitReason::Panic(ref message) if message.contains("actor state destructor"))); } - else { assert_eq!(reason, ExitReason::Kill); } + // A runtime drop ends Killed (G1); destructor panics are + // cleanup failures, never the reason. + let _ = state_panics; + assert_eq!(reason, ExitReason::Killed); assert_eq!(dropped.load(Ordering::SeqCst), 2); - assert_eq!(actor.child_handle().mailbox_cleanup_failure().as_deref(), Some("queued message destructor")); + assert_eq!(actor.child_handle().cleanup_failure().as_deref(), Some("queued message destructor")); for reply in replies { assert!(tokio::time::timeout(LIMIT, reply).await.unwrap().is_err()); } drop(hold); owner.join().unwrap(); @@ -304,7 +307,7 @@ fn abandoned_receiver_child() { release.send(()).unwrap(); assert_eq!(tokio::time::timeout(LIMIT, actor.wait_exit()).await.unwrap(), committed); assert_eq!(dropped.load(Ordering::SeqCst), 2); - assert_eq!(actor.child_handle().mailbox_cleanup_failure().as_deref(), Some("queued message destructor")); + assert_eq!(actor.child_handle().cleanup_failure().as_deref(), Some("queued message destructor")); assert!(system.shutdown(LIMIT).await.complete); } } diff --git a/concurrency/tests/metrics.rs b/concurrency/tests/metrics.rs index 71367020..e38f5fca 100644 --- a/concurrency/tests/metrics.rs +++ b/concurrency/tests/metrics.rs @@ -217,7 +217,7 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { owner.block_on(entered).unwrap(); } drop(owner); - assert_eq!(actor.exit_reason(), Some(ExitReason::Kill)); + assert_eq!(actor.exit_reason(), Some(ExitReason::Killed)); assert_eq!(actor.mailbox_depth(), 0); } let rt = tokio::runtime::Builder::new_multi_thread() diff --git a/concurrency/tests/native_mailbox_drop.rs b/concurrency/tests/native_mailbox_drop.rs index 26fba587..0e0c62b4 100644 --- a/concurrency/tests/native_mailbox_drop.rs +++ b/concurrency/tests/native_mailbox_drop.rs @@ -106,21 +106,12 @@ fn native_drop_child() { let (done, exit) = mpsc::channel(); std::thread::spawn(move || done.send(waiter.wait_exit_blocking()).unwrap()); let exit = exit.recv_timeout(LIMIT).unwrap(); - if reason != ExitReason::Normal { - assert_eq!(exit, reason); - } else if state_panics { - assert_eq!( - exit, - ExitReason::Panic("actor state destructor: native state destructor".into()) - ); - } else { - assert_eq!( - exit, - ExitReason::Panic("queued message destructor: native queued destructor".into()) - ); - } + // Destructor panics after the death point are cleanup failures: + // the final reason is the requested one. + let _ = state_panics; + assert_eq!(exit, reason); assert_eq!( - child.mailbox_cleanup_failure().as_deref(), + child.cleanup_failure().as_deref(), Some("native queued destructor") ); assert_eq!(dropped.load(Ordering::SeqCst), 2); diff --git a/concurrency/tests/panic_safety.rs b/concurrency/tests/panic_safety.rs index c945bfe4..b49cd433 100644 --- a/concurrency/tests/panic_safety.rs +++ b/concurrency/tests/panic_safety.rs @@ -61,7 +61,9 @@ fn expected(fault: Fault) -> ExitReason { Fault::DownHandler => panic("panic in handler: injected Down panic"), Fault::MessageDrop => panic("panic in handler: injected message drop panic"), Fault::ReplyDrop => panic("panic in handler: injected reply drop panic"), - Fault::QueuedMessageDrop => panic("queued message destructor: injected queued drop panic"), + // A destructor panic after the death point is a cleanup failure + // (`ChildHandle::cleanup_failure`), never the exit reason. + Fault::QueuedMessageDrop => ExitReason::Normal, } } @@ -548,20 +550,30 @@ fn exercise(h: &dyn Harness, fault: Fault, gate: &Gate, events: &Events) -> Exit ); assert_eq!(h.observer_ask(), 7, "{fault:?}: observer stopped working"); let seen = events.snapshot(); + // A victim that panics in started() can pass its death point before the + // observer's monitor or link is registered: that registration gets NoProc. + let carries = |r: &ExitReason| *r == reason || (fault == Fault::Started && *r == ExitReason::NoProc); assert_eq!( seen.iter() - .filter(|e| matches!(e, Event::Down(m, r) if *m == monitor && *r == reason)) + .filter(|e| matches!(e, Event::Down(m, r) if *m == monitor && carries(r))) .count(), 1, "{fault:?}: {seen:?}" ); assert_eq!( seen.iter() - .filter(|e| matches!(e, Event::Exit(id, r) if *id == victim.id() && *r == reason)) + .filter(|e| matches!(e, Event::Exit(id, r) if *id == victim.id() && carries(r))) .count(), 1, "{fault:?}: {seen:?}" ); + if fault == Fault::QueuedMessageDrop { + assert_eq!( + victim.cleanup_failure().as_deref(), + Some("injected queued drop panic"), + "{fault:?}" + ); + } assert_eq!(seen.len(), 2, "{fault:?}: extra notification {seen:?}"); reason } diff --git a/concurrency/tests/startup_admission.rs b/concurrency/tests/startup_admission.rs index 9723e0e9..c83877f4 100644 --- a/concurrency/tests/startup_admission.rs +++ b/concurrency/tests/startup_admission.rs @@ -112,7 +112,7 @@ async fn managed_actor_stopped_before_first_poll_runs_no_hooks_but_drops_state() assert_eq!(counts.dropped.load(Ordering::SeqCst), 1); assert_eq!(dropped.load(Ordering::SeqCst), 2); assert_eq!( - actor.child_handle().mailbox_cleanup_failure().as_deref(), + actor.child_handle().cleanup_failure().as_deref(), Some("unstarted queued destructor") ); for reply in replies { diff --git a/concurrency/tests/supervision_trace.rs b/concurrency/tests/supervision_trace.rs index 6a900fb7..c8c23034 100644 --- a/concurrency/tests/supervision_trace.rs +++ b/concurrency/tests/supervision_trace.rs @@ -154,11 +154,15 @@ async fn trace_queued_destructor_panic_restarts_transient_child() { .unwrap() .unwrap(); assert_ne!(replacement.id(), actor.id()); + // The final reason stays Normal; the destructor panic after the death + // point is a cleanup failure, which restarts a Transient child. assert_eq!( child.snapshot().last_exit, - Some(spawned_concurrency::ExitReason::Panic( - "queued message destructor: injected queued destructor panic".into() - )) + Some(spawned_concurrency::ExitReason::Normal) + ); + assert_eq!( + child.snapshot().last_error.as_deref(), + Some("injected queued destructor panic") ); assert!(supervisor.shutdown(LIMIT).await.complete); write_trace( @@ -171,7 +175,7 @@ async fn trace_queued_destructor_panic_restarts_transient_child() { } #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] -async fn trace_queued_destructor_failure_preserves_shutdown() { +async fn trace_queued_destructor_failure_after_shutdown_restarts_transient_child() { use spawned_concurrency::ExitReason; let system = ActorSystem::new().unwrap(); let group = system.async_group("trace-drain-shutdown", 1).unwrap(); @@ -191,16 +195,21 @@ async fn trace_queued_destructor_failure_preserves_shutdown() { actor.send(DropBomb).unwrap(); actor.child_handle().shutdown(); release.send(()).unwrap(); - wait_state(&child, ChildState::Stopped).await; - assert_eq!(child.snapshot().generation, 1); + // v0.8: the reason stays Shutdown, but the cleanup failure restarts a + // Transient child (the supervisor is not closing). + let replacement = spawned_rt::tasks::timeout(LIMIT, child.wait_for_generation(1)) + .await + .unwrap() + .unwrap(); + assert_ne!(replacement.id(), actor.id()); assert_eq!(child.snapshot().last_exit, Some(ExitReason::Shutdown)); assert_eq!( - actor.child_handle().mailbox_cleanup_failure().as_deref(), + actor.child_handle().cleanup_failure().as_deref(), Some("injected queued destructor panic") ); assert!(supervisor.shutdown(LIMIT).await.complete); write_trace( - "queued_drop_preserves_shutdown", + "queued_drop_shutdown_restarts", supervisor.trace_id(), &[RestartMode::Transient], 5, diff --git a/verification/tla/README.md b/verification/tla/README.md index 69dad7a3..0ba1bd63 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -401,8 +401,11 @@ existing reason model. A negative control erases Kill and must reject the resulting transient restart; the entry point adds no new model transition. `queued_drop_panic` checks a queued destructor fault during ordinary stop and the resulting transient restart. Erasing its Panic exit must reject the tape. -`queued_drop_preserves_shutdown` retains a committed Shutdown despite a destructor -failure; escalating it to Panic must reject transient retirement. +`queued_drop_shutdown_restarts` keeps the raw Shutdown reason but records the +destructor failure after the death point as a cleanup failure, which the trace +projects to the abnormal class (v0.8, otp-exit-signals.md "Phases of an +exit"): the Transient child restarts. Projecting the raw Shutdown instead must +reject the restart. All library crates continue to forbid unsafe Rust; native synchronization and Linux affinity use safe interfaces. No unsafe exemption was introduced. diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 9a96cffa..07bafc43 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -297,13 +297,15 @@ def erase_drain_failure(events): event["x"] = "Normal" require_rejected(drain, erase_drain_failure, "transient restart after erased queued destructor panic", args.tlc) - shutdown = next(t for t in traces if t["name"] == "queued_drop_preserves_shutdown") - def escalate_shutdown(events): + shutdown = next(t for t in traces if t["name"] == "queued_drop_shutdown_restarts") + def drop_cleanup_failure(events): + # v0.8: a cleanup failure after Shutdown projects to the abnormal + # class; projecting the raw Shutdown instead must reject the restart. for event in events: - if event.get("x") == "Shutdown": - event["x"] = "Panic" - require_rejected(shutdown, escalate_shutdown, - "transient retirement after incorrectly escalated shutdown", args.tlc) + if event.get("a") == "SelfExit" and event.get("x") == "Panic": + event["x"] = "Shutdown" + require_rejected(shutdown, drop_cleanup_failure, + "transient restart after a Shutdown without its cleanup failure", args.tlc) replacement = next(t for t in traces if t["name"] == "transient_replacement_normal_before_ready") def fail_instead_of_retry(events): failed = next(i for i, e in enumerate(events) From 2a6774a3a4981a8ab088cfa2ca0150e7e91a1673 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:30:59 -0300 Subject: [PATCH 068/208] test: monitors past the death point get NoProc; stopped() panic replaces the reason; state destructor is a cleanup failure --- concurrency/tests/failpoints.rs | 13 ++++++++++--- concurrency/tests/foreign_panics.rs | 21 ++++++++++++++++----- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/concurrency/tests/failpoints.rs b/concurrency/tests/failpoints.rs index 04687211..9af929ea 100644 --- a/concurrency/tests/failpoints.rs +++ b/concurrency/tests/failpoints.rs @@ -723,7 +723,7 @@ async fn peer_exit_during_link_install_still_propagates() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn target_exit_during_monitor_install_still_delivers_down() { +async fn target_exit_during_monitor_install_delivers_noproc() { let _scenario = FailScenario::setup(); failpoints::configure( "monitor.before_install", @@ -737,11 +737,13 @@ async fn target_exit_during_monitor_install_still_delivers_down() { reached("monitor.before_install", 1).await; target.send(Crash).unwrap(); target.wait_exit().await; + // The registration completes after the target's death point: exactly + // one Down, carrying NoProc (E4). let reason = tokio::time::timeout(LIMIT, received.recv()) .await .unwrap() .unwrap(); - assert!(matches!(reason, ExitReason::Panic(_))); + assert_eq!(reason, ExitReason::NoProc); watcher.context().stop(); watcher.join().await; } @@ -768,11 +770,16 @@ async fn injected_started_panic_is_an_abnormal_exit_seen_by_monitors() { other => panic!("expected panic exit, got {other:?}"), } assert!(started.handles().is_empty()); + // The monitor races the target's death point: before it, the Down + // carries the panic; after it, NoProc. let reason = tokio::time::timeout(LIMIT, received.recv()) .await .unwrap() .unwrap(); - assert!(matches!(reason, ExitReason::Panic(_))); + assert!( + matches!(reason, ExitReason::Panic(_) | ExitReason::NoProc), + "got {reason:?}" + ); watcher.context().stop(); watcher.join().await; } diff --git a/concurrency/tests/foreign_panics.rs b/concurrency/tests/foreign_panics.rs index 909bbdfa..fdef017c 100644 --- a/concurrency/tests/foreign_panics.rs +++ b/concurrency/tests/foreign_panics.rs @@ -550,7 +550,14 @@ fn subscriber_child() { println!( "{place:?} panic_in_stopped={panic_in_stopped} reason={reason:?} stopped_ran={ran}" ); - if reason != handler_panic() || !ran { + // v0.8: a panic in stopped() replaces an earlier abnormal reason + // (as a crash in OTP's terminate), unless a kill was received. + let expected = if panic_in_stopped { + ExitReason::Panic("panic in stopped(): injected stopped panic".into()) + } else { + handler_panic() + }; + if reason != expected || !ran { callback_gaps.push(format!( "{place:?} stopped_panic={panic_in_stopped}: {reason:?}, stopped() ran: {ran}" )); @@ -648,9 +655,8 @@ fn reply_drop_child() { "{place:?}: stopped() skipped" ); } - // A held reply whose Drop panics while state is destroyed: a normal - // stop escalates to a destructor panic, an unwinding actor keeps its - // handler panic as the reason. + // A held reply whose Drop panics while state is destroyed (after the + // death point): a cleanup failure, never the reason. for unwinding in [false, true] { let stopped = Arc::new(AtomicBool::new(false)); let mut victim = Victim::new(&stopped, false); @@ -666,9 +672,14 @@ fn reply_drop_child() { let expected = if unwinding { handler_panic() } else { - ExitReason::Panic(format!("actor state destructor: {REPLY_PANIC}")) + ExitReason::Normal }; assert_eq!(reason, expected, "{place:?} unwinding={unwinding}"); + assert_eq!( + actor.child().cleanup_failure(), + Some(format!("actor state destructor: {REPLY_PANIC}")), + "{place:?} unwinding={unwinding}" + ); } // The process and the remaining actors stay healthy. let stopped = Arc::new(AtomicBool::new(false)); From 5030b429fb6785834ca61845a7ea4eccb94c4d0a Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:33:12 -0300 Subject: [PATCH 069/208] feat(test)!: spawned-test exit injection follows exit_signal/2; update kit tests --- concurrency/tests/mailbox_drop_panic.rs | 4 +- testing/src/lib.rs | 8 +-- testing/src/tasks.rs | 2 +- testing/tests/tasks_kit.rs | 69 ++++++++++++++----------- testing/tests/threads_kit.rs | 63 +++++++++++----------- 5 files changed, 77 insertions(+), 69 deletions(-) diff --git a/concurrency/tests/mailbox_drop_panic.rs b/concurrency/tests/mailbox_drop_panic.rs index eab880be..e5ee8b5f 100644 --- a/concurrency/tests/mailbox_drop_panic.rs +++ b/concurrency/tests/mailbox_drop_panic.rs @@ -312,6 +312,7 @@ fn abandoned_receiver_child() { } } // An actor-state panic must not unwind through an unguarded mailbox. + // After the death point it is a cleanup failure; the reason stays Normal. let actor = Victim(Arc::default(), true).start(); let (entered, arrival) = oneshot::channel(); let (release, gate) = oneshot::channel(); @@ -320,7 +321,8 @@ fn abandoned_receiver_child() { let dropped = Arc::new(AtomicUsize::new(0)); for _ in 0..2 { actor.send(Bomb(dropped.clone())).unwrap(); } actor.context().stop(); release.send(()).unwrap(); - assert!(matches!(tokio::time::timeout(LIMIT, actor.wait_exit()).await.unwrap(), ExitReason::Panic(_))); + assert_eq!(tokio::time::timeout(LIMIT, actor.wait_exit()).await.unwrap(), ExitReason::Normal); + assert!(actor.child_handle().cleanup_failure().is_some()); assert_eq!(dropped.load(Ordering::SeqCst), 2); }); runtime.shutdown_timeout(LIMIT); diff --git a/testing/src/lib.rs b/testing/src/lib.rs index 38f353e0..547a8cc0 100644 --- a/testing/src/lib.rs +++ b/testing/src/lib.rs @@ -17,10 +17,10 @@ //! - **Harness helpers**: [`tasks::settle`], [`tasks::exit_within`], //! [`tasks::wait_for_generation`], [`tasks::shutdown_system`] and //! [`tasks::TestSystem`] turn hangs into test failures with a report. -//! - **Fault injection**: [`faults::exit`] stops any actor with a chosen -//! [`ExitReason`](spawned_concurrency::ExitReason) (`Panic` travels -//! through a link, while `Kill` is direct; an injected panic surfaces as -//! `Panic("panic in handler: ..")` and the default panic hook prints it); +//! - **Fault injection**: [`faults::exit`] sends an exact exit signal with a +//! chosen [`ExitReason`](spawned_concurrency::ExitReason) +//! (`ChildHandle::exit`; `Kill` ends the target `Killed`, `Normal` is +//! ignored by a non-trapping target, a trapping target gets an `Exit`); //! [`tasks::restart_child`] crashes a supervised child and returns its //! replacement; mocks delay, discard or crash on chosen messages. //! diff --git a/testing/src/tasks.rs b/testing/src/tasks.rs index a7c84ba2..6d0f83f6 100644 --- a/testing/src/tasks.rs +++ b/testing/src/tasks.rs @@ -340,7 +340,7 @@ pub fn wait_for_generation( /// together, even if the child restarts on its own meanwhile, so the returned /// reference is the first replacement of the targeted instance. A later /// instance is never faulted: the fault is aimed at the targeted instance's -/// own handle, and a `Kill` is committed before `faults::exit` returns. If the +/// own handle, and the signal is applied before `faults::exit` returns. If the /// targeted instance dies of its own cause first, the fault is a no-op and the /// replacement after that death is returned. #[track_caller] diff --git a/testing/tests/tasks_kit.rs b/testing/tests/tasks_kit.rs index 5520fff9..54f3fbaf 100644 --- a/testing/tests/tasks_kit.rs +++ b/testing/tests/tasks_kit.rs @@ -262,25 +262,24 @@ async fn settle_waits_for_earlier_sends() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn exit_injects_every_reason() { - for reason in [ - ExitReason::Normal, - ExitReason::Shutdown, - ExitReason::Panic("injected".into()), - ExitReason::Kill, + // Exact exit_signal/2 semantics: the target ends with the signal's reason, + // a kill ends it Killed. + for (reason, expected) in [ + (ExitReason::Shutdown, ExitReason::Shutdown), + (ExitReason::Panic("injected".into()), ExitReason::Panic("injected".into())), + (ExitReason::Kill, ExitReason::Killed), ] { let actor = Counter(Arc::default()).start(); - faults::exit(&actor.child_handle(), reason.clone()); - let observed = join_within(&actor, WAIT).await; - match reason { - ExitReason::Panic(msg) => { - assert_eq!( - observed, - ExitReason::Panic(format!("panic in handler: {msg}")) - ) - } - other => assert_eq!(observed, other), - } + faults::exit(&actor.child_handle(), reason); + assert_eq!(join_within(&actor, WAIT).await, expected); } + // Normal is ignored by a non-trapping target. + let actor = Counter(Arc::default()).start(); + faults::exit(&actor.child_handle(), ExitReason::Normal); + assert_eq!(settle(&actor, Get, WAIT).await, 0); + assert_eq!(actor.child_handle().exit_reason(), None); + actor.context().stop(); + assert_eq!(join_within(&actor, WAIT).await, ExitReason::Normal); } #[tokio::test] @@ -543,7 +542,7 @@ async fn exit_injection_never_blocks_a_current_thread_runtime() { let observed = join_within(&actor, WAIT).await; assert!(matches!( (&reason, &observed), - (ExitReason::Panic(_), ExitReason::Panic(_)) | (ExitReason::Kill, ExitReason::Kill) + (ExitReason::Panic(_), ExitReason::Panic(_)) | (ExitReason::Kill, ExitReason::Killed) )); // A second injection into the exited target is a non-blocking no-op. faults::exit(&actor.child_handle(), reason); @@ -594,18 +593,26 @@ async fn async_expect_names_its_caller() { } #[tokio::test] -async fn direct_kill_and_shutdown_stop_trapping_targets_without_killing_trapping_peers() { - for reason in [ExitReason::Kill, ExitReason::Shutdown] { - let target = Counter(Arc::default()).start(); - let observer = Counter(Arc::default()).start(); - target.context().trap_exit(true); - observer.context().trap_exit(true); - observer.context().link(&target.child_handle()); - faults::exit(&target.child_handle(), reason.clone()); - assert_eq!(join_within(&target, WAIT).await, reason); - assert_eq!(settle(&observer, Get, WAIT).await, 0); - assert_eq!(observer.child_handle().exit_reason(), None); - observer.context().stop(); - assert_eq!(join_within(&observer, WAIT).await, ExitReason::Normal); - } +async fn kill_stops_a_trapping_target_and_other_signals_are_trapped() { + // A kill is untrappable: the trapping target ends Killed and its trapping + // peer receives the (trappable) Killed notice and survives. + let target = Counter(Arc::default()).start(); + let observer = Counter(Arc::default()).start(); + target.context().trap_exit(true); + observer.context().trap_exit(true); + observer.context().link(&target.child_handle()); + faults::exit(&target.child_handle(), ExitReason::Kill); + assert_eq!(join_within(&target, WAIT).await, ExitReason::Killed); + assert_eq!(settle(&observer, Get, WAIT).await, 0); + assert_eq!(observer.child_handle().exit_reason(), None); + observer.context().stop(); + assert_eq!(join_within(&observer, WAIT).await, ExitReason::Normal); + // Any other signal to a trapping target is queued as an Exit. + let target = Counter(Arc::default()).start(); + target.context().trap_exit(true); + faults::exit(&target.child_handle(), ExitReason::Shutdown); + assert_eq!(settle(&target, Get, WAIT).await, 0); + assert_eq!(target.child_handle().exit_reason(), None); + target.child_handle().shutdown(); + assert_eq!(join_within(&target, WAIT).await, ExitReason::Shutdown); } diff --git a/testing/tests/threads_kit.rs b/testing/tests/threads_kit.rs index 4727d4f3..b1062d83 100644 --- a/testing/tests/threads_kit.rs +++ b/testing/tests/threads_kit.rs @@ -162,32 +162,26 @@ impl Handler for Idle { #[test] fn native_exit_injection() { - for reason in [ - ExitReason::Normal, - ExitReason::Shutdown, - ExitReason::Panic("injected".into()), - ] { + for reason in [ExitReason::Shutdown, ExitReason::Panic("injected".into())] { let actor = Idle.start(); settle(&actor, Ping, WAIT); faults::exit(&actor.child_handle(), reason.clone()); - let observed = join_within(&actor, WAIT); - match reason { - ExitReason::Panic(msg) => { - assert_eq!( - observed, - ExitReason::Panic(format!("panic in handler: {msg}")) - ) - } - other => assert_eq!(observed, other), - } + assert_eq!(join_within(&actor, WAIT), reason); } + // Normal is ignored by a non-trapping target. + let actor = Idle.start(); + faults::exit(&actor.child_handle(), ExitReason::Normal); + settle(&actor, Ping, WAIT); + assert_eq!(actor.child_handle().exit_reason(), None); + actor.context().stop(); + assert_eq!(join_within(&actor, WAIT), ExitReason::Normal); } #[test] fn native_kill_injection() { let actor = Idle.start(); faults::exit(&actor.child_handle(), ExitReason::Kill); - assert_eq!(join_within(&actor, WAIT), ExitReason::Kill); + assert_eq!(join_within(&actor, WAIT), ExitReason::Killed); } #[test] @@ -296,7 +290,7 @@ fn native_panic_injection_into_an_exited_target_is_a_no_op() { let actor = Idle.start(); actor.child_handle().shutdown(); assert_eq!(join_within(&actor, WAIT), ExitReason::Shutdown); - // The target's link stops the fuse before it detonates: not an error. + // A signal to an actor past its death point is dropped. faults::exit(&actor.child_handle(), ExitReason::Panic("late".into())); } @@ -306,7 +300,7 @@ fn native_kill_injection_is_reliable() { for _ in 0..200 { let actor = Idle.start(); faults::exit(&actor.child_handle(), ExitReason::Kill); - assert_eq!(join_within(&actor, WAIT), ExitReason::Kill); + assert_eq!(join_within(&actor, WAIT), ExitReason::Killed); } } @@ -413,18 +407,23 @@ fn native_helpers_name_their_caller() { } #[test] -fn direct_kill_and_shutdown_stop_trapping_targets_without_killing_trapping_peers() { - for reason in [ExitReason::Kill, ExitReason::Shutdown] { - let target = Idle.start(); - let observer = Idle.start(); - target.context().trap_exit(true); - observer.context().trap_exit(true); - observer.context().link(&target.child_handle()); - faults::exit(&target.child_handle(), reason.clone()); - assert_eq!(join_within(&target, WAIT), reason); - settle(&observer, Ping, WAIT); - assert_eq!(observer.child_handle().exit_reason(), None); - observer.context().stop(); - assert_eq!(join_within(&observer, WAIT), ExitReason::Normal); - } +fn kill_stops_a_trapping_target_and_other_signals_are_trapped() { + let target = Idle.start(); + let observer = Idle.start(); + target.context().trap_exit(true); + observer.context().trap_exit(true); + observer.context().link(&target.child_handle()); + faults::exit(&target.child_handle(), ExitReason::Kill); + assert_eq!(join_within(&target, WAIT), ExitReason::Killed); + settle(&observer, Ping, WAIT); + assert_eq!(observer.child_handle().exit_reason(), None); + observer.context().stop(); + assert_eq!(join_within(&observer, WAIT), ExitReason::Normal); + let target = Idle.start(); + target.context().trap_exit(true); + faults::exit(&target.child_handle(), ExitReason::Shutdown); + settle(&target, Ping, WAIT); + assert_eq!(target.child_handle().exit_reason(), None); + target.child_handle().shutdown(); + assert_eq!(join_within(&target, WAIT), ExitReason::Shutdown); } From f60f5b829a0aa9030d3f5ae5de8bf1786022d68d Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:37:54 -0300 Subject: [PATCH 070/208] feat(supervision): supervisor and system shutdown send the parent exit signal; kill at shutdown_timeout/grace (E3) --- concurrency/src/supervision.rs | 36 +- concurrency/src/system.rs | 45 ++- concurrency/tests/otp_exit_signals.rs | 487 ++++++++++++++++++++++++++ 3 files changed, 561 insertions(+), 7 deletions(-) create mode 100644 concurrency/tests/otp_exit_signals.rs diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs index a43a1b8c..96e7ddf8 100644 --- a/concurrency/src/supervision.rs +++ b/concurrency/src/supervision.rs @@ -523,11 +523,13 @@ impl ChildControl for View { self.changed.send_modify(|n| *n = n.wrapping_add(1)); true } + /// Collateral stop (OneForAll/RestForOne): the supervisor's parent exit + /// signal (E3); the controller escalates to a kill at `shutdown_timeout`. fn stop_generation(&self, slot: &RecoverySlot) { slot.stop.cancel(); let scope = system::lock(&self.data).scope.clone(); if let Some(scope) = scope { - scope.stop(ExitReason::Shutdown); + scope.signal_parent(ExitReason::Shutdown); } } fn snapshot(&self) -> ChildSnapshot { @@ -546,10 +548,15 @@ impl ChildControl for View { } self.changed.send_modify(|n| *n = n.wrapping_add(1)); } + /// Supervisor shutdown of this child (V9, reverse order): the parent + /// exit signal `exit(Child, shutdown)` (E3). A non-trapping child ends + /// `Shutdown` without `stopped()`; a trapping child finishes its earlier + /// messages and runs `stopped()`. The controller kills it at + /// `shutdown_timeout`. fn stop(&self) { self.stop_turn.cancel(); if let Some(scope) = system::lock(&self.data).scope.clone() { - scope.stop(ExitReason::Shutdown); + scope.signal_parent(ExitReason::Shutdown); } } fn done(&self) -> watch::Receiver { @@ -2051,7 +2058,7 @@ async fn run_child(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc { + scope.kill(); + // The second interval runs from the kill. + cleanup_until = Some(Instant::now()); + match rt::timeout(spec.shutdown_timeout, child.wait_exit_async()) + .await + { + Ok(reason) => reason, + Err(_) => { + stuck = true; + core.fail_cleanup_for( + view.snapshot(), + format!("actor cleanup stuck for {}", spec.name), + || Some(Event::new("CleanupStuck", c, "")), + ); + child.wait_exit_async().await + } + } + } Err(_) => { if !stop_turn { trace::emit(core.trace, || { diff --git a/concurrency/src/system.rs b/concurrency/src/system.rs index 2bd2ca27..21948284 100644 --- a/concurrency/src/system.rs +++ b/concurrency/src/system.rs @@ -524,16 +524,19 @@ impl SystemInner { } } ShutdownTarget::Scope(scope) => { + // The system is a standalone actor's parent: its + // Shutdown is a parent exit signal (E3). scope.set_cleanup_timeout(remaining); - scope.stop(ExitReason::Shutdown); + scope.signal_parent(ExitReason::Shutdown); let _ = rt::timeout(remaining, scope.joined()).await; } } } - // At grace expiry, no nested child may remain unsignalled behind - // a stuck sibling's longer per-child cleanup timeout. + // At grace expiry every remaining actor gets the kill (E3), so + // no nested child remains unsignalled behind a stuck sibling's + // longer per-child cleanup timeout. for scope in scopes.iter().rev() { - scope.stop(ExitReason::Shutdown); + scope.kill(); } // Runtime teardown still requires actual completion, independently // of the ordering grace and the caller's report timeout. @@ -983,6 +986,40 @@ impl Scope { pub(crate) fn close_admission(&self) { lock(&self.state).open = false; } + /// The parent's exit signal to this scope's actor (design E3): the + /// supervisor or system is the parent. Admission closes for the managed + /// tree at once (B), but nothing is committed for the actor: a + /// non-trapping actor begins exit with `reason` without `stopped()` + /// (`Normal` is ignored), a trapping one gets a parent exit queued behind + /// its messages, which ends it through `stopped()`. Before the actor is + /// attached this is an ordinary stop. + pub(crate) fn signal_parent(&self, reason: ExitReason) { + let child = { + let mut state = lock(&self.state); + state.open = false; + if state.ended_at.is_none() { + state.ended_at = Some(Instant::now()); + } + state.actor.clone() + }; + match child { + Some(child) => child.exit_from(crate::ActorId::EXTERNAL, reason, true), + None => self.stop(reason), + } + } + /// The parent's kill (E3 escalation at `shutdown_timeout` or grace): the + /// actor ends `Killed`, skipping a `stopped()` that has not begun. + pub(crate) fn kill(&self) { + let child = { + let mut state = lock(&self.state); + state.open = false; + state.actor.clone() + }; + match child { + Some(child) => child.exit_from(crate::ActorId::EXTERNAL, ExitReason::Kill, true), + None => self.stop(ExitReason::Killed), + } + } pub(crate) fn register( &self, name: &str, diff --git a/concurrency/tests/otp_exit_signals.rs b/concurrency/tests/otp_exit_signals.rs new file mode 100644 index 00000000..dc3e28d7 --- /dev/null +++ b/concurrency/tests/otp_exit_signals.rs @@ -0,0 +1,487 @@ +//! Exit signals, kills, NoProc and parent exits as in OTP +//! (docs/design/otp-exit-signals.md, ranks 5e and 5g). Each test names the +//! design rule it pins; every one fails on v0.7 (the API or the outcome +//! differs there). +#![cfg(feature = "async")] +use spawned_concurrency::{message::Message, ActorId, Down, Exit, ExitReason, NoProc}; +use std::{ + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, + time::Duration, +}; + +const LIMIT: Duration = Duration::from_secs(10); + +/// What an actor observed: `exit_received` calls, `Down`s and whether +/// `stopped()` ran. +#[derive(Default, Clone)] +struct Seen { + exits: Arc>>, + downs: Arc>>, + stopped: Arc, +} +impl Seen { + fn exits(&self) -> Vec { + self.exits.lock().unwrap().clone() + } + fn downs(&self) -> Vec { + self.downs.lock().unwrap().clone() + } + fn stopped(&self) -> bool { + self.stopped.load(Ordering::SeqCst) + } +} + +struct Ping; +impl Message for Ping { + type Result = u8; +} + +mod tasks_mode { + use super::*; + use spawned_concurrency::tasks::{Actor, ActorRef, ActorStart, Context, Handler}; + use spawned_concurrency::ChildHandle; + + pub struct Node { + pub seen: Seen, + pub trap: bool, + /// stopped() waits for this signal (to act while it runs). + pub in_stopped: Option<(tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>)>, + pub panic_in_stopped: bool, + } + impl Node { + pub fn new(seen: &Seen, trap: bool) -> Self { + Self { + seen: seen.clone(), + trap, + in_stopped: None, + panic_in_stopped: false, + } + } + } + impl Actor for Node { + async fn started(&mut self, ctx: &Context) { + if self.trap { + ctx.trap_exit(true); + } + } + async fn stopped(&mut self, _: &Context) { + self.seen.stopped.store(true, Ordering::SeqCst); + if let Some((entered, release)) = self.in_stopped.take() { + let _ = entered.send(()); + let _ = release.await; + } + if self.panic_in_stopped { + panic!("stopped() panic after the kill"); + } + } + async fn exit_received(&mut self, exit: Exit, _: &Context) { + self.seen.exits.lock().unwrap().push(exit); + } + } + impl Handler for Node { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, u8>, + _: &Context, + ) { + reply.reply(7); + } + } + impl Handler for Node { + async fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { + self.seen.downs.lock().unwrap().push(down); + reply.reply(()); + } + } + struct Do(Box) + Send>); + impl Message for Do { + type Result = (); + } + impl Handler for Node { + async fn handle( + &mut self, + f: Do, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + (f.0)(ctx); + reply.reply(()); + } + } + /// Run `f` inside one of `actor`'s handlers. + pub async fn on(actor: &ActorRef, f: impl FnOnce(&Context) + Send + 'static) { + actor.request(Do(Box::new(f))).await.unwrap(); + } + pub async fn ended(child: &ChildHandle) -> ExitReason { + tokio::time::timeout(LIMIT, child.wait_exit_async()) + .await + .expect("actor did not exit") + } + /// A request round trip, so everything queued before it was handled. + pub async fn settle(actor: &ActorRef) { + assert_eq!(actor.request(Ping).await.unwrap(), 7); + } + + // E1: kill() is untrappable and ends the target Killed without stopped(); + // links receive Killed (trappable), monitors Down { Killed }. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn kill_is_untrappable_and_propagates_killed() { + let (target_seen, peer_seen, watcher_seen, trapper_seen) = + (Seen::default(), Seen::default(), Seen::default(), Seen::default()); + let target = Node::new(&target_seen, true).start(); + let peer = Node::new(&peer_seen, false).start(); + let trapper = Node::new(&trapper_seen, true).start(); + let watcher = Node::new(&watcher_seen, false).start(); + settle(&target).await; + settle(&trapper).await; + let t = target.child_handle(); + let t2 = t.clone(); + on(&peer, move |ctx| ctx.link(&t2)).await; + let t3 = t.clone(); + on(&trapper, move |ctx| ctx.link(&t3)).await; + let t4 = t.clone(); + on(&watcher, move |ctx| { + ctx.monitor(&t4); + }) + .await; + t.kill(); + assert_eq!(ended(&t).await, ExitReason::Killed); + assert!(!target_seen.stopped(), "stopped() runs after a kill"); + assert!(target_seen.exits().is_empty(), "a kill was trapped"); + // A non-trapping peer ends Killed (and skips stopped()). + assert_eq!(ended(&peer.child_handle()).await, ExitReason::Killed); + assert!(!peer_seen.stopped()); + // A trapping peer gets the notice as a message. + settle(&trapper).await; + assert_eq!( + trapper_seen.exits(), + vec![Exit { from: t.id(), reason: ExitReason::Killed }] + ); + settle(&watcher).await; + assert_eq!(watcher_seen.downs().len(), 1); + assert_eq!(watcher_seen.downs()[0].reason, ExitReason::Killed); + trapper.child_handle().stop(); + watcher.child_handle().stop(); + ended(&trapper.child_handle()).await; + ended(&watcher.child_handle()).await; + } + + // E1 + precedence: a kill during stopped() replaces the requested reason; + // the running stopped() finishes and a panic in it does not replace Killed. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn kill_during_stopped_supersedes_and_panic_does_not_replace_it() { + let seen = Seen::default(); + let (entered_tx, entered) = tokio::sync::oneshot::channel(); + let (release, release_rx) = tokio::sync::oneshot::channel(); + let mut node = Node::new(&seen, false); + node.in_stopped = Some((entered_tx, release_rx)); + node.panic_in_stopped = true; + let actor = node.start(); + settle(&actor).await; + let child = actor.child_handle(); + child.stop_with_reason(ExitReason::Shutdown); + entered.await.unwrap(); + child.kill(); + release.send(()).unwrap(); + assert_eq!(ended(&child).await, ExitReason::Killed); + assert!(seen.stopped(), "a stopped() that began finishes"); + } + + // E1/decision 5: an administrative stop_with_reason(Kill) stays an + // administrative stop ending Kill, and stopped() runs. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn administrative_kill_reason_runs_stopped_and_ends_kill() { + let seen = Seen::default(); + let actor = Node::new(&seen, true).start(); + settle(&actor).await; + actor.child_handle().stop_with_reason(ExitReason::Kill); + assert_eq!(ended(&actor.child_handle()).await, ExitReason::Kill); + assert!(seen.stopped()); + } + + // E2: exit(Normal) is ignored by a non-trapping actor and trapped as an + // Exit from EXTERNAL by a trapping one; any other reason ends a + // non-trapping actor with it, without stopped(). + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn exit_signals_follow_exit_signal_2() { + let plain_seen = Seen::default(); + let plain = Node::new(&plain_seen, false).start(); + settle(&plain).await; + plain.child_handle().exit(ExitReason::Normal); + settle(&plain).await; + assert!(plain.child_handle().is_alive(), "Normal stopped a non-trapping actor"); + + let trap_seen = Seen::default(); + let trapper = Node::new(&trap_seen, true).start(); + settle(&trapper).await; + trapper.child_handle().exit(ExitReason::Normal); + trapper.child_handle().exit(ExitReason::Shutdown); + settle(&trapper).await; + assert_eq!( + trap_seen.exits(), + vec![ + Exit { from: ActorId::EXTERNAL, reason: ExitReason::Normal }, + Exit { from: ActorId::EXTERNAL, reason: ExitReason::Shutdown }, + ] + ); + assert!(trapper.child_handle().is_alive()); + + plain.child_handle().exit(ExitReason::Shutdown); + assert_eq!(ended(&plain.child_handle()).await, ExitReason::Shutdown); + assert!(!plain_seen.stopped(), "stopped() ran after an untrapped signal"); + trapper.child_handle().stop(); + ended(&trapper.child_handle()).await; + } + + // E2: ctx.exit sends from the calling actor; a Kill from an actor is a kill. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn ctx_exit_sends_from_the_calling_actor() { + let (a_seen, b_seen) = (Seen::default(), Seen::default()); + let a = Node::new(&a_seen, false).start(); + let b = Node::new(&b_seen, true).start(); + settle(&b).await; + let bh = b.child_handle(); + let bh2 = bh.clone(); + on(&a, move |ctx| ctx.exit(&bh2, ExitReason::Panic("from a".into()))).await; + settle(&b).await; + assert_eq!( + b_seen.exits(), + vec![Exit { from: a.id(), reason: ExitReason::Panic("from a".into()) }] + ); + let bh3 = bh.clone(); + on(&a, move |ctx| ctx.exit(&bh3, ExitReason::Kill)).await; + assert_eq!(ended(&bh).await, ExitReason::Killed); + a.child_handle().stop(); + ended(&a.child_handle()).await; + } + + // E4 + decision 1: link/monitor to an actor past its death point give + // NoProc; a non-trapping caller exits NoProc without stopped(); + // try_link returns Err(NoProc) and keeps nothing. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn registrations_after_death_report_noproc() { + let dead_seen = Seen::default(); + let dead = Node::new(&dead_seen, false).start(); + dead.child_handle().stop_with_reason(ExitReason::Panic("gone".into())); + assert!(matches!(ended(&dead.child_handle()).await, ExitReason::Panic(_))); + let d = dead.child_handle(); + + let trap_seen = Seen::default(); + let trapper = Node::new(&trap_seen, true).start(); + let (d1, d2) = (d.clone(), d.clone()); + on(&trapper, move |ctx| { + ctx.link(&d1); + ctx.monitor(&d2); + }) + .await; + settle(&trapper).await; + assert_eq!( + trap_seen.exits(), + vec![Exit { from: d.id(), reason: ExitReason::NoProc }] + ); + let downs = trap_seen.downs(); + assert_eq!(downs.len(), 1); + assert_eq!(downs[0].reason, ExitReason::NoProc); + + let try_seen = Seen::default(); + let trier = Node::new(&try_seen, false).start(); + let result = Arc::new(Mutex::new(None)); + let (d3, r) = (d.clone(), result.clone()); + on(&trier, move |ctx| *r.lock().unwrap() = Some(ctx.try_link(&d3))).await; + assert_eq!(*result.lock().unwrap(), Some(Err(NoProc))); + settle(&trier).await; + assert!(trier.child_handle().is_alive()); + + let plain_seen = Seen::default(); + let plain = Node::new(&plain_seen, false).start(); + let d4 = d.clone(); + on(&plain, move |ctx| ctx.link(&d4)).await; + assert_eq!(ended(&plain.child_handle()).await, ExitReason::NoProc); + assert!(!plain_seen.stopped()); + + trapper.child_handle().stop(); + trier.child_handle().stop(); + ended(&trapper.child_handle()).await; + ended(&trier.child_handle()).await; + } + + pub struct Parent { + pub child: Option>, + pub child_seen: Seen, + } + impl Actor for Parent {} + struct Spawn; + impl Message for Spawn { + type Result = ChildHandle; + } + impl Handler for Parent { + async fn handle( + &mut self, + _: Spawn, + reply: spawned_concurrency::reply::PendingReply<'_, ChildHandle>, + ctx: &Context, + ) { + let child = Node::new(&self.child_seen, true).start_linked(ctx); + reply.reply(child.child_handle()); + self.child = Some(child); + } + } + + // E3: a start_linked parent's exit, Normal included, ends a trapping + // child through stopped(); exit_received never sees it. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn parent_exit_ends_a_trapping_child_normally() { + for parent_reason in [ExitReason::Normal, ExitReason::Shutdown] { + let child_seen = Seen::default(); + let parent = Parent { + child: None, + child_seen: child_seen.clone(), + } + .start(); + let child = parent.request(Spawn).await.unwrap(); + parent.child_handle().stop_with_reason(parent_reason.clone()); + assert_eq!(ended(&child).await, parent_reason); + assert!(child_seen.stopped(), "the parent exit skipped stopped()"); + assert!(child_seen.exits().is_empty(), "exit_received saw the parent exit"); + } + } + + // Phase 0: signals that arrive while started() runs are resolved when it + // returns, with the trap flag read then. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn signals_during_started_are_resolved_with_the_flag_set_in_started() { + struct Slow(Seen, Option>); + impl Actor for Slow { + async fn started(&mut self, ctx: &Context) { + self.1.take().unwrap().await.unwrap(); + ctx.trap_exit(true); + } + async fn exit_received(&mut self, exit: Exit, _: &Context) { + self.0.exits.lock().unwrap().push(exit); + } + } + impl Handler for Slow { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, u8>, + _: &Context, + ) { + reply.reply(7); + } + } + let seen = Seen::default(); + let (go, gate) = tokio::sync::oneshot::channel(); + let actor = Slow(seen.clone(), Some(gate)).start(); + actor.child_handle().exit(ExitReason::Shutdown); + go.send(()).unwrap(); + assert_eq!(actor.request(Ping).await.unwrap(), 7); + assert_eq!( + seen.exits(), + vec![Exit { from: ActorId::EXTERNAL, reason: ExitReason::Shutdown }] + ); + actor.child_handle().stop(); + ended(&actor.child_handle()).await; + } + + // Decision 8: a late non-kill signal replaces the requested reason of a + // non-trapping actor before D; a running stopped() finishes. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn late_signal_replaces_the_requested_reason() { + let seen = Seen::default(); + let (entered_tx, entered) = tokio::sync::oneshot::channel(); + let (release, release_rx) = tokio::sync::oneshot::channel(); + let mut node = Node::new(&seen, false); + node.in_stopped = Some((entered_tx, release_rx)); + let actor = node.start(); + settle(&actor).await; + let child = actor.child_handle(); + child.stop(); + entered.await.unwrap(); + child.exit(ExitReason::Shutdown); + release.send(()).unwrap(); + assert_eq!(ended(&child).await, ExitReason::Shutdown); + assert!(seen.stopped()); + } +} + +mod threads_mode { + use super::*; + use spawned_concurrency::threads::{Actor, ActorStart, Context, Handler}; + use spawned_concurrency::ChildHandle; + + struct Node(Seen, bool); + impl Actor for Node { + fn started(&mut self, ctx: &Context) { + if self.1 { + ctx.trap_exit(true); + } + } + fn stopped(&mut self, _: &Context) { + self.0.stopped.store(true, Ordering::SeqCst); + } + fn exit_received(&mut self, exit: Exit, _: &Context) { + self.0.exits.lock().unwrap().push(exit); + } + } + impl Handler for Node { + fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, u8>, + _: &Context, + ) { + reply.reply(7); + } + } + fn ended(child: &ChildHandle) -> ExitReason { + let (tx, rx) = std::sync::mpsc::channel(); + let c = child.clone(); + std::thread::spawn(move || tx.send(c.wait_exit_blocking()).unwrap()); + rx.recv_timeout(LIMIT).expect("actor did not exit") + } + + #[test] + fn kill_exit_and_noproc_in_threads_mode() { + // Kill: untrappable, Killed, no stopped(); the trapping peer gets Killed. + let (target_seen, peer_seen) = (Seen::default(), Seen::default()); + let target = Node(target_seen.clone(), true).start(); + let peer = Node(peer_seen.clone(), true).start(); + assert_eq!(target.request(Ping).unwrap(), 7); + assert_eq!(peer.request(Ping).unwrap(), 7); + peer.context().link(&target.child_handle()); + target.child_handle().kill(); + assert_eq!(ended(&target.child_handle()), ExitReason::Killed); + assert!(!target_seen.stopped()); + assert_eq!(peer.request(Ping).unwrap(), 7); + assert_eq!( + peer_seen.exits(), + vec![Exit { from: target.id(), reason: ExitReason::Killed }] + ); + // NoProc for a late link and try_link. + assert_eq!(peer.context().try_link(&target.child_handle()), Err(NoProc)); + peer.context().link(&target.child_handle()); + assert_eq!(peer.request(Ping).unwrap(), 7); + assert_eq!(peer_seen.exits().last().unwrap().reason, ExitReason::NoProc); + // exit(Normal) ignored by a non-trapping actor; exit(Shutdown) ends it + // without stopped(). + let plain_seen = Seen::default(); + let plain = Node(plain_seen.clone(), false).start(); + plain.child_handle().exit(ExitReason::Normal); + assert_eq!(plain.request(Ping).unwrap(), 7); + plain.child_handle().exit(ExitReason::Shutdown); + assert_eq!(ended(&plain.child_handle()), ExitReason::Shutdown); + assert!(!plain_seen.stopped()); + peer.child_handle().stop(); + assert_eq!(ended(&peer.child_handle()), ExitReason::Normal); + } +} From 7fd4a135ce91d37eaa5938246be3cee461a39037 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:40:36 -0300 Subject: [PATCH 071/208] feat(link): link from an actor whose exit began is inert; tests and semantics X2 line; otp_exit_signals tests --- concurrency/src/link.rs | 27 ++++++++++++++++++++++++++- concurrency/src/tasks/actor.rs | 4 +++- concurrency/src/threads/actor.rs | 4 +++- concurrency/tests/otp_exit_signals.rs | 26 ++++++++++++++++++++++++++ docs/design/semantics.md | 8 ++++++++ 5 files changed, 66 insertions(+), 3 deletions(-) diff --git a/concurrency/src/link.rs b/concurrency/src/link.rs index 35806b89..5801dc2c 100644 --- a/concurrency/src/link.rs +++ b/concurrency/src/link.rs @@ -313,7 +313,8 @@ fn with_pair( #[derive(Debug, PartialEq, Eq)] pub(crate) enum LinkOutcome { Linked, - /// Nothing to do: self link, an existing record, or the caller is past D. + /// Nothing to do: self link, an existing record, or the caller's exit + /// began (inert, as `monitor`). Unchanged, /// The peer was past D. NoProc, @@ -334,6 +335,13 @@ pub(crate) fn register_link( if own.id == peer.id || Arc::ptr_eq(own_links, peer_links) { return LinkOutcome::Unchanged; } + // Sender lifetime (design Terms, semantics X2): `link` works only while + // the caller runs. Once its exit began (or past its D, checked again + // under the table lock below) the call is inert: no record, no notice, + // no NoProc. A dead process cannot call link in OTP. + if lock_request(&own.request).reason().is_some() { + return LinkOutcome::Unchanged; + } fail_point!("link.before_install"); let gen = NEXT_EDGE.fetch_add(1, Ordering::Relaxed); let outcome = with_pair(own_links, peer_links, |a, b| { @@ -720,6 +728,23 @@ mod tests { assert_eq!(r.runtime_drop(), ExitReason::Shutdown); } #[test] + fn link_from_an_exiting_or_dead_caller_is_inert() { + let seen = Arc::new(AtomicUsize::new(0)); + // Exit begun: no record on either side, even to a live peer. + let (a, at, b, bt) = pair(&seen); + lock_request(&a.request).request(ExitReason::Normal); + assert_eq!(register_link(&a, &at, &b, &bt, false), LinkOutcome::Unchanged); + assert!(at.lock().unwrap().entries.is_empty() && bt.lock().unwrap().entries.is_empty()); + // Past D, even to a dead peer: no NoProc notice, no try_link error. + let (c, ct, d, dt) = pair(&seen); + lock_request(&c.request).die(); + propagate_exit(c.id, &ct, &ExitReason::Normal); + propagate_exit(d.id, &dt, &ExitReason::Normal); + assert_eq!(register_link(&c, &ct, &d, &dt, false), LinkOutcome::Unchanged); + assert_eq!(register_link(&c, &ct, &d, &dt, true), LinkOutcome::Unchanged); + assert_eq!(seen.load(Ordering::SeqCst), 0); + } + #[test] fn self_link_is_noop() { let seen = Arc::new(AtomicUsize::new(0)); let a = Arc::new(target(&seen)); diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index e43365a6..7f7f0c30 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -688,7 +688,9 @@ impl Context { /// notice: an `Exit { reason: NoProc }` when trapping, otherwise it exits /// `NoProc` (decision 1). Use [`try_link`](Self::try_link) to get /// `Err(NoProc)` instead. An existing link, or a pending notice from that - /// peer, makes this a no-op. + /// peer, makes this a no-op. Inert once this actor's exit began (in + /// `stopped()`, from a helper after the stop, or past its death point): + /// no record and no notice, as `monitor`. /// /// [`Actor::exit_received`]: Actor::exit_received pub fn link(&self, target: &ChildHandle) { diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index a3590d09..6810cc56 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -585,7 +585,9 @@ impl Context { /// If the target is past its death point, this actor receives a `NoProc` /// notice: an `Exit { reason: NoProc }` when trapping, otherwise it exits /// `NoProc`. Use [`try_link`](Self::try_link) to get `Err(NoProc)` - /// instead. An existing link, or a pending notice from that peer, makes + /// instead. Inert once this actor's exit began (in `stopped()`, from a + /// helper after the stop, or past its death point): no record and no + /// notice, as `monitor`. An existing link, or a pending notice from that peer, makes /// this a no-op. /// /// [`Actor::exit_received`]: Actor::exit_received diff --git a/concurrency/tests/otp_exit_signals.rs b/concurrency/tests/otp_exit_signals.rs index dc3e28d7..7b2a60f9 100644 --- a/concurrency/tests/otp_exit_signals.rs +++ b/concurrency/tests/otp_exit_signals.rs @@ -314,6 +314,32 @@ mod tasks_mode { ended(&trier.child_handle()).await; } + // Sender lifetime / semantics X2: a Context kept by a helper after its + // actor exited cannot link: no record, no notice, no NoProc. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn link_from_a_dead_actor_is_inert() { + let (dead_seen, peer_seen) = (Seen::default(), Seen::default()); + let dead = Node::new(&dead_seen, true).start(); + let peer = Node::new(&peer_seen, true).start(); + settle(&peer).await; + let kept = dead.context(); + dead.child_handle().stop(); + ended(&dead.child_handle()).await; + kept.link(&peer.child_handle()); + assert_eq!(kept.try_link(&peer.child_handle()), Ok(())); + // The live peer gets no edge: its own exit notifies nobody. + let gone = Node::new(&Seen::default(), false).start(); + kept.link(&gone.child_handle()); + gone.child_handle().stop(); + ended(&gone.child_handle()).await; + peer.child_handle().exit(ExitReason::Panic("peer".into())); + settle(&peer).await; + peer.child_handle().stop_with_reason(ExitReason::Panic("peer".into())); + assert!(matches!(ended(&peer.child_handle()).await, ExitReason::Panic(_))); + assert!(dead_seen.exits().is_empty()); + assert_eq!(dead.child_handle().exit_reason(), Some(ExitReason::Normal)); + } + pub struct Parent { pub child: Option>, pub child_seen: Seen, diff --git a/docs/design/semantics.md b/docs/design/semantics.md index 50ec48f9..99dc9e41 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -243,7 +243,14 @@ Checked by: [`actor_scenarios::demonitor_before_exit_suppresses_notification_and **X2. Demonitor and late registration.** `demonitor` before the target exits suppresses `Down`; a `Down` already queued may still arrive. A monitor registered by an actor that is already stopping is inert (no waiter, no `Down`). +Likewise `ctx.link`/`ctx.try_link` called through a `Context` whose actor's +exit already began, or which is past its death point (for example from a +helper that outlives the actor), is inert: no record, no notice, no `NoProc`, +and `try_link` returns `Ok(())` (a dead process cannot call `link` in OTP; +design otp-exit-signals.md, "Sender lifetime"). Checked by: `src/*/actor.rs::demonitor_before_target_dies_suppresses_down`, +`link::tests::link_from_an_exiting_or_dead_caller_is_inert`, +[`otp_exit_signals::tasks_mode::link_from_a_dead_actor_is_inert`][otp_exit_signals], `closed_monitor_table_rejects_late_registration`, [`leaks::monitor_waiters_released_on_demonitor_owner_exit_and_target_exit`][leaks]. **X3. Link propagation.** When a linked actor exits: @@ -604,6 +611,7 @@ See the [ranked backlog](../../ROADMAP.md#ranked-backlog). [cohort_trace]: ../../concurrency/tests/cohort_trace.rs [context_exit]: ../../concurrency/tests/context_exit.rs [cross_backend]: ../../concurrency/tests/cross_backend.rs +[otp_exit_signals]: ../../concurrency/tests/otp_exit_signals.rs [delivery_simulation]: ../../concurrency/tests/delivery_simulation.rs [execution_groups]: ../../concurrency/tests/execution_groups.rs [exit_signal_matrix]: ../../concurrency/tests/exit_signal_matrix.rs From 56ce6379dbf4bf2bf6d08f8751ac8ebfddc35539 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:41:01 -0300 Subject: [PATCH 072/208] test(link): exiting caller's link is inert --- concurrency/src/link.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/concurrency/src/link.rs b/concurrency/src/link.rs index 5801dc2c..e3f73b83 100644 --- a/concurrency/src/link.rs +++ b/concurrency/src/link.rs @@ -645,11 +645,17 @@ mod tests { assert_eq!(seen.load(Ordering::SeqCst), 1); assert!(bt.lock().unwrap().entries.is_empty()); // b links to the dead a: a late NoProc notice, delivered and removed. - // b, already exiting, gets the late NoProc signal, which replaces - // its reason (decision 8). - assert_eq!(register_link(&b, &bt, &a, &at, false), LinkOutcome::NoProc); + // b is exiting (the notice began its exit): its link is inert + // (model: Link requires a running caller). + assert_eq!(register_link(&b, &bt, &a, &at, false), LinkOutcome::Unchanged); + assert_eq!(seen.load(Ordering::SeqCst), 1); + // A running c linking to the dead a gets the late NoProc notice. + let c = Arc::new(target(&seen)); + let ct = new_link_table(); + assert_eq!(register_link(&c, &ct, &a, &at, false), LinkOutcome::NoProc); assert_eq!(seen.load(Ordering::SeqCst), 2); - assert_eq!(lock_request(&b.request).reason(), Some(&ExitReason::NoProc)); + assert_eq!(lock_request(&c.request).reason(), Some(&ExitReason::NoProc)); + assert!(ct.lock().unwrap().entries.is_empty()); assert!(at.lock().unwrap().entries.is_empty() && bt.lock().unwrap().entries.is_empty()); } #[test] From 9360d45078b4f51d27abd69ffc78c74dc8f3ce0e Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:41:46 -0300 Subject: [PATCH 073/208] docs: CHANGELOG and MIGRATION for exit signals (in progress) --- CHANGELOG.md | 30 ++++++++++++++++++++++++ docs/MIGRATION.md | 60 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 365b84a6..1a809d50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,36 @@ ## Unreleased +- **Breaking (0.8), in progress:** exit signals, startup and runtime-drop + semantics as in OTP (ROADMAP ranks 5e, 5f, 5g), implementing + [the design](docs/design/otp-exit-signals.md) (#240, stacked on rank 11). + Done so far: `ExitReason` is `#[non_exhaustive]` with `Killed` and `NoProc`; + `ChildHandle::kill()` (untrappable, ends `Killed`, supersedes any reason + until the death point), `ChildHandle::exit(reason)` and + `Context::exit(&target, reason)` with `erlang:exit_signal/2` rules + (`Normal` ignored by a non-trapping actor, other reasons end it without + `stopped()`, trapping actors get an `Exit`), `ActorId::EXTERNAL`; + administrative `stop`/`stop_with_reason`/`shutdown` unchanged; one death + point after `stopped()` where the reason becomes final and the link table + closes; a late untrapped signal replaces the requested reason unless a kill + was received (decision 8); a panic in `stopped()` replaces an earlier + abnormal reason (except after a kill); `stopped()` is skipped on a kill, an + untrapped signal or link notice and a runtime drop; runtime-dropped actors + end `Killed` in both modes; `link`/`monitor` to an actor past its death + point give `NoProc` (a non-trapping caller of `link` exits `NoProc`; + `Context::try_link` returns `Err(NoProc)`); `link` from an actor whose exit + began is inert; `unlink`/`demonitor` are fences; a `start_linked` parent's + exit ends a trapping child through `stopped()` (never `exit_received`); + signals arriving during `started()` are resolved when it returns; + supervisor and system shutdown send the parent's `Shutdown` exit signal + and kill at `shutdown_timeout` (system: at grace); destructor panics after + the death point are cleanup failures (`ChildHandle::cleanup_failure()`, + `mailbox_cleanup_failure()` deprecated) that restart a Transient child + without changing the reason; `spawned-test` `faults::exit` sends an exact + exit signal; metrics `actors_exited_killed_total`, + `actors_exited_noproc_total`. Remaining: S1/S2 supervisor startup, rank 11 + reason-bearing `ActorStopped`/`NoProc` errors, TLA+ and trace work. + - **Breaking (0.8), in progress:** deferred replies and pipe-to-self (ROADMAP rank 11), implementing [the design](docs/design/deferred-replies.md). Every request now has a reply diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 4120bed4..b74edc19 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -5,6 +5,66 @@ every change; this guide covers what you need to edit when upgrading. ## Unreleased +### Exit signals, startup and runtime drops as in OTP (ranks 5e, 5f, 5g, targets 0.8, in progress) + +Design: [otp-exit-signals.md](design/otp-exit-signals.md) (its migration +table is the full list). What to edit: + +- **`match` on `ExitReason`**: add a wildcard arm (`#[non_exhaustive]`, new + `Killed` and `NoProc`). +- **Kill from outside**: `ChildHandle::kill()` is the untrappable kill; the + actor ends `Killed` and `stopped()` is skipped unless already running. + `stop_with_reason(Kill)` stays an administrative stop ending `Kill`. +- **`stopped()` no longer always runs.** It is skipped when the exit is begun + by a kill, an untrapped exit signal or link notice (a linked peer's + abnormal death), or a runtime drop. **This includes supervisor and system + shutdown of a non-trapping actor**: the supervisor's `Shutdown` is now the + parent's exit signal (OTP `exit(Child, shutdown)`), so a non-trapping child + ends `Shutdown` without `stopped()`, as a gen_server that does not trap + exits skips `terminate`. Cleanup that must run on every exit belongs in + `Drop`, or the actor calls `ctx.trap_exit(true)` in `started()`: a trapping + child processes the messages queued before the shutdown, then runs + `stopped()`. A child that has not reached its death point within + `shutdown_timeout` is killed; cleanup is stuck only after a second + `shutdown_timeout`. +- **Runtime drop** reports `Killed` (was `Kill` in tasks mode, + `Panic("unexpected framework panic")` in threads mode). +- **Late link or monitor**: to an actor past its death point, `monitor` + delivers `Down { reason: NoProc }` and `link` delivers `NoProc` (a + non-trapping caller exits `NoProc`). Use `ctx.try_link` to get + `Err(NoProc)` instead, and `ChildHandle::exit_reason()` for the real + reason. `link` from an actor whose exit already began is inert. +- **A late signal during exit**: a non-`Normal` link notice or exit signal + that reaches a non-trapping actor while it exits (for example during its + `stopped()`) now replaces its reason. The regression test + `late_link_failure_cannot_relabel_an_already_requested_normal_stop` was + inverted deliberately into + `late_link_failure_relabels_an_already_requested_normal_stop`. +- **Panic in `stopped()`** after an abnormal reason now becomes the reason + (`Panic("panic in stopped(): ..")`), except after a kill. +- **Destructor panics** (actor state, queued messages) after the death point + no longer change the exit reason: a `Normal` exit stays `Normal` and + `ChildHandle::cleanup_failure()` reports the failure + (`mailbox_cleanup_failure()` is a deprecated alias). A Transient child + restarts on an abnormal reason **or** a cleanup failure, also after + `Shutdown`; `ChildSnapshot::last_error` holds the failure. +- **`start_linked` children that trap exits** end when their parent ends, + with the parent's reason (`Normal` included), through `stopped()`; + `exit_received` no longer sees the parent's exit. Use `ctx.link` instead + of `start_linked` to observe the parent's exit as an `Exit`. +- **`spawned-test` `faults::exit`** now sends the exact exit signal + (`ChildHandle::exit`): `Normal` is ignored by a non-trapping target, + `Panic(msg)` ends it with exactly `Panic(msg)`, `Kill` ends it `Killed`, a + trapping target receives an `Exit`. +- **Supervisor start (rank 5f, not implemented yet in this branch):** any + first-start failure of a child will fail `SupervisorBuilder::start()`, + whatever its restart type, with no opt-in retry (decision 7). **ethrex**: + the Transient discovery child's factory (`DiscoveryServer::build`) can fail + at boot when pinging a bootnode fails; v0.7 retried it, v0.8 fails + `start()` and `init_network` panics on `.expect("Network starts")`. Fix in + ethrex: make the bootnode ping best-effort (log and continue) so the + factory does not fail on it, or retry `start()` at application level. + ### Deferred replies (rank 11, targets 0.8, in progress) This is a breaking release. The notes below grow with the implementation: From ea8b838f9e6781ef4d471512edf632fb4ddef4ca Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:45:55 -0300 Subject: [PATCH 074/208] docs: supervisor/system E3 reverted pending test adaptation --- CHANGELOG.md | 6 ++--- concurrency/src/supervision.rs | 36 +++------------------------ concurrency/src/system.rs | 45 +++------------------------------- docs/MIGRATION.md | 6 ++--- 4 files changed, 13 insertions(+), 80 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a809d50..ba494e88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,13 +23,13 @@ began is inert; `unlink`/`demonitor` are fences; a `start_linked` parent's exit ends a trapping child through `stopped()` (never `exit_received`); signals arriving during `started()` are resolved when it returns; - supervisor and system shutdown send the parent's `Shutdown` exit signal - and kill at `shutdown_timeout` (system: at grace); destructor panics after + destructor panics after the death point are cleanup failures (`ChildHandle::cleanup_failure()`, `mailbox_cleanup_failure()` deprecated) that restart a Transient child without changing the reason; `spawned-test` `faults::exit` sends an exact exit signal; metrics `actors_exited_killed_total`, - `actors_exited_noproc_total`. Remaining: S1/S2 supervisor startup, rank 11 + `actors_exited_noproc_total`. Remaining: supervisor/system shutdown as the parent's exit signal + with the kill escalation (E3), S1/S2 supervisor startup, rank 11 reason-bearing `ActorStopped`/`NoProc` errors, TLA+ and trace work. - **Breaking (0.8), in progress:** deferred replies and pipe-to-self diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs index 96e7ddf8..a43a1b8c 100644 --- a/concurrency/src/supervision.rs +++ b/concurrency/src/supervision.rs @@ -523,13 +523,11 @@ impl ChildControl for View { self.changed.send_modify(|n| *n = n.wrapping_add(1)); true } - /// Collateral stop (OneForAll/RestForOne): the supervisor's parent exit - /// signal (E3); the controller escalates to a kill at `shutdown_timeout`. fn stop_generation(&self, slot: &RecoverySlot) { slot.stop.cancel(); let scope = system::lock(&self.data).scope.clone(); if let Some(scope) = scope { - scope.signal_parent(ExitReason::Shutdown); + scope.stop(ExitReason::Shutdown); } } fn snapshot(&self) -> ChildSnapshot { @@ -548,15 +546,10 @@ impl ChildControl for View { } self.changed.send_modify(|n| *n = n.wrapping_add(1)); } - /// Supervisor shutdown of this child (V9, reverse order): the parent - /// exit signal `exit(Child, shutdown)` (E3). A non-trapping child ends - /// `Shutdown` without `stopped()`; a trapping child finishes its earlier - /// messages and runs `stopped()`. The controller kills it at - /// `shutdown_timeout`. fn stop(&self) { self.stop_turn.cancel(); if let Some(scope) = system::lock(&self.data).scope.clone() { - scope.signal_parent(ExitReason::Shutdown); + scope.stop(ExitReason::Shutdown); } } fn done(&self) -> watch::Receiver { @@ -2058,7 +2051,7 @@ async fn run_child(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc { - scope.kill(); - // The second interval runs from the kill. - cleanup_until = Some(Instant::now()); - match rt::timeout(spec.shutdown_timeout, child.wait_exit_async()) - .await - { - Ok(reason) => reason, - Err(_) => { - stuck = true; - core.fail_cleanup_for( - view.snapshot(), - format!("actor cleanup stuck for {}", spec.name), - || Some(Event::new("CleanupStuck", c, "")), - ); - child.wait_exit_async().await - } - } - } Err(_) => { if !stop_turn { trace::emit(core.trace, || { diff --git a/concurrency/src/system.rs b/concurrency/src/system.rs index 21948284..2bd2ca27 100644 --- a/concurrency/src/system.rs +++ b/concurrency/src/system.rs @@ -524,19 +524,16 @@ impl SystemInner { } } ShutdownTarget::Scope(scope) => { - // The system is a standalone actor's parent: its - // Shutdown is a parent exit signal (E3). scope.set_cleanup_timeout(remaining); - scope.signal_parent(ExitReason::Shutdown); + scope.stop(ExitReason::Shutdown); let _ = rt::timeout(remaining, scope.joined()).await; } } } - // At grace expiry every remaining actor gets the kill (E3), so - // no nested child remains unsignalled behind a stuck sibling's - // longer per-child cleanup timeout. + // At grace expiry, no nested child may remain unsignalled behind + // a stuck sibling's longer per-child cleanup timeout. for scope in scopes.iter().rev() { - scope.kill(); + scope.stop(ExitReason::Shutdown); } // Runtime teardown still requires actual completion, independently // of the ordering grace and the caller's report timeout. @@ -986,40 +983,6 @@ impl Scope { pub(crate) fn close_admission(&self) { lock(&self.state).open = false; } - /// The parent's exit signal to this scope's actor (design E3): the - /// supervisor or system is the parent. Admission closes for the managed - /// tree at once (B), but nothing is committed for the actor: a - /// non-trapping actor begins exit with `reason` without `stopped()` - /// (`Normal` is ignored), a trapping one gets a parent exit queued behind - /// its messages, which ends it through `stopped()`. Before the actor is - /// attached this is an ordinary stop. - pub(crate) fn signal_parent(&self, reason: ExitReason) { - let child = { - let mut state = lock(&self.state); - state.open = false; - if state.ended_at.is_none() { - state.ended_at = Some(Instant::now()); - } - state.actor.clone() - }; - match child { - Some(child) => child.exit_from(crate::ActorId::EXTERNAL, reason, true), - None => self.stop(reason), - } - } - /// The parent's kill (E3 escalation at `shutdown_timeout` or grace): the - /// actor ends `Killed`, skipping a `stopped()` that has not begun. - pub(crate) fn kill(&self) { - let child = { - let mut state = lock(&self.state); - state.open = false; - state.actor.clone() - }; - match child { - Some(child) => child.exit_from(crate::ActorId::EXTERNAL, ExitReason::Kill, true), - None => self.stop(ExitReason::Killed), - } - } pub(crate) fn register( &self, name: &str, diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index b74edc19..f71cf7da 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -17,9 +17,9 @@ table is the full list). What to edit: `stop_with_reason(Kill)` stays an administrative stop ending `Kill`. - **`stopped()` no longer always runs.** It is skipped when the exit is begun by a kill, an untrapped exit signal or link notice (a linked peer's - abnormal death), or a runtime drop. **This includes supervisor and system - shutdown of a non-trapping actor**: the supervisor's `Shutdown` is now the - parent's exit signal (OTP `exit(Child, shutdown)`), so a non-trapping child + abnormal death), or a runtime drop. **Once E3 lands (not yet in this + branch) this includes supervisor and system shutdown of a non-trapping + actor**: the supervisor's `Shutdown` becomes the parent's exit signal (OTP `exit(Child, shutdown)`), so a non-trapping child ends `Shutdown` without `stopped()`, as a gen_server that does not trap exits skips `terminate`. Cleanup that must run on every exit belongs in `Drop`, or the actor calls `ctx.trap_exit(true)` in `started()`: a trapping From 2ca44baa1988acf88c3cfd4fb540c1aec500befe Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 03:46:38 +0000 Subject: [PATCH 075/208] style: cargo fmt --- concurrency/src/child_handle.rs | 9 +- concurrency/src/error.rs | 7 +- concurrency/src/link.rs | 45 ++++++-- concurrency/src/supervision.rs | 4 +- concurrency/src/system.rs | 6 +- concurrency/src/tasks/actor.rs | 22 ++-- concurrency/src/threads/actor.rs | 34 +++--- .../actor_scenarios.proptest-regressions | 7 ++ .../tests/cross_backend.proptest-regressions | 7 ++ concurrency/tests/mailbox_drop_panic.rs | 100 ++++++++++++++---- concurrency/tests/otp_exit_signals.rs | 93 ++++++++++++---- concurrency/tests/panic_safety.rs | 3 +- testing/tests/tasks_kit.rs | 5 +- 13 files changed, 246 insertions(+), 96 deletions(-) create mode 100644 concurrency/tests/actor_scenarios.proptest-regressions create mode 100644 concurrency/tests/cross_backend.proptest-regressions diff --git a/concurrency/src/child_handle.rs b/concurrency/src/child_handle.rs index fcf8ce8b..0a46e3c1 100644 --- a/concurrency/src/child_handle.rs +++ b/concurrency/src/child_handle.rs @@ -88,9 +88,7 @@ impl ChildHandle { pub(crate) fn new(target: Arc, completion: Completion, links: LinkTable) -> Self { let stop = target.clone(); Self { - cancel: Arc::new(move || { - crate::link::normal_stop(&stop.request, || (stop.wake)()) - }), + cancel: Arc::new(move || crate::link::normal_stop(&stop.request, || (stop.wake)())), target, completion, links, @@ -370,7 +368,10 @@ pub(crate) mod tests { use crate::link::{new_link_table, new_linked_exit_reason, new_trap_exit_flag}; /// A signal target whose wake cancels `token`. - pub(crate) fn test_target(id: ActorId, token: spawned_rt::threads::CancellationToken) -> Arc { + pub(crate) fn test_target( + id: ActorId, + token: spawned_rt::threads::CancellationToken, + ) -> Arc { Arc::new(SignalTarget { id, parent: None, diff --git a/concurrency/src/error.rs b/concurrency/src/error.rs index a3c59a55..7852978f 100644 --- a/concurrency/src/error.rs +++ b/concurrency/src/error.rs @@ -34,10 +34,9 @@ impl ExitReason { pub fn is_abnormal(&self) -> bool { match self { ExitReason::Normal | ExitReason::Shutdown => false, - ExitReason::Panic(_) - | ExitReason::Kill - | ExitReason::Killed - | ExitReason::NoProc => true, + ExitReason::Panic(_) | ExitReason::Kill | ExitReason::Killed | ExitReason::NoProc => { + true + } } } } diff --git a/concurrency/src/link.rs b/concurrency/src/link.rs index e3f73b83..af4f1674 100644 --- a/concurrency/src/link.rs +++ b/concurrency/src/link.rs @@ -627,7 +627,9 @@ mod tests { send_held: Arc::new(|_, _| Ok(())), } } - fn pair(seen: &Arc) -> (Arc, LinkTable, Arc, LinkTable) { + fn pair( + seen: &Arc, + ) -> (Arc, LinkTable, Arc, LinkTable) { ( Arc::new(target(seen)), new_link_table(), @@ -647,7 +649,10 @@ mod tests { // b links to the dead a: a late NoProc notice, delivered and removed. // b is exiting (the notice began its exit): its link is inert // (model: Link requires a running caller). - assert_eq!(register_link(&b, &bt, &a, &at, false), LinkOutcome::Unchanged); + assert_eq!( + register_link(&b, &bt, &a, &at, false), + LinkOutcome::Unchanged + ); assert_eq!(seen.load(Ordering::SeqCst), 1); // A running c linking to the dead a gets the late NoProc notice. let c = Arc::new(target(&seen)); @@ -676,12 +681,19 @@ mod tests { let notices = close_links(a.id, &at, &ExitReason::Panic("x".into())); unregister_link(b.id, &bt, a.id, &at); notices.deliver(); - assert_eq!(seen.load(Ordering::SeqCst), 0, "unlink returned before delivery"); + assert_eq!( + seen.load(Ordering::SeqCst), + 0, + "unlink returned before delivery" + ); // A relink while the notice is pending is a no-op, as an existing record. let (c, ct, d, dt) = pair(&seen); register_link(&c, &ct, &d, &dt, false); let notices = close_links(c.id, &ct, &ExitReason::Panic("y".into())); - assert_eq!(register_link(&d, &dt, &c, &ct, false), LinkOutcome::Unchanged); + assert_eq!( + register_link(&d, &dt, &c, &ct, false), + LinkOutcome::Unchanged + ); notices.deliver(); assert_eq!(seen.load(Ordering::SeqCst), 1); } @@ -695,7 +707,10 @@ mod tests { assert_eq!(lock_request(&t.request).reason(), None); // Any other reason begins exit without stopped(). deliver_signal(&t, from, ExitReason::Shutdown, Origin::Direct, false); - assert_eq!(lock_request(&t.request).reason(), Some(&ExitReason::Shutdown)); + assert_eq!( + lock_request(&t.request).reason(), + Some(&ExitReason::Shutdown) + ); assert!(!lock_request(&t.request).begin_hook()); // Trapping: queued as Exit, nothing committed. let t = target(&seen); @@ -716,7 +731,10 @@ mod tests { assert!(r.request(ExitReason::Normal)); assert!(r.untrapped(ExitReason::Shutdown)); assert_eq!(r.reason(), Some(&ExitReason::Shutdown)); - assert!(r.begin_hook(), "a late signal does not skip a self-exit's stopped()"); + assert!( + r.begin_hook(), + "a late signal does not skip a self-exit's stopped()" + ); r.panic(ExitReason::Panic("stopped".into())); assert_eq!(r.reason(), Some(&ExitReason::Panic("stopped".into()))); assert!(r.kill()); @@ -739,15 +757,24 @@ mod tests { // Exit begun: no record on either side, even to a live peer. let (a, at, b, bt) = pair(&seen); lock_request(&a.request).request(ExitReason::Normal); - assert_eq!(register_link(&a, &at, &b, &bt, false), LinkOutcome::Unchanged); + assert_eq!( + register_link(&a, &at, &b, &bt, false), + LinkOutcome::Unchanged + ); assert!(at.lock().unwrap().entries.is_empty() && bt.lock().unwrap().entries.is_empty()); // Past D, even to a dead peer: no NoProc notice, no try_link error. let (c, ct, d, dt) = pair(&seen); lock_request(&c.request).die(); propagate_exit(c.id, &ct, &ExitReason::Normal); propagate_exit(d.id, &dt, &ExitReason::Normal); - assert_eq!(register_link(&c, &ct, &d, &dt, false), LinkOutcome::Unchanged); - assert_eq!(register_link(&c, &ct, &d, &dt, true), LinkOutcome::Unchanged); + assert_eq!( + register_link(&c, &ct, &d, &dt, false), + LinkOutcome::Unchanged + ); + assert_eq!( + register_link(&c, &ct, &d, &dt, true), + LinkOutcome::Unchanged + ); assert_eq!(seen.load(Ordering::SeqCst), 0); } #[test] diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs index a43a1b8c..8ed2c705 100644 --- a/concurrency/src/supervision.rs +++ b/concurrency/src/supervision.rs @@ -44,7 +44,9 @@ impl RestartMode { fn restarts(self, reason: Option<&ExitReason>, cleanup_failed: bool) -> bool { match self { Self::Permanent => true, - Self::Transient => cleanup_failed || reason.map(ExitReason::is_abnormal).unwrap_or(true), + Self::Transient => { + cleanup_failed || reason.map(ExitReason::is_abnormal).unwrap_or(true) + } Self::Temporary => false, } } diff --git a/concurrency/src/system.rs b/concurrency/src/system.rs index 2bd2ca27..3ab6b777 100644 --- a/concurrency/src/system.rs +++ b/concurrency/src/system.rs @@ -1273,7 +1273,11 @@ impl Drop for Lease { .scope .child() .and_then(|c| c.exit_reason()) - .or_else(|| crate::link::lock_request(&self.scope.cause).reason().cloned()) + .or_else(|| { + crate::link::lock_request(&self.scope.cause) + .reason() + .cloned() + }) .unwrap_or(ExitReason::Killed); self.scope.ready.send_replace(StartupState::Failed(reason)); } diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 7f7f0c30..97981a14 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -1,10 +1,10 @@ -use crate::child_handle::{ActorId, ChildHandle}; -use crate::error::{panic_message, ActorError, ExitReason}; use crate::child_handle::Completion; +use crate::child_handle::{ActorId, ChildHandle}; use crate::error::NoProc; +use crate::error::{panic_message, ActorError, ExitReason}; use crate::link::{ - self, new_link_table, new_linked_exit_reason, new_trap_exit_flag, Exit, LinkOutcome, - LinkTable, LinkedExitReason, SendExitFn, SendHeldFn, SignalTarget, TrapExitFlag, + self, new_link_table, new_linked_exit_reason, new_trap_exit_flag, Exit, LinkOutcome, LinkTable, + LinkedExitReason, SendExitFn, SendHeldFn, SignalTarget, TrapExitFlag, }; use crate::mailbox::tasks as mpsc; use crate::message::Message; @@ -343,7 +343,9 @@ fn signal_target( }); let send_parent_exit: SendExitFn = Arc::new(move |exit: Exit| { let envelope: Box + Send> = Box::new(ParentExitEnvelope { exit }); - parent_tx.send(envelope).map_err(|_| ActorError::ActorStopped) + parent_tx + .send(envelope) + .map_err(|_| ActorError::ActorStopped) }); let send_held: SendHeldFn = Arc::new(move |exit: Exit, parent: bool| { let envelope: Box + Send> = Box::new(HeldEnvelope { exit, parent }); @@ -1688,14 +1690,8 @@ fn test_context( }), }; let noop: fn() = || {}; - let cleanup = crate::actor_cleanup::ActorCleanup::new( - id, - links, - request, - monitors, - completion_tx, - noop, - ); + let cleanup = + crate::actor_cleanup::ActorCleanup::new(id, links, request, monitors, completion_tx, noop); (ctx, cleanup) } diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 6810cc56..694cbafe 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -11,8 +11,8 @@ use std::{ use crate::child_handle::{ActorId, ChildHandle, Completion}; use crate::error::{panic_message, ActorError, ExitReason, NoProc}; use crate::link::{ - self, new_link_table, new_linked_exit_reason, new_trap_exit_flag, Exit, LinkOutcome, - LinkTable, LinkedExitReason, SendExitFn, SendHeldFn, SignalTarget, TrapExitFlag, + self, new_link_table, new_linked_exit_reason, new_trap_exit_flag, Exit, LinkOutcome, LinkTable, + LinkedExitReason, SendExitFn, SendHeldFn, SignalTarget, TrapExitFlag, }; use crate::message::Message; use crate::monitor::{Down, MonitorCleanup, MonitorRef, MonitorTable}; @@ -2043,14 +2043,7 @@ mod idle_stop_regression { trap_exit: trap.clone(), links: new_link_table(), linked_reason: request.clone(), - signal: signal_target::<_>( - id, - None, - &tx, - trap.clone(), - request.clone(), - token.clone(), - ), + signal: signal_target::<_>(id, None, &tx, trap.clone(), request.clone(), token.clone()), mailbox: crate::instrumentation::Mailbox::default(), replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), @@ -2093,7 +2086,11 @@ mod idle_stop_regression { token.cancel(); release.wait(); actor.join().unwrap(); - assert_eq!(link::lock_request(&request).reason(), None, "a plain stop request"); + assert_eq!( + link::lock_request(&request).reason(), + None, + "a plain stop request" + ); assert_eq!( count.load(Ordering::SeqCst), 0, @@ -2181,14 +2178,7 @@ mod stop_wakeup_regression { trap_exit: trap.clone(), links: new_link_table(), linked_reason: request.clone(), - signal: signal_target::<_>( - id, - None, - &tx, - trap.clone(), - request.clone(), - token.clone(), - ), + signal: signal_target::<_>(id, None, &tx, trap.clone(), request.clone(), token.clone()), mailbox: crate::instrumentation::Mailbox::default(), replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), @@ -2215,7 +2205,11 @@ mod stop_wakeup_regression { .recv_timeout(Duration::from_secs(2)) .expect("cancel did not wake recv"); actor.join().unwrap(); - assert_eq!(link::lock_request(&request).reason(), None, "a plain stop request"); + assert_eq!( + link::lock_request(&request).reason(), + None, + "a plain stop request" + ); } struct Crash; diff --git a/concurrency/tests/actor_scenarios.proptest-regressions b/concurrency/tests/actor_scenarios.proptest-regressions new file mode 100644 index 00000000..4b880b83 --- /dev/null +++ b/concurrency/tests/actor_scenarios.proptest-regressions @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc ff1196c6c1795d0ebaecb9a90df2fc2bae945dc1b58db213e805227cbc55510c # shrinks to count = 4, operations = [(11, 21)], endings = [0, 2, 1, 2, 1] diff --git a/concurrency/tests/cross_backend.proptest-regressions b/concurrency/tests/cross_backend.proptest-regressions new file mode 100644 index 00000000..c10357e7 --- /dev/null +++ b/concurrency/tests/cross_backend.proptest-regressions @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 7fd0a8196070dc964f5995916f8f5422ab5c0049bb709a8c539c63e90997041c # shrinks to endings = [Stop], double = [false] diff --git a/concurrency/tests/mailbox_drop_panic.rs b/concurrency/tests/mailbox_drop_panic.rs index e5ee8b5f..5677bbfc 100644 --- a/concurrency/tests/mailbox_drop_panic.rs +++ b/concurrency/tests/mailbox_drop_panic.rs @@ -242,31 +242,47 @@ fn abandoned_receiver_child() { let (entered, active) = oneshot::channel(); let (hold, gate) = oneshot::channel(); let owner = std::thread::spawn(move || { - let mut builder = if polled { tokio::runtime::Builder::new_multi_thread() } - else { tokio::runtime::Builder::new_current_thread() }; + let mut builder = if polled { + tokio::runtime::Builder::new_multi_thread() + } else { + tokio::runtime::Builder::new_current_thread() + }; let runtime = builder.worker_threads(1).enable_all().build().unwrap(); { let _entered = runtime.enter(); let actor = Victim(Arc::default(), state_panics).start(); - if polled { actor.send(Hold(entered, gate, false)).unwrap(); } + if polled { + actor.send(Hold(entered, gate, false)).unwrap(); + } send.send(actor).unwrap(); } stopped.recv_timeout(LIMIT).unwrap(); runtime.shutdown_timeout(LIMIT); }); let actor = tokio::time::timeout(LIMIT, arrival).await.unwrap().unwrap(); - if polled { tokio::time::timeout(LIMIT, active).await.unwrap().unwrap(); } + if polled { + tokio::time::timeout(LIMIT, active).await.unwrap().unwrap(); + } let dropped = Arc::new(AtomicUsize::new(0)); - let replies: Vec<_> = (0..2).map(|_| actor.request_raw(Bomb(dropped.clone())).unwrap()).collect(); + let replies: Vec<_> = (0..2) + .map(|_| actor.request_raw(Bomb(dropped.clone())).unwrap()) + .collect(); shutdown.send(()).unwrap(); - let reason = tokio::time::timeout(LIMIT, actor.wait_exit()).await.unwrap(); + let reason = tokio::time::timeout(LIMIT, actor.wait_exit()) + .await + .unwrap(); // A runtime drop ends Killed (G1); destructor panics are // cleanup failures, never the reason. let _ = state_panics; assert_eq!(reason, ExitReason::Killed); assert_eq!(dropped.load(Ordering::SeqCst), 2); - assert_eq!(actor.child_handle().cleanup_failure().as_deref(), Some("queued message destructor")); - for reply in replies { assert!(tokio::time::timeout(LIMIT, reply).await.unwrap().is_err()); } + assert_eq!( + actor.child_handle().cleanup_failure().as_deref(), + Some("queued message destructor") + ); + for reply in replies { + assert!(tokio::time::timeout(LIMIT, reply).await.unwrap().is_err()); + } drop(hold); owner.join().unwrap(); } @@ -274,40 +290,72 @@ fn abandoned_receiver_child() { // A committed stop on a never-polled startup must also survive a // panicking actor-state destructor and two queued destructor failures. for committed in [ExitReason::Shutdown, ExitReason::Kill] { - let owner = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + let owner = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); let actor = { let _entered = owner.enter(); Victim(Arc::default(), true).start() }; let dropped = Arc::new(AtomicUsize::new(0)); - for _ in 0..2 { actor.send(Bomb(dropped.clone())).unwrap(); } + for _ in 0..2 { + actor.send(Bomb(dropped.clone())).unwrap(); + } actor.context().stop_with_reason(committed.clone()); owner.shutdown_background(); - assert_eq!(tokio::time::timeout(LIMIT, actor.wait_exit()).await.unwrap(), committed); + assert_eq!( + tokio::time::timeout(LIMIT, actor.wait_exit()) + .await + .unwrap(), + committed + ); assert_eq!(dropped.load(Ordering::SeqCst), 2); } - for (managed, state_panics) in [(false, false), (false, true), (true, false), (true, true)] { - for committed in [ExitReason::Shutdown, ExitReason::Kill, ExitReason::Panic("linked peer cause".into())] { + for (managed, state_panics) in [(false, false), (false, true), (true, false), (true, true)] + { + for committed in [ + ExitReason::Shutdown, + ExitReason::Kill, + ExitReason::Panic("linked peer cause".into()), + ] { let system = ActorSystem::new().unwrap(); let group = system.async_group("preserve-cause", 1).unwrap(); - let actor = if managed { Victim(Arc::default(), state_panics).start_on(&group).unwrap() } - else { Victim(Arc::default(), state_panics).start() }; + let actor = if managed { + Victim(Arc::default(), state_panics) + .start_on(&group) + .unwrap() + } else { + Victim(Arc::default(), state_panics).start() + }; let (entered, arrival) = oneshot::channel(); let (release, gate) = oneshot::channel(); actor.send(Hold(entered, gate, false)).unwrap(); tokio::time::timeout(LIMIT, arrival).await.unwrap().unwrap(); let dropped = Arc::new(AtomicUsize::new(0)); - for _ in 0..2 { actor.send(Bomb(dropped.clone())).unwrap(); } + for _ in 0..2 { + actor.send(Bomb(dropped.clone())).unwrap(); + } if matches!(committed, ExitReason::Panic(_)) { let peer = Victim(Arc::default(), false).start(); actor.context().link(&peer.child_handle()); peer.context().stop_with_reason(committed.clone()); tokio::time::timeout(LIMIT, peer.wait_exit()).await.unwrap(); - } else { actor.context().stop_with_reason(committed.clone()); } + } else { + actor.context().stop_with_reason(committed.clone()); + } release.send(()).unwrap(); - assert_eq!(tokio::time::timeout(LIMIT, actor.wait_exit()).await.unwrap(), committed); + assert_eq!( + tokio::time::timeout(LIMIT, actor.wait_exit()) + .await + .unwrap(), + committed + ); assert_eq!(dropped.load(Ordering::SeqCst), 2); - assert_eq!(actor.child_handle().cleanup_failure().as_deref(), Some("queued message destructor")); + assert_eq!( + actor.child_handle().cleanup_failure().as_deref(), + Some("queued message destructor") + ); assert!(system.shutdown(LIMIT).await.complete); } } @@ -319,9 +367,17 @@ fn abandoned_receiver_child() { actor.send(Hold(entered, gate, false)).unwrap(); tokio::time::timeout(LIMIT, arrival).await.unwrap().unwrap(); let dropped = Arc::new(AtomicUsize::new(0)); - for _ in 0..2 { actor.send(Bomb(dropped.clone())).unwrap(); } - actor.context().stop(); release.send(()).unwrap(); - assert_eq!(tokio::time::timeout(LIMIT, actor.wait_exit()).await.unwrap(), ExitReason::Normal); + for _ in 0..2 { + actor.send(Bomb(dropped.clone())).unwrap(); + } + actor.context().stop(); + release.send(()).unwrap(); + assert_eq!( + tokio::time::timeout(LIMIT, actor.wait_exit()) + .await + .unwrap(), + ExitReason::Normal + ); assert!(actor.child_handle().cleanup_failure().is_some()); assert_eq!(dropped.load(Ordering::SeqCst), 2); }); diff --git a/concurrency/tests/otp_exit_signals.rs b/concurrency/tests/otp_exit_signals.rs index 7b2a60f9..185273fb 100644 --- a/concurrency/tests/otp_exit_signals.rs +++ b/concurrency/tests/otp_exit_signals.rs @@ -48,7 +48,10 @@ mod tasks_mode { pub seen: Seen, pub trap: bool, /// stopped() waits for this signal (to act while it runs). - pub in_stopped: Option<(tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>)>, + pub in_stopped: Option<( + tokio::sync::oneshot::Sender<()>, + tokio::sync::oneshot::Receiver<()>, + )>, pub panic_in_stopped: bool, } impl Node { @@ -135,8 +138,12 @@ mod tasks_mode { // links receive Killed (trappable), monitors Down { Killed }. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn kill_is_untrappable_and_propagates_killed() { - let (target_seen, peer_seen, watcher_seen, trapper_seen) = - (Seen::default(), Seen::default(), Seen::default(), Seen::default()); + let (target_seen, peer_seen, watcher_seen, trapper_seen) = ( + Seen::default(), + Seen::default(), + Seen::default(), + Seen::default(), + ); let target = Node::new(&target_seen, true).start(); let peer = Node::new(&peer_seen, false).start(); let trapper = Node::new(&trapper_seen, true).start(); @@ -164,7 +171,10 @@ mod tasks_mode { settle(&trapper).await; assert_eq!( trapper_seen.exits(), - vec![Exit { from: t.id(), reason: ExitReason::Killed }] + vec![Exit { + from: t.id(), + reason: ExitReason::Killed + }] ); settle(&watcher).await; assert_eq!(watcher_seen.downs().len(), 1); @@ -218,7 +228,10 @@ mod tasks_mode { settle(&plain).await; plain.child_handle().exit(ExitReason::Normal); settle(&plain).await; - assert!(plain.child_handle().is_alive(), "Normal stopped a non-trapping actor"); + assert!( + plain.child_handle().is_alive(), + "Normal stopped a non-trapping actor" + ); let trap_seen = Seen::default(); let trapper = Node::new(&trap_seen, true).start(); @@ -229,15 +242,24 @@ mod tasks_mode { assert_eq!( trap_seen.exits(), vec![ - Exit { from: ActorId::EXTERNAL, reason: ExitReason::Normal }, - Exit { from: ActorId::EXTERNAL, reason: ExitReason::Shutdown }, + Exit { + from: ActorId::EXTERNAL, + reason: ExitReason::Normal + }, + Exit { + from: ActorId::EXTERNAL, + reason: ExitReason::Shutdown + }, ] ); assert!(trapper.child_handle().is_alive()); plain.child_handle().exit(ExitReason::Shutdown); assert_eq!(ended(&plain.child_handle()).await, ExitReason::Shutdown); - assert!(!plain_seen.stopped(), "stopped() ran after an untrapped signal"); + assert!( + !plain_seen.stopped(), + "stopped() ran after an untrapped signal" + ); trapper.child_handle().stop(); ended(&trapper.child_handle()).await; } @@ -251,11 +273,17 @@ mod tasks_mode { settle(&b).await; let bh = b.child_handle(); let bh2 = bh.clone(); - on(&a, move |ctx| ctx.exit(&bh2, ExitReason::Panic("from a".into()))).await; + on(&a, move |ctx| { + ctx.exit(&bh2, ExitReason::Panic("from a".into())) + }) + .await; settle(&b).await; assert_eq!( b_seen.exits(), - vec![Exit { from: a.id(), reason: ExitReason::Panic("from a".into()) }] + vec![Exit { + from: a.id(), + reason: ExitReason::Panic("from a".into()) + }] ); let bh3 = bh.clone(); on(&a, move |ctx| ctx.exit(&bh3, ExitReason::Kill)).await; @@ -271,8 +299,12 @@ mod tasks_mode { async fn registrations_after_death_report_noproc() { let dead_seen = Seen::default(); let dead = Node::new(&dead_seen, false).start(); - dead.child_handle().stop_with_reason(ExitReason::Panic("gone".into())); - assert!(matches!(ended(&dead.child_handle()).await, ExitReason::Panic(_))); + dead.child_handle() + .stop_with_reason(ExitReason::Panic("gone".into())); + assert!(matches!( + ended(&dead.child_handle()).await, + ExitReason::Panic(_) + )); let d = dead.child_handle(); let trap_seen = Seen::default(); @@ -286,7 +318,10 @@ mod tasks_mode { settle(&trapper).await; assert_eq!( trap_seen.exits(), - vec![Exit { from: d.id(), reason: ExitReason::NoProc }] + vec![Exit { + from: d.id(), + reason: ExitReason::NoProc + }] ); let downs = trap_seen.downs(); assert_eq!(downs.len(), 1); @@ -296,7 +331,10 @@ mod tasks_mode { let trier = Node::new(&try_seen, false).start(); let result = Arc::new(Mutex::new(None)); let (d3, r) = (d.clone(), result.clone()); - on(&trier, move |ctx| *r.lock().unwrap() = Some(ctx.try_link(&d3))).await; + on(&trier, move |ctx| { + *r.lock().unwrap() = Some(ctx.try_link(&d3)) + }) + .await; assert_eq!(*result.lock().unwrap(), Some(Err(NoProc))); settle(&trier).await; assert!(trier.child_handle().is_alive()); @@ -334,8 +372,12 @@ mod tasks_mode { ended(&gone.child_handle()).await; peer.child_handle().exit(ExitReason::Panic("peer".into())); settle(&peer).await; - peer.child_handle().stop_with_reason(ExitReason::Panic("peer".into())); - assert!(matches!(ended(&peer.child_handle()).await, ExitReason::Panic(_))); + peer.child_handle() + .stop_with_reason(ExitReason::Panic("peer".into())); + assert!(matches!( + ended(&peer.child_handle()).await, + ExitReason::Panic(_) + )); assert!(dead_seen.exits().is_empty()); assert_eq!(dead.child_handle().exit_reason(), Some(ExitReason::Normal)); } @@ -374,10 +416,15 @@ mod tasks_mode { } .start(); let child = parent.request(Spawn).await.unwrap(); - parent.child_handle().stop_with_reason(parent_reason.clone()); + parent + .child_handle() + .stop_with_reason(parent_reason.clone()); assert_eq!(ended(&child).await, parent_reason); assert!(child_seen.stopped(), "the parent exit skipped stopped()"); - assert!(child_seen.exits().is_empty(), "exit_received saw the parent exit"); + assert!( + child_seen.exits().is_empty(), + "exit_received saw the parent exit" + ); } } @@ -413,7 +460,10 @@ mod tasks_mode { assert_eq!(actor.request(Ping).await.unwrap(), 7); assert_eq!( seen.exits(), - vec![Exit { from: ActorId::EXTERNAL, reason: ExitReason::Shutdown }] + vec![Exit { + from: ActorId::EXTERNAL, + reason: ExitReason::Shutdown + }] ); actor.child_handle().stop(); ended(&actor.child_handle()).await; @@ -491,7 +541,10 @@ mod threads_mode { assert_eq!(peer.request(Ping).unwrap(), 7); assert_eq!( peer_seen.exits(), - vec![Exit { from: target.id(), reason: ExitReason::Killed }] + vec![Exit { + from: target.id(), + reason: ExitReason::Killed + }] ); // NoProc for a late link and try_link. assert_eq!(peer.context().try_link(&target.child_handle()), Err(NoProc)); diff --git a/concurrency/tests/panic_safety.rs b/concurrency/tests/panic_safety.rs index b49cd433..ce07ec78 100644 --- a/concurrency/tests/panic_safety.rs +++ b/concurrency/tests/panic_safety.rs @@ -552,7 +552,8 @@ fn exercise(h: &dyn Harness, fault: Fault, gate: &Gate, events: &Events) -> Exit let seen = events.snapshot(); // A victim that panics in started() can pass its death point before the // observer's monitor or link is registered: that registration gets NoProc. - let carries = |r: &ExitReason| *r == reason || (fault == Fault::Started && *r == ExitReason::NoProc); + let carries = + |r: &ExitReason| *r == reason || (fault == Fault::Started && *r == ExitReason::NoProc); assert_eq!( seen.iter() .filter(|e| matches!(e, Event::Down(m, r) if *m == monitor && carries(r))) diff --git a/testing/tests/tasks_kit.rs b/testing/tests/tasks_kit.rs index 54f3fbaf..4703971e 100644 --- a/testing/tests/tasks_kit.rs +++ b/testing/tests/tasks_kit.rs @@ -266,7 +266,10 @@ async fn exit_injects_every_reason() { // a kill ends it Killed. for (reason, expected) in [ (ExitReason::Shutdown, ExitReason::Shutdown), - (ExitReason::Panic("injected".into()), ExitReason::Panic("injected".into())), + ( + ExitReason::Panic("injected".into()), + ExitReason::Panic("injected".into()), + ), (ExitReason::Kill, ExitReason::Killed), ] { let actor = Counter(Arc::default()).start(); From 2fa4ca0471494f65b9fd31847c8ea9731976fb65 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:47:09 -0300 Subject: [PATCH 076/208] test: name the action type --- concurrency/tests/otp_exit_signals.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/concurrency/tests/otp_exit_signals.rs b/concurrency/tests/otp_exit_signals.rs index 185273fb..5c31db4c 100644 --- a/concurrency/tests/otp_exit_signals.rs +++ b/concurrency/tests/otp_exit_signals.rs @@ -105,7 +105,8 @@ mod tasks_mode { reply.reply(()); } } - struct Do(Box) + Send>); + type Action = Box) + Send>; + struct Do(Action); impl Message for Do { type Result = (); } From 4facf6bb59f0dfcabd39debf3b7d5fa2b689cc18 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:52:26 -0300 Subject: [PATCH 077/208] docs: supervised actors skip stopped() on shutdown unless trapping exits (design client mapping, MIGRATION note) --- docs/MIGRATION.md | 45 +++++++++++++++++++++++++-------- docs/design/otp-exit-signals.md | 2 +- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index f71cf7da..088fa38e 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -17,16 +17,41 @@ table is the full list). What to edit: `stop_with_reason(Kill)` stays an administrative stop ending `Kill`. - **`stopped()` no longer always runs.** It is skipped when the exit is begun by a kill, an untrapped exit signal or link notice (a linked peer's - abnormal death), or a runtime drop. **Once E3 lands (not yet in this - branch) this includes supervisor and system shutdown of a non-trapping - actor**: the supervisor's `Shutdown` becomes the parent's exit signal (OTP `exit(Child, shutdown)`), so a non-trapping child - ends `Shutdown` without `stopped()`, as a gen_server that does not trap - exits skips `terminate`. Cleanup that must run on every exit belongs in - `Drop`, or the actor calls `ctx.trap_exit(true)` in `started()`: a trapping - child processes the messages queued before the shutdown, then runs - `stopped()`. A child that has not reached its death point within - `shutdown_timeout` is killed; cleanup is stuck only after a second - `shutdown_timeout`. + abnormal death), or a runtime drop (decision 2, as OTP skips `terminate`). + Cleanup that must run on every exit belongs in `Drop`. + +> **Supervised and system-owned actors: `stopped()` is skipped on shutdown +> unless the actor traps exits.** Supervisor and system shutdown send the +> parent's `Shutdown` exit signal (OTP `exit(Child, shutdown)`). A +> non-trapping child ends `Shutdown` at once (after its current handler) +> **without `stopped()`**, as a gen_server that does not trap exits skips +> `terminate`. An actor that needs `stopped()` for cleanup on shutdown must +> trap exits, as in OTP; it then finishes the messages queued before the +> shutdown and runs `stopped()`. A child that has not finished within its +> `shutdown_timeout` is killed (`Killed`, `stopped()` skipped if it has not +> begun); cleanup counts as stuck only after a second `shutdown_timeout`. +> Unmanaged `.start()` actors have no parent and are unaffected. +> +> ```rust +> #[actor(protocol = StoreProtocol)] +> impl Store { +> #[started] +> async fn started(&mut self, ctx: &Context) { +> // v0.8: needed for stopped() to run on supervisor/system shutdown. +> ctx.trap_exit(true); +> } +> +> #[stopped] +> async fn stopped(&mut self, _ctx: &Context) { +> self.flush_to_disk().await; +> } +> } +> ``` +> +> A trapping actor also receives linked peers' deaths as `Exit` messages +> (`exit_received`) instead of dying with them; trap only where that is +> intended, or move the cleanup into `Drop`. + - **Runtime drop** reports `Killed` (was `Kill` in tasks mode, `Panic("unexpected framework panic")` in threads mode). - **Late link or monitor**: to an actor past its death point, `monitor` diff --git a/docs/design/otp-exit-signals.md b/docs/design/otp-exit-signals.md index 31c9cc5f..4af32c43 100644 --- a/docs/design/otp-exit-signals.md +++ b/docs/design/otp-exit-signals.md @@ -764,7 +764,7 @@ uses `stop_with_reason`, `Kill`, `is_abnormal`, `exit_reason()` or |---|---|---| | E1 Killed | none | none | | E2 exit signals | none, because `Context::stop` stays a self-exit. Two sites call `stop()` on a `Context` clone from outside the handler: [`rlpx/connection/server.rs:1162`](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/server.rs#L1162) (a stream listener stopping its own actor) and the test [`shutdown_order_tests.rs:197`](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/test/tests/p2p/shutdown_order_tests.rs#L197) (`table.actor_ref().context().stop()` on a mock, then `join`) | none: [`http_server.rs:87`](https://github.com/lambdaclass/ethlambda/blob/6a12bc753ae18c9d63f3dbf930edc700c4c4704c/bin/ethlambda/src/http_server.rs#L87) stops its own actor from a helper so the Permanent supervisor restarts it | -| E3 parent exit | none (no trapping actors; supervised and system-owned actors stop as today) | none | +| E3 parent exit | **behaviour change**: no actor traps exits, so a supervised or system-owned actor stopped by supervisor or system shutdown now ends `Shutdown` **without `stopped()`** (decision 2, as OTP skips `terminate` for a non-trapping gen_server); an actor that needs `stopped()` for cleanup on shutdown must call `ctx.trap_exit(true)` in `started()`. The two ethrex actors with a `stopped()` ([`rlpx/connection/server.rs:624`](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/server.rs#L624), `tooling/monitor/src/app.rs:126`) are started with `.start()` (no parent) and are unaffected | none (no actor defines `stopped()`) | | E4 NoProc, E6 fences | none | none | | E5 non_exhaustive | no break ([`discovery/supervised.rs:261`](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/discovery/supervised.rs#L261) uses `matches!`); `shutdown.rs:170` logs with `Display` | no break ([`http_server.rs:242`](https://github.com/lambdaclass/ethlambda/blob/6a12bc753ae18c9d63f3dbf930edc700c4c4704c/bin/ethlambda/src/http_server.rs#L242)) | | S1 startup aborts | **behaviour change**: the Transient discovery child's factory [`DiscoveryServer::build`](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/discovery/supervised.rs#L145) can fail at boot (a UDP send error pinging a bootnode); v0.7 retries it, v0.8 fails `start()` and `init_network` panics on `.expect("Network starts")`. Fix: make the bootnode ping best-effort | none: the HTTP child's factory is infallible; a bind failure happens after readiness and is a normal restart | From daef3c303db681e53f33823de26b679e0cc6025b Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:37:54 -0300 Subject: [PATCH 078/208] feat(supervision): supervisor and system shutdown send the parent exit signal; kill at shutdown_timeout/grace (E3) --- concurrency/src/supervision.rs | 36 ++++++++++++++++++++++++--- concurrency/src/system.rs | 45 +++++++++++++++++++++++++++++++--- 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs index 8ed2c705..bf0e36ff 100644 --- a/concurrency/src/supervision.rs +++ b/concurrency/src/supervision.rs @@ -525,11 +525,13 @@ impl ChildControl for View { self.changed.send_modify(|n| *n = n.wrapping_add(1)); true } + /// Collateral stop (OneForAll/RestForOne): the supervisor's parent exit + /// signal (E3); the controller escalates to a kill at `shutdown_timeout`. fn stop_generation(&self, slot: &RecoverySlot) { slot.stop.cancel(); let scope = system::lock(&self.data).scope.clone(); if let Some(scope) = scope { - scope.stop(ExitReason::Shutdown); + scope.signal_parent(ExitReason::Shutdown); } } fn snapshot(&self) -> ChildSnapshot { @@ -548,10 +550,15 @@ impl ChildControl for View { } self.changed.send_modify(|n| *n = n.wrapping_add(1)); } + /// Supervisor shutdown of this child (V9, reverse order): the parent + /// exit signal `exit(Child, shutdown)` (E3). A non-trapping child ends + /// `Shutdown` without `stopped()`; a trapping child finishes its earlier + /// messages and runs `stopped()`. The controller kills it at + /// `shutdown_timeout`. fn stop(&self) { self.stop_turn.cancel(); if let Some(scope) = system::lock(&self.data).scope.clone() { - scope.stop(ExitReason::Shutdown); + scope.signal_parent(ExitReason::Shutdown); } } fn done(&self) -> watch::Receiver { @@ -2053,7 +2060,7 @@ async fn run_child(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc { + scope.kill(); + // The second interval runs from the kill. + cleanup_until = Some(Instant::now()); + match rt::timeout(spec.shutdown_timeout, child.wait_exit_async()) + .await + { + Ok(reason) => reason, + Err(_) => { + stuck = true; + core.fail_cleanup_for( + view.snapshot(), + format!("actor cleanup stuck for {}", spec.name), + || Some(Event::new("CleanupStuck", c, "")), + ); + child.wait_exit_async().await + } + } + } Err(_) => { if !stop_turn { trace::emit(core.trace, || { diff --git a/concurrency/src/system.rs b/concurrency/src/system.rs index 3ab6b777..0fb91cc1 100644 --- a/concurrency/src/system.rs +++ b/concurrency/src/system.rs @@ -524,16 +524,19 @@ impl SystemInner { } } ShutdownTarget::Scope(scope) => { + // The system is a standalone actor's parent: its + // Shutdown is a parent exit signal (E3). scope.set_cleanup_timeout(remaining); - scope.stop(ExitReason::Shutdown); + scope.signal_parent(ExitReason::Shutdown); let _ = rt::timeout(remaining, scope.joined()).await; } } } - // At grace expiry, no nested child may remain unsignalled behind - // a stuck sibling's longer per-child cleanup timeout. + // At grace expiry every remaining actor gets the kill (E3), so + // no nested child remains unsignalled behind a stuck sibling's + // longer per-child cleanup timeout. for scope in scopes.iter().rev() { - scope.stop(ExitReason::Shutdown); + scope.kill(); } // Runtime teardown still requires actual completion, independently // of the ordering grace and the caller's report timeout. @@ -983,6 +986,40 @@ impl Scope { pub(crate) fn close_admission(&self) { lock(&self.state).open = false; } + /// The parent's exit signal to this scope's actor (design E3): the + /// supervisor or system is the parent. Admission closes for the managed + /// tree at once (B), but nothing is committed for the actor: a + /// non-trapping actor begins exit with `reason` without `stopped()` + /// (`Normal` is ignored), a trapping one gets a parent exit queued behind + /// its messages, which ends it through `stopped()`. Before the actor is + /// attached this is an ordinary stop. + pub(crate) fn signal_parent(&self, reason: ExitReason) { + let child = { + let mut state = lock(&self.state); + state.open = false; + if state.ended_at.is_none() { + state.ended_at = Some(Instant::now()); + } + state.actor.clone() + }; + match child { + Some(child) => child.exit_from(crate::ActorId::EXTERNAL, reason, true), + None => self.stop(reason), + } + } + /// The parent's kill (E3 escalation at `shutdown_timeout` or grace): the + /// actor ends `Killed`, skipping a `stopped()` that has not begun. + pub(crate) fn kill(&self) { + let child = { + let mut state = lock(&self.state); + state.open = false; + state.actor.clone() + }; + match child { + Some(child) => child.exit_from(crate::ActorId::EXTERNAL, ExitReason::Kill, true), + None => self.stop(ExitReason::Killed), + } + } pub(crate) fn register( &self, name: &str, From 8beb70382b53d5ab7a629b4462ad6e6d501ea53e Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:55:49 -0300 Subject: [PATCH 079/208] test: shutdown-order actors trap exits (E3); kill at grace yields Killed; deliver link notices before publishing completion --- concurrency/src/actor_cleanup.rs | 8 ++++-- concurrency/src/supervision.rs | 10 +++++++ concurrency/tests/canary_feedback.rs | 7 ++--- concurrency/tests/cancellation_safety.rs | 5 ++++ concurrency/tests/coverage_trace.rs | 5 ++++ concurrency/tests/cross_backend.rs | 31 ++++++++++++++++++--- concurrency/tests/execution_groups.rs | 32 ++++++++++++++++++---- concurrency/tests/failpoints.rs | 14 ++++++++-- concurrency/tests/nested_supervision.rs | 5 ++++ concurrency/tests/supervision.rs | 35 +++++++++++++++--------- concurrency/tests/supervision_trace.rs | 5 ++++ 11 files changed, 125 insertions(+), 32 deletions(-) diff --git a/concurrency/src/actor_cleanup.rs b/concurrency/src/actor_cleanup.rs index 98145fc1..69e1526a 100644 --- a/concurrency/src/actor_cleanup.rs +++ b/concurrency/src/actor_cleanup.rs @@ -100,12 +100,14 @@ impl Drop for ActorCleanup { (self.cancel)(); drop(self.monitors.take()); self.metrics.finish(&reason); - // Publication: wait_exit, join and monitors observe the final reason, - // then the link notices taken at D are delivered. - self.completion.send_replace(Some(reason)); + // Publication: the link notices taken at D are delivered, then + // wait_exit, join and monitors observe the final reason. Delivering + // first keeps v0.7's guarantee that once `wait_exit` returns, every + // linked peer's `Exit` is already queued (or its exit requested). if let Some(notices) = self.notices.take() { notices.deliver(); } + self.completion.send_replace(Some(reason)); } } diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs index bf0e36ff..a7e62acf 100644 --- a/concurrency/src/supervision.rs +++ b/concurrency/src/supervision.rs @@ -2561,6 +2561,11 @@ mod policy_tests { release: Option>, } impl Actor for Ordered { + async fn started(&mut self, ctx: &crate::tasks::Context) { + // v0.8: stopped() records the order only for a trapping + // child (E3, decision 2). + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &crate::tasks::Context) { system::lock(&self.log).push(self.id); if let Some(entered) = self.entered.take() { @@ -2660,6 +2665,11 @@ mod policy_tests { release: Option>, } impl Actor for Ordered { + async fn started(&mut self, ctx: &crate::tasks::Context) { + // v0.8: stopped() records the order only for a trapping + // child (E3, decision 2). + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &crate::tasks::Context) { system::lock(&self.log).push(self.id); if let Some(entered) = self.entered.take() { diff --git a/concurrency/tests/canary_feedback.rs b/concurrency/tests/canary_feedback.rs index 790e0ae0..39af49fb 100644 --- a/concurrency/tests/canary_feedback.rs +++ b/concurrency/tests/canary_feedback.rs @@ -441,10 +441,9 @@ async fn panicked_turn_is_cleared_before_stopped_hook() { assert_eq!(detail.turn_age, None); release.send(()).unwrap(); assert!(report.wait(LIMIT).await.complete); - assert!(matches!( - actor.child_handle().exit_reason(), - Some(ExitReason::Panic(_)) - )); + // The system's kill at grace reached it inside stopped(), before its + // death point: Killed replaces the panic (E1). + assert_eq!(actor.child_handle().exit_reason(), Some(ExitReason::Killed)); } #[cfg(feature = "diagnostics")] diff --git a/concurrency/tests/cancellation_safety.rs b/concurrency/tests/cancellation_safety.rs index 85baab53..8f3c460a 100644 --- a/concurrency/tests/cancellation_safety.rs +++ b/concurrency/tests/cancellation_safety.rs @@ -158,6 +158,11 @@ impl Probe { } impl Actor for Probe { async fn started(&mut self, ctx: &Context) { + // A stop gate slows stopped(); on a parent's shutdown it runs only + // for a trapping actor (v0.8, E3). + if self.stop_gate.is_some() { + ctx.trap_exit(true); + } if let Some(gate) = &self.start_gate { gate.notified().await; } diff --git a/concurrency/tests/coverage_trace.rs b/concurrency/tests/coverage_trace.rs index 1abefb58..bd40faef 100644 --- a/concurrency/tests/coverage_trace.rs +++ b/concurrency/tests/coverage_trace.rs @@ -244,6 +244,11 @@ struct SlowStop { release: Option>, } impl Actor for SlowStop { + async fn started(&mut self, ctx: &Context) { + // v0.8: stopped() runs on a parent's shutdown only for a trapping + // actor (E3, decision 2). + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &Context) { if let Some(release) = self.release.take() { let _ = release.await; diff --git a/concurrency/tests/cross_backend.rs b/concurrency/tests/cross_backend.rs index a5237545..7ca223ca 100644 --- a/concurrency/tests/cross_backend.rs +++ b/concurrency/tests/cross_backend.rs @@ -103,6 +103,9 @@ struct State { ticks_seen: u32, pulses_wanted: u32, pulses_seen: u32, + /// Trap exits from started(): v0.8 runs stopped() on a parent's + /// (supervisor or system) shutdown only for a trapping actor (E3). + trap: bool, } impl State { fn new(index: u32, log: Log) -> Self { @@ -115,8 +118,13 @@ impl State { ticks_seen: 0, pulses_wanted: 0, pulses_seen: 0, + trap: false, } } + fn trapping(mut self) -> Self { + self.trap = true; + self + } fn echo(&mut self, v: u64) -> Reply { self.handled += 1; Reply::Value(v.wrapping_mul(1_000) + u64::from(self.index) * 10 + self.handled) @@ -144,6 +152,11 @@ mod tasks_node { pub interval: Option, } impl Actor for Node { + async fn started(&mut self, ctx: &Context) { + if self.state.trap { + ctx.trap_exit(true); + } + } async fn stopped(&mut self, _: &Context) { self.state.log.push(Obs::Stopped(self.state.index)); } @@ -298,6 +311,11 @@ mod threads_node { pub interval: Option, } impl Actor for Node { + fn started(&mut self, ctx: &Context) { + if self.state.trap { + ctx.trap_exit(true); + } + } fn stopped(&mut self, _: &Context) { self.state.log.push(Obs::Stopped(self.state.index)); } @@ -1021,7 +1039,7 @@ fn supervisor_workload( let log = log.clone(); tree.add(ChildSpec::new(format!("c{i}"), &group, move |_| { calls[i].fetch_add(1, Ordering::SeqCst); - let state = State::new(i as u32, log.clone()); + let state = State::new(i as u32, log.clone()).trapping(); async move { Ok::<_, String>(tasks_node::Node { state, @@ -1093,8 +1111,13 @@ struct ShutdownOutcome { fn shutdown_workload(env: &mut Env, before: u32, sup_children: u32, after: u32) -> ShutdownOutcome { let log = Recorder::new(); let mut ids = Vec::new(); + // Trapping, so stopped() records the system's shutdown order (E3). + let trap = |actor: Ref| { + assert_eq!(actor.request(Cmd::Trap).unwrap(), Reply::Unit); + actor.child().id() + }; for i in 0..before { - ids.push(env.start(i, vec![], &log).child().id()); + ids.push(trap(env.start(i, vec![], &log))); } let group = env.group.clone().unwrap(); let system = env.system.take().unwrap(); @@ -1104,7 +1127,7 @@ fn shutdown_workload(env: &mut Env, before: u32, sup_children: u32, after: u32) for i in 0..sup_children { let log = log.clone(); tree.add(ChildSpec::new(format!("s{i}"), &group, move |_| { - let state = State::new(100 + i, log.clone()); + let state = State::new(100 + i, log.clone()).trapping(); async move { Ok::<_, String>(tasks_node::Node { state, @@ -1117,7 +1140,7 @@ fn shutdown_workload(env: &mut Env, before: u32, sup_children: u32, after: u32) } let supervisor = (sup_children > 0).then(|| env.bounded(tree.start()).unwrap().unwrap()); for i in 0..after { - ids.push(env.start(200 + i, vec![], &log).child().id()); + ids.push(trap(env.start(200 + i, vec![], &log))); } // ActorSystem::shutdown owns ordering; Env::finish must not stop actors first. env.started.clear(); diff --git a/concurrency/tests/execution_groups.rs b/concurrency/tests/execution_groups.rs index 7bd3c476..a511adca 100644 --- a/concurrency/tests/execution_groups.rs +++ b/concurrency/tests/execution_groups.rs @@ -118,7 +118,10 @@ struct Starting { stopped: Arc, } impl Actor for Starting { - async fn started(&mut self, _: &Context) { + async fn started(&mut self, ctx: &Context) { + // v0.8: stopped() runs on the system's shutdown only for a + // trapping actor (E3, decision 2). + ctx.trap_exit(true); self.entered.take().unwrap().send(()).unwrap(); let _ = self.release.take().unwrap().await; } @@ -475,6 +478,11 @@ async fn stuck_standalone_cleanup_does_not_leave_earlier_actor_running_forever() release: Option>, } impl Actor for Stuck { + async fn started(&mut self, ctx: &Context) { + // v0.8: stopped() runs on the system's shutdown only for a + // trapping actor (E3, decision 2). + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &Context) { let _ = self.entered.take().unwrap().send(()); let _ = self.release.take().unwrap().await; @@ -493,11 +501,13 @@ async fn stuck_standalone_cleanup_does_not_leave_earlier_actor_running_forever() .await .unwrap() .unwrap(); - assert_eq!( - spawned_rt::tasks::timeout(LIMIT, first.wait_exit()) - .await - .unwrap(), - ExitReason::Shutdown + // Signalled after the grace expired, it also gets the kill at grace (E3). + let reason = spawned_rt::tasks::timeout(LIMIT, first.wait_exit()) + .await + .unwrap(); + assert!( + matches!(reason, ExitReason::Shutdown | ExitReason::Killed), + "{reason:?}" ); release.send(()).unwrap(); assert!(report.wait(LIMIT).await.complete); @@ -544,6 +554,11 @@ struct SnapshotHeld { release: Option>, } impl Actor for SnapshotHeld { + async fn started(&mut self, ctx: &Context) { + // v0.8: stopped() runs on the system's shutdown only for a + // trapping actor (E3, decision 2). + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &Context) { let _ = self.release.take().unwrap().await; } @@ -596,6 +611,11 @@ struct Dependent { observed: Arc, } impl Actor for Dependent { + async fn started(&mut self, ctx: &Context) { + // v0.8: stopped() runs on the system's shutdown only for a + // trapping actor (E3, decision 2). + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &Context) { assert!(self.earlier.request(CheckAdmission).await.unwrap()); self.observed.fetch_add(1, Ordering::SeqCst); diff --git a/concurrency/tests/failpoints.rs b/concurrency/tests/failpoints.rs index 9af929ea..d60ce257 100644 --- a/concurrency/tests/failpoints.rs +++ b/concurrency/tests/failpoints.rs @@ -68,6 +68,8 @@ impl Started { } struct Worker { started: Started, + /// Trap exits, so stopped() runs on a parent's shutdown (v0.8, E3). + trap: bool, name: Option<&'static str>, registration: Arc>>, } @@ -75,6 +77,7 @@ impl Worker { fn new(started: &Started) -> Self { Self { started: started.clone(), + trap: false, name: None, registration: Default::default(), } @@ -82,6 +85,9 @@ impl Worker { } impl Actor for Worker { async fn started(&mut self, ctx: &Context) { + if self.trap { + ctx.trap_exit(true); + } self.started .0 .lock() @@ -811,14 +817,18 @@ async fn slow_stopped_reports_at_deadline_then_completes() { .unwrap(); let system = ActorSystem::new().unwrap(); let group = system.async_group("fp-stopped", 1).unwrap(); - let actor = group.start(Worker::new(&Started::default())).unwrap(); + let mut worker = Worker::new(&Started::default()); + worker.trap = true; + let actor = group.start(worker).unwrap(); actor.ready().await.unwrap(); let report = system.shutdown(Duration::from_millis(100)).await; assert!(!report.complete); assert_eq!(report.unfinished_actors, [actor.id()]); let report = report.wait(LIMIT).await; assert!(report.complete); - assert_eq!(actor.exit_reason(), Some(ExitReason::Shutdown)); + // The kill at grace arrived while stopped() ran: it finishes, but the + // reason becomes Killed (E1). + assert_eq!(actor.exit_reason(), Some(ExitReason::Killed)); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/concurrency/tests/nested_supervision.rs b/concurrency/tests/nested_supervision.rs index 79866908..af648a1a 100644 --- a/concurrency/tests/nested_supervision.rs +++ b/concurrency/tests/nested_supervision.rs @@ -367,6 +367,11 @@ async fn native_leaf_helper_blocks_subtree_replacement_and_remains_owned_after_a struct HeldStop(Arc); impl Actor for HeldStop { + async fn started(&mut self, ctx: &Context) { + // v0.8: stopped() runs on a parent's shutdown only for a trapping + // actor (E3, decision 2). + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &Context) { self.0.notified().await; } diff --git a/concurrency/tests/supervision.rs b/concurrency/tests/supervision.rs index fbcc409c..42b10641 100644 --- a/concurrency/tests/supervision.rs +++ b/concurrency/tests/supervision.rs @@ -358,6 +358,10 @@ struct SlowStop { release: Option>, } impl Actor for SlowStop { + async fn started(&mut self, ctx: &Context) { + // Traps exits so stopped() also runs on its parent's shutdown (E3). + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &Context) { let _ = self.entered.take().unwrap().send(()); let _ = self.release.take().unwrap().await; @@ -430,7 +434,9 @@ async fn stuck_stopped_hook_triggers_cleanup_deadline_before_terminal_publicatio let report = report.wait(LIMIT).await; assert!(report.complete); assert_eq!(calls.load(Ordering::SeqCst), 1); - assert!(matches!(old.exit_reason(), Some(ExitReason::Panic(_)))); + // The system's kill at grace reached the actor inside its stuck + // stopped() (before its death point): Killed replaces the panic (E1). + assert_eq!(old.exit_reason(), Some(ExitReason::Killed)); } struct Ordered { @@ -438,7 +444,10 @@ struct Ordered { log: Arc>>, } impl Actor for Ordered { - async fn started(&mut self, _: &Context) { + async fn started(&mut self, ctx: &Context) { + // v0.8: only a trapping child runs stopped() on its parent's + // shutdown (E3, decision 2); the order is observed through it. + ctx.trap_exit(true); self.log .lock() .unwrap() @@ -803,18 +812,18 @@ async fn stuck_supervisor_does_not_prevent_earlier_standalone_shutdown_after_sys .await .unwrap() .unwrap(); - assert_eq!( - spawned_rt::tasks::timeout(LIMIT, earlier.wait_exit()) + // Signalled after the system's grace expired, they also get the kill + // at grace (E3): Shutdown if they reached their death point first, + // otherwise Killed. + for actor in [earlier.child_handle(), inner_actor.child_handle()] { + let reason = spawned_rt::tasks::timeout(LIMIT, actor.wait_exit_async()) .await - .unwrap(), - ExitReason::Shutdown - ); - assert_eq!( - spawned_rt::tasks::timeout(LIMIT, inner_actor.wait_exit()) - .await - .unwrap(), - ExitReason::Shutdown - ); + .unwrap(); + assert!( + matches!(reason, ExitReason::Shutdown | ExitReason::Killed), + "{reason:?}" + ); + } release.send(()).unwrap(); assert!(report.wait(LIMIT).await.complete); } diff --git a/concurrency/tests/supervision_trace.rs b/concurrency/tests/supervision_trace.rs index c8c23034..8d45e131 100644 --- a/concurrency/tests/supervision_trace.rs +++ b/concurrency/tests/supervision_trace.rs @@ -629,6 +629,11 @@ struct SlowStop { release: Option>, } impl Actor for SlowStop { + async fn started(&mut self, ctx: &Context) { + // v0.8: stopped() runs on a parent's shutdown only for a trapping + // actor (E3, decision 2). + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &Context) { if let Some(release) = self.release.take() { let _ = release.await; From 73289af3ced8ab0d8dda577e07ed3cc37f4a9576 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:57:02 -0300 Subject: [PATCH 080/208] fix(system): kill remaining actors only at grace expiry; test updates --- concurrency/src/system.rs | 16 ++++++++++++---- concurrency/tests/execution_groups.rs | 5 ++++- concurrency/tests/supervision.rs | 3 ++- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/concurrency/src/system.rs b/concurrency/src/system.rs index 0fb91cc1..ebb36c94 100644 --- a/concurrency/src/system.rs +++ b/concurrency/src/system.rs @@ -532,11 +532,19 @@ impl SystemInner { } } } - // At grace expiry every remaining actor gets the kill (E3), so - // no nested child remains unsignalled behind a stuck sibling's - // longer per-child cleanup timeout. + // The ordered pass ended: every target joined, or the grace + // expired. At grace expiry every remaining actor gets the kill + // (E3), so no nested child remains unsignalled behind a stuck + // sibling's longer per-child cleanup timeout. Before it, the + // remaining scopes only get the parent's Shutdown (a no-op for + // actors already past their death point). + let expired = began.elapsed() >= grace; for scope in scopes.iter().rev() { - scope.kill(); + if expired { + scope.kill(); + } else { + scope.signal_parent(ExitReason::Shutdown); + } } // Runtime teardown still requires actual completion, independently // of the ordering grace and the caller's report timeout. diff --git a/concurrency/tests/execution_groups.rs b/concurrency/tests/execution_groups.rs index a511adca..d6e568ff 100644 --- a/concurrency/tests/execution_groups.rs +++ b/concurrency/tests/execution_groups.rs @@ -155,7 +155,10 @@ async fn expired_shutdown_retains_closed_runtime_until_startup_cleanup_finishes( release.send(()).unwrap(); assert!(report.wait(LIMIT).await.complete); assert!(actor.ready().await.is_err()); - assert_eq!(stopped.load(Ordering::SeqCst), 1); + // The kill at grace (5 ms) arrived during started(): it ends Killed and + // skips the stopped() that had not begun (E1, decision 2). + assert_eq!(actor.wait_exit().await, ExitReason::Killed); + assert_eq!(stopped.load(Ordering::SeqCst), 0); } struct Probe(Arc); diff --git a/concurrency/tests/supervision.rs b/concurrency/tests/supervision.rs index 42b10641..45ac247f 100644 --- a/concurrency/tests/supervision.rs +++ b/concurrency/tests/supervision.rs @@ -1309,7 +1309,8 @@ async fn system_deadline_signals_remaining_supervised_siblings() { spawned_rt::tasks::timeout(Duration::from_secs(1), actor.wait_exit()) .await .expect("earlier sibling was not stopped after the system deadline"), - ExitReason::Shutdown + ExitReason::Killed, + "signalled at the system's grace expiry: the kill (E3)" ); release.send(()).unwrap(); assert!(report.wait(LIMIT).await.complete); From 09ea3fbf8a39228cdccba12c341b282cbcfed9e9 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:57:53 -0300 Subject: [PATCH 081/208] test(cancellation): zero-grace shutdown kills a stuck stopped() --- concurrency/tests/cancellation_safety.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/concurrency/tests/cancellation_safety.rs b/concurrency/tests/cancellation_safety.rs index 8f3c460a..1695d996 100644 --- a/concurrency/tests/cancellation_safety.rs +++ b/concurrency/tests/cancellation_safety.rs @@ -816,11 +816,12 @@ async fn system_shutdown_is_cancel_safe() { "{op}" ); gate.notify_one(); + // Zero grace: the kill reached it inside stopped() (E1). assert_eq!( spawned_rt::tasks::timeout(LIMIT, actor.wait_exit()) .await .unwrap(), - ExitReason::Shutdown + ExitReason::Killed ); fx.finish(&[]).await; outcome.finished() From c5690d9edc538293d32f0ee4e38fb159571bd301 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 00:58:17 -0300 Subject: [PATCH 082/208] test: supervisor shutdown as the parent's exit signal (E3) --- concurrency/tests/otp_exit_signals.rs | 66 +++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/concurrency/tests/otp_exit_signals.rs b/concurrency/tests/otp_exit_signals.rs index 5c31db4c..905fc99a 100644 --- a/concurrency/tests/otp_exit_signals.rs +++ b/concurrency/tests/otp_exit_signals.rs @@ -383,6 +383,72 @@ mod tasks_mode { assert_eq!(dead.child_handle().exit_reason(), Some(ExitReason::Normal)); } + // E3 with decision 2: supervisor shutdown is the parent's exit signal. + // A non-trapping child ends Shutdown without stopped(); a trapping child + // runs stopped(); a child that does not reach its death point within + // shutdown_timeout is killed. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn supervisor_shutdown_is_the_parents_exit_signal() { + use spawned_concurrency::supervision::{ChildSpec, SupervisorConfig}; + use spawned_concurrency::system::ActorSystem; + let system = ActorSystem::new().unwrap(); + let group = system.async_group("e3", 1).unwrap(); + let (plain_seen, trap_seen, stuck_seen) = + (Seen::default(), Seen::default(), Seen::default()); + let (entered_tx, entered) = tokio::sync::oneshot::channel(); + let (hold, hold_rx) = tokio::sync::oneshot::channel::<()>(); + let stuck_args = Arc::new(Mutex::new(Some((entered_tx, hold_rx)))); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let (p, t, k) = (plain_seen.clone(), trap_seen.clone(), stuck_seen.clone()); + let plain = tree + .add(ChildSpec::new("plain", &group, move |_| { + let seen = p.clone(); + async move { Ok::<_, String>(Node::new(&seen, false)) } + })) + .unwrap(); + let trapping = tree + .add(ChildSpec::new("trapping", &group, move |_| { + let seen = t.clone(); + async move { Ok::<_, String>(Node::new(&seen, true)) } + })) + .unwrap(); + let stuck = tree + .add( + ChildSpec::new("stuck", &group, move |_| { + let seen = k.clone(); + let gate = stuck_args.lock().unwrap().take().unwrap(); + async move { + let mut node = Node::new(&seen, true); + node.in_stopped = Some(gate); + Ok::<_, String>(node) + } + }) + .shutdown_timeout(Duration::from_millis(50)), + ) + .unwrap(); + let supervisor = tree.start().await.unwrap(); + let (plain, trapping, stuck) = ( + plain.current().unwrap().child_handle(), + trapping.current().unwrap().child_handle(), + stuck.current().unwrap().child_handle(), + ); + let shutdown = tokio::spawn(async move { supervisor.shutdown(LIMIT).await.complete }); + entered.await.unwrap(); + // The stuck stopped() outlives shutdown_timeout (50 ms): the kill + // lands while it runs; Rust cannot interrupt it, so release it after. + tokio::time::sleep(Duration::from_millis(300)).await; + drop(hold); + assert!(shutdown.await.unwrap()); + assert_eq!(ended(&plain).await, ExitReason::Shutdown); + assert!(!plain_seen.stopped(), "a non-trapping child ran stopped()"); + assert_eq!(ended(&trapping).await, ExitReason::Shutdown); + assert!(trap_seen.stopped()); + assert!(trap_seen.exits().is_empty(), "the parent exit reached exit_received"); + assert_eq!(ended(&stuck).await, ExitReason::Killed); + assert!(stuck_seen.stopped()); + assert!(system.shutdown(LIMIT).await.complete); + } + pub struct Parent { pub child: Option>, pub child_seen: Seen, From a5b67d7ad010a0b07dc742d95e30b809579a3537 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 01:04:59 -0300 Subject: [PATCH 083/208] docs: E3 supervisor/system shutdown is in --- CHANGELOG.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba494e88..3514e40e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,13 +23,16 @@ began is inert; `unlink`/`demonitor` are fences; a `start_linked` parent's exit ends a trapping child through `stopped()` (never `exit_received`); signals arriving during `started()` are resolved when it returns; - destructor panics after + supervisor and system shutdown send the parent's `Shutdown` exit signal + (a non-trapping child ends `Shutdown` without `stopped()`, a trapping one + finishes its backlog and runs `stopped()`), with the kill at + `shutdown_timeout` (supervisor, then a second interval before cleanup is + stuck) or at grace (system); destructor panics after the death point are cleanup failures (`ChildHandle::cleanup_failure()`, `mailbox_cleanup_failure()` deprecated) that restart a Transient child without changing the reason; `spawned-test` `faults::exit` sends an exact exit signal; metrics `actors_exited_killed_total`, - `actors_exited_noproc_total`. Remaining: supervisor/system shutdown as the parent's exit signal - with the kill escalation (E3), S1/S2 supervisor startup, rank 11 + `actors_exited_noproc_total`. Remaining: S1/S2 supervisor startup, rank 11 reason-bearing `ActorStopped`/`NoProc` errors, TLA+ and trace work. - **Breaking (0.8), in progress:** deferred replies and pipe-to-self From 60ae442c82fbcbf82a7596c10c66f40b72d570dd Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 04:05:28 +0000 Subject: [PATCH 084/208] style: cargo fmt --- concurrency/tests/otp_exit_signals.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/concurrency/tests/otp_exit_signals.rs b/concurrency/tests/otp_exit_signals.rs index 905fc99a..b603127b 100644 --- a/concurrency/tests/otp_exit_signals.rs +++ b/concurrency/tests/otp_exit_signals.rs @@ -443,7 +443,10 @@ mod tasks_mode { assert!(!plain_seen.stopped(), "a non-trapping child ran stopped()"); assert_eq!(ended(&trapping).await, ExitReason::Shutdown); assert!(trap_seen.stopped()); - assert!(trap_seen.exits().is_empty(), "the parent exit reached exit_received"); + assert!( + trap_seen.exits().is_empty(), + "the parent exit reached exit_received" + ); assert_eq!(ended(&stuck).await, ExitReason::Killed); assert!(stuck_seen.stopped()); assert!(system.shutdown(LIMIT).await.complete); From 22a18db71f04e75c2a9942a78ccd3695c17315e7 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 01:41:56 -0300 Subject: [PATCH 085/208] feat(supervision): S1/S2 startup as in OTP (rank 5f) A child's failed first start fails SupervisorBuilder::start() whatever its restart type (FailedToStartChild); a ready child's exit during the initial start is deferred until the tree commits ready; start() returns its error only after the rollback. Supervision.tla/Cohorts.tla, trace mapping, inductive lemmas, coverage cache and negative controls follow; tests that relied on first-start retries now exercise replacements. --- CHANGELOG.md | 17 +- concurrency/src/lifecycle_trace.rs | 10 +- concurrency/src/supervision.rs | 243 +++++++++++++++----- concurrency/tests/canary_feedback.rs | 10 +- concurrency/tests/cancellation_safety.rs | 4 +- concurrency/tests/coverage_trace.rs | 76 +++--- concurrency/tests/failpoints.rs | 77 ++++--- concurrency/tests/nested_supervision.rs | 13 +- concurrency/tests/otp_exit_signals.rs | 212 +++++++++++++++++ concurrency/tests/panic_safety.rs | 21 +- concurrency/tests/supervision.rs | 30 ++- concurrency/tests/supervision_strategies.rs | 40 +++- concurrency/tests/supervision_trace.rs | 154 +++++++++++-- docs/MIGRATION.md | 15 +- docs/design/semantics.md | 27 ++- verification/tla/Cohorts.tla | 7 + verification/tla/CohortsInd.tla | 12 +- verification/tla/CovSupervision.tla | 2 + verification/tla/README.md | 12 +- verification/tla/Supervision.tla | 66 ++++-- verification/tla/SupervisionCoverage.tla | 3 +- verification/tla/SupervisionInd.tla | 20 +- verification/tla/TraceSupervision.tla | 4 +- verification/tla/apalache.sh | 10 +- verification/tla/coverage-model.json.gz | Bin 39318 -> 34701 bytes verification/tla/negative_checks.py | 62 +++++ 26 files changed, 924 insertions(+), 223 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3514e40e..3ef85f6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,8 +32,21 @@ `mailbox_cleanup_failure()` deprecated) that restart a Transient child without changing the reason; `spawned-test` `faults::exit` sends an exact exit signal; metrics `actors_exited_killed_total`, - `actors_exited_noproc_total`. Remaining: S1/S2 supervisor startup, rank 11 - reason-bearing `ActorStopped`/`NoProc` errors, TLA+ and trace work. + `actors_exited_noproc_total`. Supervisor startup (S1/S2): any failed first + start of a child (factory error or panic, panic or stop in `started()`, + `startup_timeout`, placement or group closure) fails `start()` whatever the + restart type, with no retry, as `SupervisorError::FailedToStartChild + { child, cause, rollback_failures }` (a stuck cleanup during the initial + start is `StartupCleanupStuck`); a ready child's exit during the initial + start is deferred and processed under the strategy once the tree commits + ready (OneForOne no longer restarts it at once, OneForAll/RestForOne no + longer fail the start); `start()` returns its error only after the + rollback (every child terminal, or `Failed` once its cleanup is stuck). + `Supervision.tla` (no restart before the tree is ready, deferral actions + `DeferExit`/`Undefer`, traced) and `Cohorts.tla` (round 0 once per child, + `FirstStartOnce`) follow, with the v0.7 behaviours as negative controls. + Remaining: rank 11 reason-bearing `ActorStopped`/`NoProc` errors, + lifecycle-trace mapping to `Lifecycle.tla`, Loom and simulator coverage. - **Breaking (0.8), in progress:** deferred replies and pipe-to-self (ROADMAP rank 11), implementing diff --git a/concurrency/src/lifecycle_trace.rs b/concurrency/src/lifecycle_trace.rs index 3f4dd745..0e237995 100644 --- a/concurrency/src/lifecycle_trace.rs +++ b/concurrency/src/lifecycle_trace.rs @@ -376,6 +376,14 @@ mod imp { lock(&TRACES).remove(&id).unwrap_or_default() } + /// Whether a supervisor's trace already holds an event named `action` + /// (lets a test wait for an ordering point without taking the trace). + pub fn recorded(id: u64, action: &str) -> bool { + lock(&TRACES) + .get(&id) + .is_some_and(|events| events.iter().any(|e| e.action == action)) + } + /// What a dumped trace needs besides its events: the constants /// `trace_check.py` instantiates the model with. struct Meta { @@ -567,4 +575,4 @@ mod imp { } #[cfg(feature = "lifecycle-trace")] -pub use imp::{take, take_jobs, take_replies, take_tree, watch_deferred}; +pub use imp::{recorded, take, take_jobs, take_replies, take_tree, watch_deferred}; diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs index a7e62acf..2f413d81 100644 --- a/concurrency/src/supervision.rs +++ b/concurrency/src/supervision.rs @@ -177,10 +177,29 @@ pub enum SupervisorError { DuplicateChild(String), #[error("child capacity exceeded")] Capacity, + /// The start was cancelled or the system closed before it completed. #[error("supervisor stopped before startup completed")] Stopped, #[error("supervisor failed: {0}")] Failed(String), + /// A child's first start failed (S1, OTP `failed_to_start_child`): its + /// factory errored or panicked, it stopped or panicked before readiness, + /// it missed `startup_timeout`, or its placement or group failed. + /// Returned after the rollback finished; `rollback_failures` lists + /// cleanup failures of that rollback. + #[error("failed to start child {child}: {cause}")] + FailedToStartChild { + child: String, + cause: String, + rollback_failures: Vec, + }, + /// A child's cleanup got stuck during the initial start (S2). + #[error("child cleanup stuck during initial start: {child}: {cause}")] + StartupCleanupStuck { + child: String, + cause: String, + rollback_failures: Vec, + }, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -421,6 +440,29 @@ impl View { self.done.send_replace(true); true } + /// Cleanup stuck past its deadlines: the snapshot settles as Failed, + /// which releases a waiting `start()` (S2), while the controller keeps + /// awaiting the real join. + fn mark_stuck(&self) { + self.change(|data| { + data.current = None; + data.snapshot.state = ChildState::Failed; + }); + } + /// Terminal, or settled as Failed by a stuck cleanup. + async fn settled(&self) { + let mut changes = self.changed.subscribe(); + loop { + changes.borrow_and_update(); + let state = system::lock(&self.data).snapshot.state; + if matches!(state, ChildState::Stopped | ChildState::Failed) || *self.done.borrow() { + return; + } + if changes.changed().await.is_err() { + return; + } + } + } fn finish(&self, failed: bool) { self.change(|data| { data.current = None; @@ -502,6 +544,7 @@ trait ChildControl: Send + Sync { fn initial(&self) -> watch::Receiver; fn shutdown_timeout(&self) -> Duration; fn finish_unstarted(&self, failed: bool); + fn settled(&self) -> BoxFuture<'_, ()>; } impl ChildControl for View { fn assign_recovery(&self, slot: Arc) -> bool { @@ -575,6 +618,9 @@ impl ChildControl for View { self.finish(failed); } } + fn settled(&self) -> BoxFuture<'_, ()> { + Box::pin(View::settled(self)) + } } struct RetryBudget { @@ -626,6 +672,8 @@ pub(crate) fn admit_in_window( #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum FailureKind { Other, + /// A child's first start failed (S1). + FirstStart, Cleanup, Coordinator, RestartBudget, @@ -672,6 +720,9 @@ struct Core { failure: Mutex>, failed: Arc, cleanup_failure: Mutex>, + /// Every recorded cleanup failure, for the rollback list of a failed + /// start (S2). + cleanup_failures: Mutex>, children: Mutex>>, budget: Mutex, config: SupervisorConfig, @@ -773,6 +824,7 @@ impl Core { let _ = make; } fn record_cleanup_failure(&self, error: &str) { + system::lock(&self.cleanup_failures).push(error.to_string()); let mut failure = system::lock(&self.cleanup_failure); if failure.is_none() { *failure = Some(error.to_string()); @@ -1070,6 +1122,7 @@ impl SupervisorBuilder { failure: Mutex::new(None), failed: Arc::new(std::sync::atomic::AtomicBool::new(false)), cleanup_failure: Mutex::new(None), + cleanup_failures: Mutex::new(Vec::new()), children: Mutex::new(Vec::new()), budget: Mutex::new(RetryBudget { attempts: VecDeque::new(), @@ -1222,16 +1275,52 @@ impl SupervisorBuilder { // Readiness is historical, just like ActorRef::ready(). A later system // admission close must not turn successful startup into rollback. let initialized = *core.ready.borrow(); - let result = if initialized { + if initialized { guard.armed = false; - Ok(Supervisor { core: core.clone() }) - } else { - Err(system::lock(&core.failure) - .clone() - .map(|failure| SupervisorError::Failed(failure.message)) - .unwrap_or(SupervisorError::Stopped)) + return Ok(Supervisor { core: core.clone() }); + } + // S2: the error is returned only after the rollback finished: every + // child is terminal, or settled as Failed by its stuck cleanup. + let children = system::lock(&core.children).clone(); + for child in &children { + child.settled().await; + } + Err(core.start_error()) + } +} +impl Core { + /// The cause of a failed initial start (S2), with the rollback's cleanup + /// failures listed separately. + fn start_error(&self) -> SupervisorError { + let Some(failure) = system::lock(&self.failure).clone() else { + return SupervisorError::Stopped; }; - result + let rollback_failures: Vec = system::lock(&self.cleanup_failures) + .iter() + .filter(|f| **f != failure.message) + .cloned() + .collect(); + match (failure.kind, failure.origin) { + (FailureKind::FirstStart, Some(origin)) => { + let prefix = format!("failed to start child {}: ", origin.child); + let cause = failure + .message + .strip_prefix(&prefix) + .unwrap_or(&failure.message) + .to_string(); + SupervisorError::FailedToStartChild { + child: origin.child, + cause, + rollback_failures, + } + } + (FailureKind::Cleanup, Some(origin)) => SupervisorError::StartupCleanupStuck { + child: origin.child, + cause: failure.message, + rollback_failures, + }, + _ => SupervisorError::Failed(failure.message), + } } } impl Drop for SupervisorBuilder { @@ -1821,7 +1910,11 @@ async fn run_child(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc); + // v0.8 (S1): a first start is not retried, so the first generation + // is healthy and every replacement panics in started(). + struct FailsStarted(Arc, bool); impl Actor for FailsStarted { async fn started(&mut self, _: &crate::tasks::Context) { - self.0.fetch_add(1, Ordering::SeqCst); - panic!("injected started failure"); + if self.1 { + self.0.fetch_add(1, Ordering::SeqCst); + panic!("injected started failure"); + } } } for allowed in [0, 1, 3] { @@ -2853,25 +2978,25 @@ mod policy_tests { }) .unwrap(); let child = builder - .add(ChildSpec::new("startup-failure", &group, move |_| { + .add(ChildSpec::new("startup-failure", &group, move |ctx| { let attempts = attempts.clone(); - async move { Ok::<_, String>(FailsStarted(attempts)) } + let fail = ctx.generation() > 1; + async move { Ok::<_, String>(FailsStarted(attempts, fail)) } })) .unwrap(); let core = builder.core.clone(); - assert!(matches!( - rt::timeout(Duration::from_secs(30), builder.start()) - .await - .unwrap(), - Err(SupervisorError::Failed(_)) - )); + rt::timeout(Duration::from_secs(30), builder.start()) + .await + .unwrap() + .unwrap(); + child.current().unwrap().child_handle().kill(); assert!(rt::timeout( Duration::from_secs(30), system::wait_true(core.done.subscribe()) ) .await .unwrap()); - assert_eq!(count.load(Ordering::SeqCst), allowed + 1); + assert_eq!(count.load(Ordering::SeqCst), allowed); assert_eq!(child.snapshot().generation as usize, allowed + 1); assert_eq!( system::lock(&core.failure).as_ref().unwrap().kind, diff --git a/concurrency/tests/canary_feedback.rs b/concurrency/tests/canary_feedback.rs index 39af49fb..2ca94dee 100644 --- a/concurrency/tests/canary_feedback.rs +++ b/concurrency/tests/canary_feedback.rs @@ -100,14 +100,14 @@ async fn unfinished_report_identifies_child_and_active_turn() { assert_eq!(detail.message_type, None); assert_eq!(detail.turn_age, None); } + // The system kills what remains at grace (E3); let the kill land + // before the turn returns, so the outcome does not race it. + spawned_rt::tasks::sleep(Duration::from_millis(100)).await; release.send(()).unwrap(); let report = report.wait(LIMIT).await; assert!(report.complete); assert!(report.unfinished_actor_details.is_empty()); - assert_eq!( - actor.child_handle().exit_reason(), - Some(ExitReason::Shutdown) - ); + assert_eq!(actor.child_handle().exit_reason(), Some(ExitReason::Killed)); } // Not simulated: uses spawn_blocking. @@ -346,7 +346,7 @@ async fn supervision_logs_restart_budget_exhaustion_and_give_up() { && fields.get("child").map(String::as_str) == Some("canary-give-up") && fields .get("error") - .is_some_and(|e| e.contains("non-restartable child")))); + .is_some_and(|e| e.contains("failed to start child")))); let exhausted: Vec<_> = events .iter() .filter(|(_, fields)| { diff --git a/concurrency/tests/cancellation_safety.rs b/concurrency/tests/cancellation_safety.rs index 1695d996..f49b0e36 100644 --- a/concurrency/tests/cancellation_safety.rs +++ b/concurrency/tests/cancellation_safety.rs @@ -815,8 +815,10 @@ async fn system_shutdown_is_cancel_safe() { !matches!(outcome, Attempt::Done(ref r) if r.complete), "{op}" ); + // Zero grace: the system kills at once after its ordered pass + // (E3); let the kill land while stopped() still runs (E1). + spawned_rt::tasks::sleep(Duration::from_millis(50)).await; gate.notify_one(); - // Zero grace: the kill reached it inside stopped() (E1). assert_eq!( spawned_rt::tasks::timeout(LIMIT, actor.wait_exit()) .await diff --git a/concurrency/tests/coverage_trace.rs b/concurrency/tests/coverage_trace.rs index bd40faef..9573ef3a 100644 --- a/concurrency/tests/coverage_trace.rs +++ b/concurrency/tests/coverage_trace.rs @@ -153,39 +153,48 @@ async fn first_attempt_abandoned_when_startup_is_dropped() { write_trace("first_attempt_closed", trace_id, &modes, 5); } -/// `T|Close|..|backoff|starting|open`, then `BackoffDone/closed` and -/// `SupStartDone/notready` with a child in backoff: the first child's -/// construction fails during initial startup and, while it waits out its -/// backoff, the caller gives up on the startup (drops the start future). +/// `T|Close|..|deferred|starting|open`, then `Undefer` and `Decide/closed`: +/// a ready child's exit during the initial start is deferred (S1) and the +/// caller gives up on the startup (drops the start future) while it waits. +/// (v0.7 reached `T|Close|..|backoff|starting|open` here through a retried +/// first start, which v0.8 no longer makes.) #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn startup_cancelled_while_first_child_backs_off() { +async fn startup_cancelled_while_a_ready_exit_is_deferred() { let _scenario = FailScenario::setup(); let system = ActorSystem::new().unwrap(); - let group = system.async_group("cov-start-backoff", 1).unwrap(); + let group = system.async_group("cov-start-deferred", 2).unwrap(); let modes = [RestartMode::Permanent, RestartMode::Transient]; - let mut policy = config(5); - policy.backoff.initial = Duration::from_secs(60); - policy.backoff.maximum = Duration::from_secs(60); - let mut tree = system.supervisor(policy).unwrap(); - let flaky = tree - .add(ChildSpec::new("flaky", &group, |ctx| async move { - if ctx.generation() == 1 { - Err("first construction fails".to_string()) - } else { - Ok(Worker) - } - })) + let (_open, gate) = tokio::sync::watch::channel(false); + let mut tree = system.supervisor(config(5)).unwrap(); + let early = tree.add(worker("early", &group)).unwrap(); + let later = tree + .add( + ChildSpec::new("later", &group, move |_| { + let mut gate = gate.clone(); + async move { + let _ = gate.wait_for(|open| *open).await; + Ok::<_, String>(Worker) + } + }) + .restart(modes[1]), + ) .unwrap(); - let later = tree.add(worker("later", &group).restart(modes[1])).unwrap(); let trace_id = tree.trace_id(); let start = tokio::spawn(tree.start()); - until("backoff", || flaky.snapshot().state == ChildState::Backoff).await; + until("the later child under construction", || { + later.snapshot().state == ChildState::Starting + }) + .await; + crash(&early).await; + until("the deferred exit", || { + lifecycle_trace::recorded(trace_id, "DeferExit") + }) + .await; start.abort(); assert!(start.await.is_err()); assert!(system.shutdown(LIMIT).await.complete); - assert_eq!(flaky.snapshot().generation, 1); - assert_eq!(later.snapshot().generation, 0); - write_trace("startup_cancelled_during_backoff", trace_id, &modes, 5); + assert_eq!(early.snapshot().generation, 1); + write_trace("startup_cancelled_while_deferred", trace_id, &modes, 5); } /// Shutdown begins while a restart is under construction and its deadline @@ -310,10 +319,11 @@ impl Handler for QuitInStarted { } } -/// A ready child crashes and restarts while a later sibling is still being -/// constructed during initial startup (`T|SelfExit/Panic|..|starting|open`). +/// A ready child crashes while a later sibling is still being constructed +/// during initial startup (`T|SelfExit/Panic|..|starting|open`); its exit is +/// deferred (S1) and it restarts once the tree commits ready. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn ready_child_restarts_while_a_sibling_is_still_starting() { +async fn ready_child_exit_deferred_while_a_sibling_is_still_starting() { let _scenario = FailScenario::setup(); let system = ActorSystem::new().unwrap(); let group = system.async_group("cov-start-sibling", 2).unwrap(); @@ -333,22 +343,28 @@ async fn ready_child_restarts_while_a_sibling_is_still_starting() { .restart(modes[1]), ) .unwrap(); + let trace_id = tree.trace_id(); let start = tokio::spawn(tree.start()); until("the late child under construction", || { late.snapshot().state == ChildState::Starting }) .await; crash(&early).await; - tokio::time::timeout(LIMIT, early.wait_for_generation(1)) - .await - .unwrap() - .unwrap(); + until("the deferred exit", || { + lifecycle_trace::recorded(trace_id, "DeferExit") + }) + .await; + assert_eq!(early.snapshot().generation, 1); open.send_replace(true); let supervisor = tokio::time::timeout(LIMIT, start) .await .unwrap() .unwrap() .unwrap(); + tokio::time::timeout(LIMIT, early.wait_for_generation(1)) + .await + .unwrap() + .unwrap(); assert_eq!(early.snapshot().generation, 2); assert!(supervisor.shutdown(LIMIT).await.complete); write_trace("restart_during_startup", supervisor.trace_id(), &modes, 5); diff --git a/concurrency/tests/failpoints.rs b/concurrency/tests/failpoints.rs index d60ce257..31f138a3 100644 --- a/concurrency/tests/failpoints.rs +++ b/concurrency/tests/failpoints.rs @@ -124,7 +124,6 @@ impl Handler for Worker { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn factory_errors_restart_within_budget_and_count_generations() { let _scenario = FailScenario::setup(); - failpoints::configure("child.factory", FailPoint::new(Action::Error).times(3)).unwrap(); let system = ActorSystem::new().unwrap(); let group = system.async_group("fp-factory", 1).unwrap(); let started = Started::default(); @@ -141,13 +140,17 @@ async fn factory_errors_restart_within_budget_and_count_generations() { })) .unwrap(); let tree = tree.start().await.unwrap(); - let current = child.wait_ready().await.unwrap(); + // v0.8 (S1): a first start is never retried, so the failures are + // injected into the replacements after a crash. + failpoints::configure("child.factory", FailPoint::new(Action::Error).times(3)).unwrap(); + child.wait_ready().await.unwrap().send(Crash).unwrap(); + let current = child.wait_for_generation(1).await.unwrap(); // Three injected failures, each its own generation, then one success. assert_eq!(failpoints::stats("child.factory").fired, 3); - assert_eq!(child.snapshot().generation, 4); - assert_eq!(constructed.load(Ordering::SeqCst), 1); - assert_eq!(started.handles().len(), 1); - assert_eq!(started.handles()[0].id(), current.id()); + assert_eq!(child.snapshot().generation, 5); + assert_eq!(constructed.load(Ordering::SeqCst), 2); + assert_eq!(started.handles().len(), 2); + assert_eq!(started.handles()[1].id(), current.id()); assert!(tree.failure().is_none()); let report = tree.shutdown(LIMIT).await; assert!(report.complete && report.failure.is_none()); @@ -155,7 +158,7 @@ async fn factory_errors_restart_within_budget_and_count_generations() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn persistent_factory_failure_exhausts_budget_and_fails_startup() { +async fn persistent_factory_failure_fails_startup_without_retry() { let _scenario = FailScenario::setup(); failpoints::configure("child.factory", Action::Error).unwrap(); let system = ActorSystem::new().unwrap(); @@ -170,13 +173,14 @@ async fn persistent_factory_failure_exhausts_budget_and_fails_startup() { })) .unwrap(); match tree.start().await { - Err(SupervisorError::Failed(error)) => { - assert!(error.contains("restart budget exhausted"), "{error}") + Err(SupervisorError::FailedToStartChild { child, cause, .. }) => { + assert_eq!(child, "worker"); + assert!(cause.contains("failpoint child.factory"), "{cause}") } - other => panic!("expected budget failure, got {:?}", other.err()), + other => panic!("expected a first-start failure, got {:?}", other.err()), } - // One initial attempt plus max_restarts retries. - assert_eq!(failpoints::stats("child.factory").fired, 3); + // v0.8 (S1): one attempt; a first start is not retried. + assert_eq!(failpoints::stats("child.factory").fired, 1); assert!(started.handles().is_empty()); assert!(child.current().is_err()); assert!(system.shutdown(LIMIT).await.complete); @@ -185,7 +189,6 @@ async fn persistent_factory_failure_exhausts_budget_and_fails_startup() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn factory_panic_is_a_failed_start_not_a_controller_failure() { let _scenario = FailScenario::setup(); - failpoints::configure("child.factory", FailPoint::new(Action::Panic).times(1)).unwrap(); let system = ActorSystem::new().unwrap(); let group = system.async_group("fp-factory-panic", 1).unwrap(); let started = Started::default(); @@ -198,9 +201,12 @@ async fn factory_panic_is_a_failed_start_not_a_controller_failure() { })) .unwrap(); let tree = tree.start().await.unwrap(); - child.wait_ready().await.unwrap(); + // v0.8 (S1): injected into the replacement; a first start is not retried. + failpoints::configure("child.factory", FailPoint::new(Action::Panic).times(1)).unwrap(); + child.wait_ready().await.unwrap().send(Crash).unwrap(); + child.wait_for_generation(1).await.unwrap(); let snapshot = child.snapshot(); - assert_eq!(snapshot.generation, 2); + assert_eq!(snapshot.generation, 3); assert!(snapshot .last_error .as_deref() @@ -253,11 +259,6 @@ async fn seeded_probabilistic_factory_failures_match_generation_count() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn ready_generation_discarded_before_publication_is_never_current() { let _scenario = FailScenario::setup(); - failpoints::configure( - "child.after_ready_before_publish", - FailPoint::new(Action::Error).times(1), - ) - .unwrap(); let system = ActorSystem::new().unwrap(); let group = system.async_group("fp-discard", 1).unwrap(); let started = Started::default(); @@ -270,17 +271,25 @@ async fn ready_generation_discarded_before_publication_is_never_current() { })) .unwrap(); let tree = tree.start().await.unwrap(); - let current = child.wait_ready().await.unwrap(); + // v0.8 (S1): a discarded first start fails start(), so discard a + // replacement. + failpoints::configure( + "child.after_ready_before_publish", + FailPoint::new(Action::Error).times(1), + ) + .unwrap(); + child.wait_ready().await.unwrap().send(Crash).unwrap(); + let current = child.wait_for_generation(1).await.unwrap(); let handles = started.handles(); - assert_eq!(handles.len(), 2); - assert_ne!(handles[0].id(), current.id()); - assert_eq!(handles[1].id(), current.id()); + assert_eq!(handles.len(), 3); + assert_ne!(handles[1].id(), current.id()); + assert_eq!(handles[2].id(), current.id()); // The discarded generation was stopped, not leaked. - let reason = tokio::time::timeout(LIMIT, handles[0].wait_exit_async()) + let reason = tokio::time::timeout(LIMIT, handles[1].wait_exit_async()) .await .unwrap(); assert_eq!(reason, ExitReason::Shutdown); - assert_eq!(child.snapshot().generation, 2); + assert_eq!(child.snapshot().generation, 3); assert!(tree.shutdown(LIMIT).await.complete); assert!(system.shutdown(LIMIT).await.complete); } @@ -541,20 +550,14 @@ async fn placement_failure_is_reported_and_fails_the_supervisor() { })) .unwrap(); match tree.start().await { - Err(SupervisorError::Failed(error)) => { - assert!(error.contains("placement failed"), "{error}") + Err(SupervisorError::FailedToStartChild { child, cause, .. }) => { + assert_eq!(child, "worker"); + assert!(cause.contains("placement failed"), "{cause}") } other => panic!("expected placement failure, got {:?}", other.err()), } - // start() returns once the supervisor has failed; the child's controller - // publishes its terminal state afterwards. Never started: `Failed` is - // sticky, so the outcome does not depend on whether the unstarted-child - // sweep or the controller finishes last. - let began = Instant::now(); - while child.snapshot().state == ChildState::Waiting { - assert!(began.elapsed() < LIMIT, "child never left Waiting"); - tokio::time::sleep(Duration::from_millis(2)).await; - } + // v0.8 (S2): start() returns after the rollback, so the never-started + // child is already terminal (#223 polled for it). assert_eq!(child.snapshot().state, ChildState::Failed); assert_eq!(child.snapshot().generation, 0); failpoints::remove("system.reserve"); diff --git a/concurrency/tests/nested_supervision.rs b/concurrency/tests/nested_supervision.rs index af648a1a..40ab1f78 100644 --- a/concurrency/tests/nested_supervision.rs +++ b/concurrency/tests/nested_supervision.rs @@ -166,7 +166,9 @@ async fn failed_nested_startup_is_cleaned_before_parent_retries() { "fallible", &placement, move |_| async move { - if attempt == 0 { + // v0.8 (S1): the parent's first start of the nested + // tree is not retried, so its replacement fails. + if attempt == 1 { Err("nested initial construction failed".to_string()) } else { Ok(Counter(0)) @@ -181,8 +183,13 @@ async fn failed_nested_startup_is_cleaned_before_parent_retries() { .await .unwrap() .unwrap(); - assert_eq!(calls.load(Ordering::SeqCst), 2); - assert_eq!(nested.snapshot().generation, 2); + nested.current().unwrap().stop(); + spawned_rt::tasks::timeout(LIMIT, nested.wait_for_generation(2)) + .await + .unwrap() + .unwrap(); + assert_eq!(calls.load(Ordering::SeqCst), 3); + assert_eq!(nested.snapshot().generation, 3); assert!(nested.current().is_ok()); assert!(supervisor.shutdown(LIMIT).await.complete); assert!(system.shutdown(LIMIT).await.complete); diff --git a/concurrency/tests/otp_exit_signals.rs b/concurrency/tests/otp_exit_signals.rs index b603127b..f030c24a 100644 --- a/concurrency/tests/otp_exit_signals.rs +++ b/concurrency/tests/otp_exit_signals.rs @@ -558,6 +558,218 @@ mod tasks_mode { assert_eq!(ended(&child).await, ExitReason::Shutdown); assert!(seen.stopped()); } + + mod startup { + use super::*; + use spawned_concurrency::supervision::{ + ChildSpec, ChildState, RestartMode, SupervisorConfig, SupervisorError, + SupervisorStrategy, + }; + use spawned_concurrency::system::ActorSystem; + use std::sync::atomic::AtomicUsize; + + fn terminal(state: ChildState) -> bool { + matches!(state, ChildState::Stopped | ChildState::Failed) + } + + // S1: a child's first start that fails aborts start() whatever the + // restart type: no retry (v0.7 retried Permanent and Transient + // children and timeouts within the restart budget). + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn first_start_failure_aborts_start_whatever_the_restart_type() { + for (mode, timeout) in [ + (RestartMode::Permanent, false), + (RestartMode::Transient, false), + (RestartMode::Permanent, true), + (RestartMode::Transient, true), + ] { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("s1", 1).unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let seen = Seen::default(); + tree.add(ChildSpec::new("first", &group, move |_| { + let seen = seen.clone(); + async move { Ok::<_, String>(Node::new(&seen, false)) } + })) + .unwrap(); + let counted = calls.clone(); + tree.add( + ChildSpec::new("failing", &group, move |_| { + counted.fetch_add(1, Ordering::SeqCst); + async move { + if timeout { + std::future::pending::<()>().await; + } + Err::("boot failed") + } + }) + .restart(mode) + .startup_timeout(Duration::from_millis(50)), + ) + .unwrap(); + let result = tokio::time::timeout(LIMIT, tree.start()).await.unwrap(); + match result { + Err(SupervisorError::FailedToStartChild { child, .. }) => { + assert_eq!(child, "failing") + } + other => panic!("{mode:?}/{timeout}: unexpected {:?}", other.err()), + } + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "{mode:?}: first start retried" + ); + assert!(system.shutdown(LIMIT).await.complete); + } + } + + // S1: a ready child's exit during the initial start is recorded and + // processed after the tree commits ready, under the strategy (v0.7: + // OneForOne restarted at once, OneForAll/RestForOne failed start()). + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn ready_child_exit_during_startup_is_processed_afterwards() { + for strategy in [ + SupervisorStrategy::OneForOne, + SupervisorStrategy::OneForAll, + SupervisorStrategy::RestForOne, + ] { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("s1", 1).unwrap(); + let config = SupervisorConfig { + strategy, + ..SupervisorConfig::default() + }; + let mut tree = system.supervisor(config).unwrap(); + let seen = Seen::default(); + let first = tree + .add(ChildSpec::new("first", &group, move |_| { + let seen = seen.clone(); + async move { Ok::<_, String>(Node::new(&seen, false)) } + })) + .unwrap(); + let (entered_tx, entered) = tokio::sync::oneshot::channel::<()>(); + let (release, release_rx) = tokio::sync::oneshot::channel::<()>(); + let gate = Arc::new(Mutex::new(Some((entered_tx, release_rx)))); + let second = tree + .add(ChildSpec::new("second", &group, move |_| { + let gate = gate.lock().unwrap().take(); + let seen = Seen::default(); + async move { + if let Some((entered, release)) = gate { + let _ = entered.send(()); + let _ = release.await; + } + Ok::<_, String>(Node::new(&seen, false)) + } + })) + .unwrap(); + let start = tokio::spawn(tree.start()); + entered.await.unwrap(); + let handle = first.current().unwrap().child_handle(); + handle.kill(); + assert_eq!(ended(&handle).await, ExitReason::Killed); + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!( + first.snapshot().generation, + 1, + "{strategy:?}: restarted before the initial start completed" + ); + release.send(()).unwrap(); + let supervisor = tokio::time::timeout(LIMIT, start) + .await + .unwrap() + .unwrap() + .unwrap_or_else(|e| panic!("{strategy:?}: start failed: {e}")); + tokio::time::timeout(LIMIT, first.wait_for_generation(1)) + .await + .unwrap() + .unwrap(); + if strategy == SupervisorStrategy::OneForOne { + assert_eq!(second.snapshot().generation, 1); + } else { + tokio::time::timeout(LIMIT, second.wait_for_generation(1)) + .await + .unwrap() + .unwrap(); + } + assert!(supervisor.shutdown(LIMIT).await.complete); + assert!(system.shutdown(LIMIT).await.complete); + } + } + + // S2: start() returns its error only after the rollback finished: + // every started child is terminal, including one whose stopped() + // gets stuck (killed at shutdown_timeout, then stuck after a second + // interval), whose cleanup failure is listed separately. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn start_returns_its_error_after_rollback() { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("s2", 2).unwrap(); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let slow_seen = Seen::default(); + let s = slow_seen.clone(); + let slow = tree + .add(ChildSpec::new("slow", &group, move |_| { + let seen = s.clone(); + let (tx, rx) = tokio::sync::oneshot::channel(); + // stopped() waits until the sender's timer fires. + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(200)).await; + let _ = tx.send(()); + }); + let (entered, _) = tokio::sync::oneshot::channel(); + async move { + let mut node = Node::new(&seen, true); + node.in_stopped = Some((entered, rx)); + Ok::<_, String>(node) + } + })) + .unwrap(); + let (_hold, hold_rx) = tokio::sync::oneshot::channel::<()>(); + let hold_rx = Arc::new(Mutex::new(Some(hold_rx))); + let stuck = tree + .add( + ChildSpec::new("stuck", &group, move |_| { + let rx = hold_rx.lock().unwrap().take().unwrap(); + let (entered, _) = tokio::sync::oneshot::channel(); + async move { + let mut node = Node::new(&Seen::default(), true); + node.in_stopped = Some((entered, rx)); + Ok::<_, String>(node) + } + }) + .shutdown_timeout(Duration::from_millis(50)), + ) + .unwrap(); + tree.add( + ChildSpec::new("failing", &group, |_| async { + Err::("boot failed") + }) + .restart(RestartMode::Temporary), + ) + .unwrap(); + let result = tokio::time::timeout(LIMIT, tree.start()).await.unwrap(); + assert!(slow_seen.stopped(), "start() returned before rollback"); + assert!(terminal(slow.snapshot().state), "{:?}", slow.snapshot()); + assert_eq!(stuck.snapshot().state, ChildState::Failed); + match result { + Err(SupervisorError::FailedToStartChild { + child, + rollback_failures, + .. + }) => { + assert_eq!(child, "failing"); + assert!( + rollback_failures.iter().any(|f| f.contains("stuck")), + "{rollback_failures:?}" + ); + } + other => panic!("unexpected {:?}", other.err()), + } + let _ = system.shutdown(Duration::from_millis(100)).await; + } + } } mod threads_mode { diff --git a/concurrency/tests/panic_safety.rs b/concurrency/tests/panic_safety.rs index ce07ec78..14800018 100644 --- a/concurrency/tests/panic_safety.rs +++ b/concurrency/tests/panic_safety.rs @@ -909,17 +909,21 @@ mod tasks_mode { let child_gate = gate.clone(); let child = tree .add(ChildSpec::new("victim", &group, move |ctx| { - let fault = (ctx.generation() == 1).then_some(fault); + // v0.8 (S1): a first start is not retried, so a + // started() panic is injected into the replacement. + let faulty = if fault == Fault::Started { 2 } else { 1 }; + let fault = (ctx.generation() == faulty).then_some(fault); let victim = TaskVictim(Victim::new(fault, &child_gate)); async move { Ok::<_, String>(victim) } })) .unwrap(); let supervisor = env.bounded(tree.start()).unwrap(); if fault == Fault::Started { - // The first generation never became ready; the retry did. - let next = env.bounded(child.wait_ready()).unwrap(); + // The replacement never became ready; its retry did. + child.current().unwrap().child_handle().kill(); + let next = env.bounded(child.wait_for_generation(2)).unwrap(); assert_eq!(env.bounded(next.request(Ask)).unwrap(), 1); - assert_eq!(child.snapshot().generation, 2); + assert_eq!(child.snapshot().generation, 3); assert_eq!(child.snapshot().last_exit, Some(expected(fault))); } else { let first = child.current().unwrap(); @@ -964,7 +968,9 @@ mod tasks_mode { let mut tree = env.system.as_ref().unwrap().supervisor(config()).unwrap(); let child = tree .add(ChildSpec::new("factory", &group, move |ctx| { - let first = ctx.generation() == 1; + // v0.8 (S1): a first start is not retried, so the + // panic is injected into the replacement. + let first = ctx.generation() == 2; if first && !in_future { panic!("injected factory panic"); } @@ -978,10 +984,11 @@ mod tasks_mode { })) .unwrap(); let supervisor = env.bounded(tree.start()).unwrap(); - let actor = child.current().unwrap(); + child.current().unwrap().child_handle().kill(); + let actor = env.bounded(child.wait_for_generation(2)).unwrap(); assert_eq!(env.bounded(actor.request(Ask)).unwrap(), 1); let snapshot = child.snapshot(); - assert_eq!(snapshot.generation, 2, "{place:?} in_future={in_future}"); + assert_eq!(snapshot.generation, 3, "{place:?} in_future={in_future}"); assert!(snapshot .last_error .as_deref() diff --git a/concurrency/tests/supervision.rs b/concurrency/tests/supervision.rs index 45ac247f..7d02a14c 100644 --- a/concurrency/tests/supervision.rs +++ b/concurrency/tests/supervision.rs @@ -231,9 +231,11 @@ async fn failed_factories_cancel_owned_helpers_before_retry_and_catch_panics() { }) .unwrap(); drop(helper); + // v0.8 (S1): a first start is never retried, so the failed + // constructions are replacements after a crash. match attempt { - 0 => Err("injected construction error".to_string()), - 1 => panic!("injected factory panic"), + 1 => Err("injected construction error".to_string()), + 2 => panic!("injected factory panic"), _ => Ok(Counter(0)), } } @@ -243,9 +245,14 @@ async fn failed_factories_cancel_owned_helpers_before_retry_and_catch_panics() { .await .unwrap() .unwrap(); - assert_eq!(attempts.load(Ordering::SeqCst), 3); - assert_eq!(child.snapshot().generation, 3); - assert_eq!(child.current().unwrap().request(Next).await.unwrap(), 1); + assert!(child.current().unwrap().request(Crash).await.is_err()); + let replacement = spawned_rt::tasks::timeout(LIMIT, child.wait_for_generation(1)) + .await + .unwrap() + .unwrap(); + assert_eq!(attempts.load(Ordering::SeqCst), 4); + assert_eq!(child.snapshot().generation, 4); + assert_eq!(replacement.request(Next).await.unwrap(), 1); assert!(supervisor.shutdown(LIMIT).await.complete); assert!(!live.load(Ordering::SeqCst)); assert!(system.shutdown(LIMIT).await.complete); @@ -1035,8 +1042,10 @@ async fn transient_startup_timeout_retries_after_actual_cleanup() { ChildSpec::new("slow-init", &group, move |ctx| { attempts.fetch_add(1, Ordering::SeqCst); async move { + // v0.8 (S1): a first start is never retried, so the + // slow start is the first replacement's. Ok::<_, String>(SlowInit { - first: ctx.generation() == 1, + first: ctx.generation() == 2, }) } }) @@ -1048,8 +1057,13 @@ async fn transient_startup_timeout_retries_after_actual_cleanup() { .await .unwrap() .unwrap(); - assert_eq!(calls.load(Ordering::SeqCst), 2); - assert_eq!(child.snapshot().generation, 2); + child.current().unwrap().child_handle().kill(); + spawned_rt::tasks::timeout(LIMIT, child.wait_for_generation(2)) + .await + .unwrap() + .unwrap(); + assert_eq!(calls.load(Ordering::SeqCst), 3); + assert_eq!(child.snapshot().generation, 3); assert_eq!(child.snapshot().last_exit, Some(ExitReason::Shutdown)); assert!(child.current().is_ok()); assert!(supervisor.shutdown(LIMIT).await.complete); diff --git a/concurrency/tests/supervision_strategies.rs b/concurrency/tests/supervision_strategies.rs index a93941e9..03ecdd8c 100644 --- a/concurrency/tests/supervision_strategies.rs +++ b/concurrency/tests/supervision_strategies.rs @@ -3,13 +3,15 @@ #![cfg_attr(spawned_sim, allow(unused_imports, dead_code))] use spawned_concurrency::{ message::Message, - supervision::{Backoff, ChildSpec, RestartMode, SupervisorConfig, SupervisorStrategy}, + supervision::{ + Backoff, ChildSpec, RestartMode, SupervisorConfig, SupervisorError, SupervisorStrategy, + }, system::ActorSystem, tasks::{Actor, Context, Handler}, }; use std::{ sync::{ - atomic::{AtomicUsize, Ordering}, + atomic::{AtomicBool, AtomicUsize, Ordering}, Arc, Mutex, }, time::Duration, @@ -602,7 +604,11 @@ async fn upstream_failure_supersedes_a_round_while_a_dependent_factory_is_pendin } #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] -async fn ready_dependency_failure_rolls_back_slow_initial_tree_startup() { +/// S1: a ready dependency's exit during the initial start is deferred: the +/// start continues (v0.7 rolled back at once), the dependency is not replaced +/// alone, and when a later first start fails the deferred exit is part of the +/// rollback. +async fn ready_dependency_exit_is_deferred_during_slow_initial_tree_startup() { for strategy in [ SupervisorStrategy::OneForAll, SupervisorStrategy::RestForOne, @@ -639,21 +645,41 @@ async fn ready_dependency_failure_rolls_back_slow_initial_tree_startup() { }) } }) - .startup_timeout(Duration::from_secs(30)), + .startup_timeout(if index == 2 { + Duration::from_millis(500) + } else { + Duration::from_secs(30) + }), ) .unwrap(), ); } - let startup = spawned_rt::tasks::spawn(tree.start()); + let finished = Arc::new(AtomicBool::new(false)); + let flag = finished.clone(); + let startup = spawned_rt::tasks::spawn(async move { + let result = tree.start().await; + flag.store(true, Ordering::SeqCst); + result + }); spawned_rt::tasks::timeout(LIMIT, entered.notified()) .await .unwrap(); assert!(children[0].current().unwrap().request(Crash).await.is_err()); - assert!(spawned_rt::tasks::timeout(LIMIT, startup) + spawned_rt::tasks::sleep(Duration::from_millis(100)).await; + assert!( + !finished.load(Ordering::SeqCst), + "{strategy:?}: start rolled back at once" + ); + match spawned_rt::tasks::timeout(LIMIT, startup) .await .unwrap() .unwrap() - .is_err()); + { + Err(SupervisorError::FailedToStartChild { child, .. }) => { + assert_eq!(child, "startup-dependency-2") + } + other => panic!("{strategy:?}: {:?}", other.err()), + } assert_eq!( children[0].snapshot().generation, 1, diff --git a/concurrency/tests/supervision_trace.rs b/concurrency/tests/supervision_trace.rs index 8d45e131..072abe8e 100644 --- a/concurrency/tests/supervision_trace.rs +++ b/concurrency/tests/supervision_trace.rs @@ -400,6 +400,88 @@ async fn trace_external_shutdown_of_permanent_child_restarts() { assert!(system.shutdown(LIMIT).await.complete); } +/// S1: a Permanent child whose first construction fails aborts the start (v0.7 +/// retried it); the model's Decide fails because the child never completed +/// its first start. +#[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] +async fn trace_first_start_factory_error_fails_start() { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("trace-factory-first", 1).unwrap(); + let modes = [RestartMode::Permanent, RestartMode::Permanent]; + let mut tree = system.supervisor(config(5)).unwrap(); + tree.add(worker("a", &group).restart(modes[0])).unwrap(); + tree.add( + ChildSpec::new("flaky", &group, |_| async { + Err::("first construction fails") + }) + .restart(modes[1]), + ) + .unwrap(); + let trace_id = tree.trace_id(); + assert!(spawned_rt::tasks::timeout(LIMIT, tree.start()) + .await + .unwrap() + .is_err()); + assert!(system.shutdown(LIMIT).await.complete); + write_trace("first_start_factory_error", trace_id, &modes, 5); +} + +/// S1: a ready child's exit during the initial start is deferred (DeferExit) +/// and processed once the tree commits ready (Undefer, then the restart). +#[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] +async fn trace_ready_exit_deferred_during_startup() { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("trace-deferred", 2).unwrap(); + let modes = [RestartMode::Permanent, RestartMode::Permanent]; + let mut tree = system.supervisor(config(5)).unwrap(); + let a = tree.add(worker("a", &group).restart(modes[0])).unwrap(); + let entered = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let (e, r) = (entered.clone(), release.clone()); + tree.add( + ChildSpec::new("b", &group, move |ctx| { + let (e, r) = (e.clone(), r.clone()); + async move { + if ctx.generation() == 1 { + e.notify_one(); + r.notified().await; + } + Ok::<_, String>(Worker) + } + }) + .restart(modes[1]), + ) + .unwrap(); + let trace_id = tree.trace_id(); + let start = spawned_rt::tasks::spawn(tree.start()); + spawned_rt::tasks::timeout(LIMIT, entered.notified()) + .await + .unwrap(); + crash(&a).await; + // The exit is recorded before the start completes. + spawned_rt::tasks::timeout(LIMIT, async { + while !lifecycle_trace::recorded(trace_id, "DeferExit") { + spawned_rt::tasks::sleep(Duration::from_millis(1)).await; + } + }) + .await + .unwrap(); + release.notify_one(); + let supervisor = spawned_rt::tasks::timeout(LIMIT, start) + .await + .unwrap() + .unwrap() + .unwrap(); + spawned_rt::tasks::timeout(LIMIT, a.wait_for_generation(1)) + .await + .unwrap() + .unwrap(); + assert!(supervisor.shutdown(LIMIT).await.complete); + write_trace("ready_exit_deferred", trace_id, &modes, 5); + assert!(system.shutdown(LIMIT).await.complete); +} + +/// A replacement whose construction fails is retried within the budget. #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] async fn trace_factory_error_then_success() { let system = ActorSystem::new().unwrap(); @@ -408,19 +490,25 @@ async fn trace_factory_error_then_success() { let calls = Arc::new(AtomicUsize::new(0)); let mut tree = system.supervisor(config(5)).unwrap(); let count = calls.clone(); - tree.add(ChildSpec::new("flaky", &group, move |_| { - let attempt = count.fetch_add(1, Ordering::SeqCst); - async move { - if attempt == 0 { - Err("first construction fails".to_string()) - } else { - Ok(Worker) + let flaky = tree + .add(ChildSpec::new("flaky", &group, move |_| { + let attempt = count.fetch_add(1, Ordering::SeqCst); + async move { + if attempt == 1 { + Err("replacement construction fails".to_string()) + } else { + Ok(Worker) + } } - } - })) - .unwrap(); + })) + .unwrap(); let supervisor = tree.start().await.unwrap(); - assert_eq!(calls.load(Ordering::SeqCst), 2); + crash(&flaky).await; + spawned_rt::tasks::timeout(LIMIT, flaky.wait_for_generation(2)) + .await + .unwrap() + .unwrap(); + assert_eq!(calls.load(Ordering::SeqCst), 3); assert!(supervisor.shutdown(LIMIT).await.complete); write_trace( "factory_error_then_success", @@ -600,26 +688,58 @@ impl Actor for SlowInit { } } } +/// S1: a first start that misses startup_timeout aborts the start, whatever +/// the restart type (v0.7 retried a Transient child). #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] -async fn trace_startup_timeout_retries() { +async fn trace_first_start_timeout_fails_start() { let system = ActorSystem::new().unwrap(); - let group = system.async_group("trace-timeout", 1).unwrap(); + let group = system.async_group("trace-timeout-first", 1).unwrap(); let modes = [RestartMode::Transient]; let mut tree = system.supervisor(config(3)).unwrap(); tree.add( - ChildSpec::new("slow-init", &group, |ctx| async move { - Ok::<_, String>(SlowInit { - first: ctx.generation() == 1, - }) + ChildSpec::new("slow-init", &group, |_| async move { + Ok::<_, String>(SlowInit { first: true }) }) .restart(modes[0]) .startup_timeout(Duration::from_millis(20)), ) .unwrap(); + let trace_id = tree.trace_id(); + assert!(spawned_rt::tasks::timeout(LIMIT, tree.start()) + .await + .unwrap() + .is_err()); + assert!(system.shutdown(LIMIT).await.complete); + write_trace("first_start_timeout", trace_id, &modes, 3); +} + +/// A replacement that misses startup_timeout is retried. +#[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] +async fn trace_startup_timeout_retries() { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("trace-timeout", 1).unwrap(); + let modes = [RestartMode::Transient]; + let mut tree = system.supervisor(config(3)).unwrap(); + let child = tree + .add( + ChildSpec::new("slow-init", &group, |ctx| async move { + Ok::<_, String>(SlowInit { + first: ctx.generation() == 2, + }) + }) + .restart(modes[0]) + .startup_timeout(Duration::from_millis(20)), + ) + .unwrap(); let supervisor = spawned_rt::tasks::timeout(LIMIT, tree.start()) .await .unwrap() .unwrap(); + child.current().unwrap().child_handle().kill(); + spawned_rt::tasks::timeout(LIMIT, child.wait_for_generation(2)) + .await + .unwrap() + .unwrap(); assert!(supervisor.shutdown(LIMIT).await.complete); write_trace("startup_timeout", supervisor.trace_id(), &modes, 3); assert!(system.shutdown(LIMIT).await.complete); diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 088fa38e..419b1ccf 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -81,9 +81,18 @@ table is the full list). What to edit: (`ChildHandle::exit`): `Normal` is ignored by a non-trapping target, `Panic(msg)` ends it with exactly `Panic(msg)`, `Kill` ends it `Killed`, a trapping target receives an `Exit`. -- **Supervisor start (rank 5f, not implemented yet in this branch):** any - first-start failure of a child will fail `SupervisorBuilder::start()`, - whatever its restart type, with no opt-in retry (decision 7). **ethrex**: +- **Supervisor start (rank 5f):** any first-start failure of a child (factory + error or panic, panic or stop in `started()`, `startup_timeout`, placement) + fails `SupervisorBuilder::start()`, whatever its restart type, with no + opt-in retry (decision 7), as `SupervisorError::FailedToStartChild { child, + cause, rollback_failures }`; match on it (or keep treating any `Err` as a + failed start). Replacements still retry within the restart budget. A ready + child that exits while later siblings start is no longer restarted at once + (OneForOne) or fatal (OneForAll/RestForOne): its exit is processed after + the initial start completes. `start()` now returns its error after the + rollback, so it can take up to two `shutdown_timeout`s per started child; a + child whose cleanup is stuck is reported `Failed` and its work stays owned + by the system. **ethrex**: the Transient discovery child's factory (`DiscoveryServer::build`) can fail at boot when pinging a bootnode fails; v0.7 retried it, v0.8 fails `start()` and `init_network` panics on `.expect("Network starts")`. Fix in diff --git a/docs/design/semantics.md b/docs/design/semantics.md index 99dc9e41..fd460cd6 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -377,16 +377,21 @@ Checked by: Kani `backoff_bound_is_capped`, `backoff_jitter_stays_in_range`, [`supervision::backoff_exponent_resets_only_after_a_stable_generation`][supervision]. **V7. Startup.** Children start serially in declaration order. A tree reports -ready only when every child is ready; a non-restartable failure (or exhausted -retries) rolls back the started children in reverse order. Each attempt has a -`startup_timeout` (default 5 s) that counts as a failure. A child that stops -`Normal` (or `Shutdown`) before its initial readiness is a startup failure and, -if Transient, fails the start, as a failed child `init` fails an OTP -supervisor's start. A replacement that stops before readiness is a failed -restart attempt and is retried (V1). Factories run on the child's -group; a factory error or panic is a failed attempt. Cancelling the start future -before readiness rolls the tree back. -Checked by: `Supervision!ReadyMeansAllStarted`; [`supervision::failed_partial_startup_rolls_back_without_starting_later_children`][supervision], +ready only when every child is ready. A child's first start that fails for +any reason (factory error or panic, panic or stop in `started()`, a missed +`startup_timeout`, default 5 s, placement or group closure) fails the start +whatever the restart type, with no retry, as OTP's `start_children` returns +`failed_to_start_child`: the started children roll back in reverse order and +`start()` returns `SupervisorError::FailedToStartChild` only after the +rollback (every child terminal, or `Failed` once its cleanup is stuck). A +ready child's exit while later siblings start is recorded and processed under +the strategy once the tree commits ready, as OTP handles the queued `'EXIT'` +after `init`; if the start aborts, it is part of the rollback. A replacement +that fails before readiness is a failed restart attempt and is retried (V1). +Factories run on the child's group. Cancelling the start future before +readiness rolls the tree back. +Checked by: `Supervision!ReadyMeansAllStarted`, `Supervision!RestartFollowsPolicy` (no restart before the tree is ready), `Cohorts!FirstStartOnce`; [`supervision::failed_partial_startup_rolls_back_without_starting_later_children`][supervision], +`otp_exit_signals::first_start_failure_aborts_start_whatever_the_restart_type`, `ready_child_exit_during_startup_is_processed_afterwards`, `start_returns_its_error_after_rollback`, `transient_startup_timeout_retries_after_actual_cleanup`, `normal_pre_ready_exit_records_real_cause_and_fails_transient_startup`, `transient_replacement_exiting_before_readiness_is_retried`, [`supervision_trace::trace_replacement_exiting_before_readiness_is_retried`][supervision_trace], `failed_factories_cancel_owned_helpers_before_retry_and_catch_panics`, @@ -572,7 +577,7 @@ ignored by non-trapping peers (X3), and the Permanent/Transient/Temporary table | A kill propagated over a link stays `Kill` (X4) | It arrives as `killed` | Planned v0.8, rank 5e: a distinct trappable `Killed` | | Linking or monitoring an already-exited actor delivers its real exit reason (X1, X5) | `noproc` | Planned v0.8, rank 5e: `NoProc` | | The restart budget counts every child factory call, collateral ones included (V5) | Intensity counts recovery rounds | Planned v0.8, rank 5f | -| Initial startup retries a Permanent or Transient child whose factory fails, panics or times out (V7), within the budget | `start_link` aborts on any child start error | Planned v0.8, rank 5f | +| Initial startup retried a Permanent or Transient child whose factory failed, panicked or timed out (V7), within the budget | `start_link` aborts on any child start error | Aligned in v0.8 (rank 5f, #240): any first-start failure fails `start()` | | An actor dropped by its runtime exits `Kill` in tasks; the threads fallback is `Panic("unexpected framework panic")` (L4) | No counterpart | Planned v0.8, rank 5g: one reason in both modes | | Stop is cooperative: a running handler or blocking call is never interrupted (L3) | `kill` ends a process at once | Rust-forced: Rust cannot preempt or kill a thread | | Panics are caught only with unwinding (P2) | Process crashes are always isolated | Rust-forced: `panic = "abort"` and a panicking panic hook end the process | diff --git a/verification/tla/Cohorts.tla b/verification/tla/Cohorts.tla index 9d1ea42f..d4235420 100644 --- a/verification/tla/Cohorts.tla +++ b/verification/tla/Cohorts.tla @@ -97,9 +97,13 @@ CResume(c,r) == \* requests a new round (CRequest), which OneForAll applies to the whole \* cohort, as OTP's try_again_restart re-runs the strategy. Fault \* "InPlaceRetry" restores the old in-round retry of that child alone. +\* Round 0 (the first start) begins once per child: a failed first start +\* fails the supervisor's start (S1, otp-exit-signals.md). Fault +\* "FirstStartRetry" restores v0.7's retry of it. CBegin(c,r,n) == /\ c \in Active /\ n = s.gen[c] + 1 /\ n <= MaxGen /\ s.open \/ Fault = "Closed" + /\ r = 0 => n = 1 \/ Fault = "FirstStartRetry" /\ \/ /\ r = 0 /\ ~s.committed /\ c \notin s.initial /\ \A j \in Children : j < c => j \in s.initial \/ /\ r = s.assigned[c] /\ r > 0 /\ c \in s.resumed /\ c \notin s.ack @@ -163,10 +167,13 @@ OneStartPerRound == ~s.badRetry NoStartAfterClose == ~s.badClosed NoRetiredParticipant == ~s.badTerminal TemporaryStartsOnce == \A c \in Children : Mode[c] = "Temporary" => s.gen[c] <= 1 +\* S1: a child that never completed its first start had one attempt. +FirstStartOnce == \A c \in Children : c \notin s.initial => s.gen[c] <= 1 DoneMeansQuiescent == s.phase = "Done" => \A c \in Children : s.live[c] = {} Safety == TypeOK /\ AtMostOneGeneration /\ PublishedIsLive /\ StrategySelection /\ WholeCohortBarrier /\ NoStartAfterClose /\ NoRetiredParticipant /\ TemporaryStartsOnce /\ DoneMeansQuiescent /\ OneStartPerRound + /\ FirstStartOnce Next == \/ \E c \in Children, r \in Epochs, x \in {"true","false"} : CRequest(c,r,x) \/ \E c \in Children, r \in Epochs, n \in Children : CSelect(c,r,n) diff --git a/verification/tla/CohortsInd.tla b/verification/tla/CohortsInd.tla index 9212617e..88ee83b5 100644 --- a/verification/tla/CohortsInd.tla +++ b/verification/tla/CohortsInd.tla @@ -46,7 +46,17 @@ H_barrier == \* false. H_done == s.phase = "Done" => ~s.open -IndInv == TypeInd /\ H_barrier /\ H_done /\ Safety +\* S1: recovery rounds exist only after the tree committed ready, which +\* needs every child's first start. CRequest needs committed; CSelect needs +\* a pending request; CAssign assigns the selected round. So a round-r > 0 +\* CBegin never runs for a child without its first start (FirstStartOnce). +H_first == + /\ s.committed => s.initial = Children + /\ s.pending # {} => s.committed + /\ s.epoch > 0 => s.committed + /\ \A c \in Children : s.assigned[c] > 0 => s.committed + +IndInv == TypeInd /\ H_barrier /\ H_done /\ H_first /\ Safety \* Probe (apalache.sh requires it violated): IndInv admits a mid-round state \* with live generations, i.e. the step check is not over an empty set. diff --git a/verification/tla/CovSupervision.tla b/verification/tla/CovSupervision.tla index a8cbe491..78a12e69 100644 --- a/verification/tla/CovSupervision.tla +++ b/verification/tla/CovSupervision.tla @@ -42,6 +42,8 @@ CovNext == \/ StopObserved(c) /\ L("StopObserved", c) \/ CleanupDone(c) /\ L("CleanupDone", c) \/ CleanupStuck(c) /\ L("CleanupStuck", c) + \/ DeferExit(c) /\ L("DeferExit", c) + \/ Undefer(c) /\ L("Undefer", c) \/ Decide(c) /\ L("Decide", c) \/ Finish(c) /\ L("Finish", c) diff --git a/verification/tla/README.md b/verification/tla/README.md index 0ba1bd63..ddd15193 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -54,8 +54,11 @@ keeps the generated modules and TLC logs. Scenarios: normal start and stop of three children with mixed modes; Permanent and Transient panics and restarts; Transient normal exit (no restart); Temporary exit; `ChildHandle::shutdown` of running children; -factory error then success; non-restartable startup failure; restart budget -exhaustion and disabled retries; shutdown during backoff; startup timeout; +replacement factory error then success; first-start factory error and +first-start timeout (both fail the start, S1); a ready child's exit deferred +during the initial start (`DeferExit`/`Undefer`, S1); non-restartable startup +failure; restart budget exhaustion and disabled retries; shutdown during +backoff; replacement startup timeout; `Supervisor::shutdown` with a short deadline and a hanging `stopped()` (`ShutdownDeadline`, `SupChildTimeout`, `CleanupStuck`); and seeded random crashes, normal exits and external shutdowns racing a stop or shutdown. @@ -373,7 +376,7 @@ and `--negative-controls` removes one guard at a time and requires induction to fail: for `Supervision` the retry limit, ordered start, join wait, restart table, cleanup, unpublish-on-stop, admission gate on `Attempt` and the reverse-shutdown wait; for `Cohorts` and `NestedSupervision` the existing -`Fault` mutations (`Selection`, `Cleanup`, `Retirement`, `Closed`, `InPlaceRetry`, `Join`); +`Fault` mutations (`Selection`, `Cleanup`, `Retirement`, `Closed`, `InPlaceRetry`, `FirstStartRetry`, `Join`); for `Coordinator` see below. The type annotations are `\* @type:` comments, ignored by TLC. Apalache also @@ -536,6 +539,9 @@ exactly the property it targets: | Model (configuration) | Mutation | Breaks | |---|---|---| +| `Supervision` (`MCSupervision`, a rewrite of `Supervision.tla`: no `Fault` constant) | `FirstStartRetry` (v0.7 retried a failed first start) | `RestartFollowsPolicy` | +| `Supervision` (`MCSupervision`, rewrite) | `NoDeferral` (v0.7 decided on a ready child's exit during the initial start) | `RestartFollowsPolicy` | +| `Cohorts` (`MCCohortsAll`) | `FirstStartRetry` | `FirstStartOnce` (in `Safety`) | | `DeferredReplies` | `NoAutoReply` | `NoOrphan` | | `DeferredReplies` | `NoAtomicExpiry` (abstract double discharge; the real race needs Loom) | `AtMostOneDischarge` | | `DeferredReplies` | `LateSuccess` | `NoSuccessAfterDeadline` | diff --git a/verification/tla/Supervision.tla b/verification/tla/Supervision.tla index 235852aa..cfc34a87 100644 --- a/verification/tla/Supervision.tla +++ b/verification/tla/Supervision.tla @@ -52,17 +52,18 @@ Restarts(m, r) == ELSE \/ m = "Permanent" \/ m = "Transient" /\ r \in {"none", "Panic", "Kill"} -\* Whether a controller retries after cleanup. A generation that never became -\* ready after the child's initial readiness is a failed restart attempt, -\* retried whatever its exit reason (as OTP retries a failed restart), unless -\* Temporary. Before initial readiness the restart table applies, so a -\* Transient child stopping Normal or Shutdown in its first startup fails it. +\* Whether a controller retries after cleanup. A child's first start that +\* fails (init FALSE) is never retried, whatever its restart type: it fails +\* the supervisor's start (S1 in docs/design/otp-exit-signals.md, OTP's +\* failed_to_start_child). After it, the restart table applies, and a +\* generation that never became ready is a failed restart attempt, retried +\* whatever its exit reason (as OTP retries a failed restart), unless +\* Temporary. Retries(m, r, ready, init) == - \/ Restarts(m, r) - \/ ~ready /\ init /\ m # "Temporary" + init /\ (Restarts(m, r) \/ (~ready /\ m # "Temporary")) Pcs == {"idle", "top", "backoff", "attempt", "constructing", "running", - "cleanup", "decide", "finish", "finished"} + "cleanup", "decide", "deferred", "finish", "finished"} VARIABLES \* @type: Bool; @@ -292,8 +293,25 @@ CleanupStuck(c) == /\ UNCHANGED <> +\* S1: a ready child's exit during the initial start is recorded; its +\* restart decision waits until the tree commits ready (or the start +\* aborts and the exit is part of the rollback), as OTP handles the queued +\* 'EXIT' after init returns. Cleanup already ran. +DeferExit(c) == + /\ pc[c] = "decide" + /\ open /\ ~sready /\ readyGen[c] + /\ Goto(c, "deferred") + /\ UNCHANGED <> + +Undefer(c) == + /\ pc[c] = "deferred" + /\ sready \/ ~open + /\ Goto(c, "decide") + /\ UNCHANGED <> + Decide(c) == /\ pc[c] = "decide" + /\ ~(open /\ ~sready /\ readyGen[c]) /\ IF ~open THEN Goto(c, "finish") /\ UNCHANGED <> ELSE IF Retries(Mode[c], exitR[c], readyGen[c], initial[c]) THEN Goto(c, "top") /\ UNCHANGED <> ELSE IF ~readyGen[c] THEN Fail /\ Goto(c, "finish") \* non-restartable startup failure @@ -381,7 +399,7 @@ ChildStep(c) == \/ \E r \in StartupResults : StartupFailed(c, r) \/ \E r \in SelfExits : SelfExit(c, r) \/ StopObserved(c) \/ CleanupDone(c) \/ CleanupStuck(c) - \/ Decide(c) \/ Finish(c) + \/ DeferExit(c) \/ Undefer(c) \/ Decide(c) \/ Finish(c) SupStep == \/ SupLaunch \/ SupStartDone \/ SupCancelSeen \/ SupStopTurn @@ -399,6 +417,7 @@ Fairness == /\ WF_vars(RetryReserve(c)) /\ WF_vars(RetryRelease(c)) /\ WF_vars(LoopTop(c)) /\ WF_vars(BackoffDone(c)) /\ WF_vars(Attempt(c)) /\ WF_vars(StopObserved(c)) /\ WF_vars(CleanupDone(c)) + /\ WF_vars(DeferExit(c)) /\ WF_vars(Undefer(c)) /\ WF_vars(Decide(c)) /\ WF_vars(Finish(c)) /\ WF_vars(\E r \in {"Shutdown"} : StartupFailed(c, r)) /\ WF_vars(SupLaunch) /\ WF_vars(SupStartDone) /\ WF_vars(SupCancelSeen) @@ -441,24 +460,29 @@ Safety == NoStartAfterClose == [][\A c \in Children : (gen'[c] > gen[c] \/ (pub'[c] # 0 /\ pub[c] = 0)) => open]_vars -\* The design's restart table (docs/design/execution-groups-supervision.md), -\* stated independently of the Restarts operator the actions use: -\* Permanent restarts after any exit, Transient only after an abnormal exit -\* or failed startup, Temporary never. +\* The design's restart table (docs/design/execution-groups-supervision.md, +\* with S1 of otp-exit-signals.md), stated independently of the Restarts +\* operator the actions use: Permanent restarts after any exit, Transient +\* only after an abnormal exit, Temporary never. A failed startup ("none", +\* "timeout") is restarted only as a failed restart attempt below; a first +\* start is never retried. DesignAllowsRestart(m, r) == CASE m = "Temporary" -> FALSE [] m = "Permanent" -> TRUE - [] m = "Transient" -> r \in {"none", "timeout", "Panic", "Kill"} - -\* A new generation after the first only follows an exit for which the -\* design table allows a restart, or a failed restart attempt (a generation -\* that never became ready after the child's initial readiness) of a child -\* that is not Temporary. + [] m = "Transient" -> r \in {"Panic", "Kill"} + +\* A new generation after the first happens only after the tree committed +\* ready (S1: with ReadyMeansAllStarted, no first start is retried and no +\* exit during the initial start is processed before it completes), and +\* only follows an exit for which the design table allows a restart, or a +\* failed restart attempt (a generation that never became ready after the +\* child's initial readiness) of a child that is not Temporary. RestartFollowsPolicy == [][\A c \in Children : (gen'[c] > gen[c] /\ gen[c] > 0) => - \/ DesignAllowsRestart(Mode[c], exitR[c]) - \/ ~readyGen[c] /\ initial[c] /\ Mode[c] # "Temporary"]_vars + /\ sready + /\ \/ DesignAllowsRestart(Mode[c], exitR[c]) + \/ ~readyGen[c] /\ initial[c] /\ Mode[c] # "Temporary"]_vars \* Ready children are stopped in reverse declaration order: a stop turn is \* only given once every later child finished or was abandoned, unless the diff --git a/verification/tla/SupervisionCoverage.tla b/verification/tla/SupervisionCoverage.tla index 81931226..26c62a4b 100644 --- a/verification/tla/SupervisionCoverage.tla +++ b/verification/tla/SupervisionCoverage.tla @@ -26,7 +26,8 @@ EXTENDS Supervision, TLC ChildActions == {"LoopTop", "RetryReserve", "RetryRelease", "BackoffDone", "Attempt", "BecomeReady", "StartupFailed", "SelfExit", - "StopObserved", "CleanupDone", "CleanupStuck", "Decide", "Finish"} + "StopObserved", "CleanupDone", "CleanupStuck", "DeferExit", "Undefer", + "Decide", "Finish"} Bucket(b) == IF b = 0 THEN "0" ELSE IF b >= MaxRestarts THEN "max" ELSE "mid" diff --git a/verification/tla/SupervisionInd.tla b/verification/tla/SupervisionInd.tla index 7893dc71..81a879f2 100644 --- a/verification/tla/SupervisionInd.tla +++ b/verification/tla/SupervisionInd.tla @@ -84,7 +84,18 @@ H_initial == H_retry == \A c \in Children : /\ pc[c] \in {"top", "backoff", "attempt"} /\ gen[c] > 0 => Retries(Mode[c], exitR[c], readyGen[c], initial[c]) - /\ pc[c] = "decide" => gen[c] > 0 + /\ pc[c] \in {"decide", "deferred"} => gen[c] > 0 + +\* S1: nothing restarts before the tree commits ready. Decide sends a +\* controller back to "top" only with initial[c] (Retries); before sready a +\* child has at most its first generation, whose readiness is initial[c], +\* so Decide's deferral guard (open, ~sready, readyGen) leaves no restart. +\* A deferred controller holds a ready generation's exit. +H_first == \A c \in Children : + /\ gen[c] <= 1 /\ initial[c] => readyGen[c] + /\ gen[c] > 1 => sready + /\ pc[c] \in {"top", "backoff", "attempt"} /\ gen[c] > 0 => sready + /\ pc[c] = "deferred" => readyGen[c] \* Reverse-order shutdown: while stopping, every child after sidx has \* finished or was abandoned on timeout (cdone and timedOut only grow). @@ -115,7 +126,7 @@ SafetyInd == Helpers == /\ H_gen /\ H_launch /\ H_initial - /\ H_retry /\ H_shutdown /\ H_done /\ H_budget + /\ H_retry /\ H_first /\ H_shutdown /\ H_done /\ H_budget IndInv == TypeInd /\ H_alive /\ H_pub /\ Helpers /\ SafetyInd @@ -128,8 +139,9 @@ NoStartAfterCloseStep == RestartFollowsPolicyStep == \A c \in Children : (gen'[c] > gen[c] /\ gen[c] > 0) => - \/ DesignAllowsRestart(Mode[c], exitR[c]) - \/ ~readyGen[c] /\ initial[c] /\ Mode[c] # "Temporary" + /\ sready + /\ \/ DesignAllowsRestart(Mode[c], exitR[c]) + \/ ~readyGen[c] /\ initial[c] /\ Mode[c] # "Temporary" ReverseShutdownStep == \A c \in Children : (stopTurn'[c] /\ ~stopTurn[c] /\ ~deadline') => diff --git a/verification/tla/TraceSupervision.tla b/verification/tla/TraceSupervision.tla index 9760f787..a8aa5d2d 100644 --- a/verification/tla/TraceSupervision.tla +++ b/verification/tla/TraceSupervision.tla @@ -33,7 +33,7 @@ Bool(x) == x = "true" ChildEvents == {"RetryReserve", "RetryRelease", "LoopTop", "BackoffDone", "Attempt", "BecomeReady", "StartupFailed", "SelfExit", "StopObserved", "CleanupDone", - "CleanupStuck", "Decide", "Finish"} + "CleanupStuck", "DeferExit", "Undefer", "Decide", "Finish"} \* The model action named by logged event e, constrained by its parameters. Step(e) == @@ -70,6 +70,8 @@ Step(e) == [] e.a = "StopObserved" -> StopObserved(c) [] e.a = "CleanupDone" -> CleanupDone(c) [] e.a = "CleanupStuck" -> CleanupStuck(c) + [] e.a = "DeferExit" -> DeferExit(c) + [] e.a = "Undefer" -> Undefer(c) [] e.a = "Decide" -> /\ Decide(c) /\ CASE e.x = "closed" -> ~open diff --git a/verification/tla/apalache.sh b/verification/tla/apalache.sh index 5881e109..4c695713 100755 --- a/verification/tla/apalache.sh +++ b/verification/tla/apalache.sh @@ -134,6 +134,14 @@ for model in $models; do # TemporaryStartsOnce: restart regardless of mode and exit reason. mutate "$module" restart-policy Supervision.tla \ 's/ELSE IF Retries\(Mode\[c\], exitR\[c\], readyGen\[c\], initial\[c\]\) THEN/ELSE IF TRUE THEN/' IndInv + # H_first (S1): v0.7's retry of a failed first start (restart before + # the tree committed ready). + mutate "$module" first-start-retry Supervision.tla \ + 's/ init \/\\ \(Restarts\(m, r\) \\\/ \(~ready \/\\ m # "Temporary"\)\)/ Restarts(m, r) \\\/ (~ready \/\\ init \/\\ m # "Temporary")/' IndInv + # H_first (S1): v0.7's immediate restart of a ready child that exits + # during the initial start (no deferral). + mutate "$module" no-deferral Supervision.tla \ + 's/(Decide\(c\) ==\s*\/\\ pc\[c\] = "decide")\s*\/\\ ~\(open \/\\ ~sready \/\\ readyGen\[c\]\)/$1/s' IndInv # AtMostOneGeneration: a self-exit skips cleanup and goes straight to decide. mutate "$module" skip-cleanup Supervision.tla \ 's/(SelfExit\(c, r\) ==.*?)Goto\(c, "cleanup"\)/$1Goto(c, "decide")/s' IndInv @@ -153,7 +161,7 @@ for model in $models; do Cohorts|Nested) module="MC${model}Ind" if [[ "$model" == Cohorts ]]; then - probes=NoMidRoundState faults="Selection Cleanup Retirement Closed InPlaceRetry" + probes=NoMidRoundState faults="Selection Cleanup Retirement Closed InPlaceRetry FirstStartRetry" else probes=NoFinishedSubtree faults="Join Closed" fi diff --git a/verification/tla/coverage-model.json.gz b/verification/tla/coverage-model.json.gz index 282b1e9749b07f747b8bb9178f32613d1c189d7c..b071e81741ece5381992e7b4a75ae08ba581c453 100644 GIT binary patch literal 34701 zcmZ5n19V;6){dPfjoBoPZMLx++qP{RjT_sxZQHhO^FR0g_rBM@jFB`^#n*cl#6L z8N;t$9glk{BOh#@jc3NWAm1w+^k>>y?&=fs~KdS8Ep!hmW9+w>zu%t%r{Hmlp;ttq*GtyBF!h z^g}#Pw~qv`wujBf{da_ikC&E?m%b0DkAe;yFRbLJ2S7In+#g%__m4X-D;1g@E$`Nm zFIE#Dj&JYIcSp2dtEjxX4YcG&uwC|T^XKN=N-{1Durwcw_XRhw{QW7+| z-jB}u_FB-r8k-i5xA*V+Hn>(bJG`EIR(8fRK3tz3FFx#D94=-GKGrp~IzG;y-;~fZ z{9T_0F1W$puOqE8vN~L(cT#pd5T&@d-}idn23%f88U~!K@ESXw@e(f9(_W9XDy%$P z+1_^Ew~w#yhBn&Yw$AUz?z}iXdkQ{YX4?8MXfkSF?|FS%9zN_V#;(T1W@z8{n%>Yq ztog3DH_FQgm>Lqezf@dUX1pIgzjb(g(BBVSIA}?9v^TnXxIdg-6@hTQ9tDj!S+&di z7Ie5jbkDE7cr~@Z`Yvg#YP7v?-F$f7ZtI1&Hn+FA6q@;YalE~}LAJKIFu63k-bhn& zy-j?ywl*WYPI!5Ex!g<@Sb44EUF@}RXs*4#Dl>YvKUVx4S;^p5=+E#%jdQ(u^f10^ zz^jWZ#G`#bwob6Z_=KDXN9K;f49CK(w(NH3T@uU{PmakQmwfY<@uDcFICBtitja;9 z;gN@G{CO8CUQpt@-lsWFcjr_75n5~2%6Aw!0aPn+bucYHJK`8my@(-~LM(N#FW>0e ztDosAH8kIZH5w#gARJDhtIi9)aAYnZ@H-rJs|+m%vFASL65D2Zg{OH%u`7N?BB2VA z=7ZuHML6XmSrsg2redzpI@x9gYN*e^ z@Kkx>ouRAzDB5Wn*r_s>&JQ~(LLs2paiLJa5@Ri|N_*(Vvw#A95zyD6#-fwwd+Lc zn{{8To3iP8Wa-a|;vfE4P$^q98+14%KQN^q=}a-qP3vn}(~IGUW5a5i7Uam8l78^O zMsDgGgAu0F2bppYAN7@LuT@^%eg&uTMVIqKGCc%BQ1JWNsR!Awq*&4xtkRWfIfj^* zOxoG_+Wb(=cxE!hp`30qFxVHwB5E71;O`GD_B9p}1o4KjWkCtg(&}8(t29&)i-o$m z(1h8r*bEL=G|Df8WMmmF6AFEyZ%{{`2P|n%l$4D;k)2LrKONlwQzNt#C%0)0?k6tl zdw}KK(|WP}qTl6cWG1D8_$Q1lW_km822qDxve^7ut^LrFX_Gdpn5F{*zc5Rc@Ajg* zj!>E&q;Y!&5Z{^i?NwDmN*NN3J1_Tln(n@WX|GRYMyiT7ivZYtGd;JdJXN zq=Jub!-Xr(_iL=XM2A|&mvf@=g{XQbwSfMr0fuP|XeCc0`&B90R1Lq=OW8ZNgEpiX zfTGV1ivltb*)uLdYpysnldGc9N#a;M}eOYLS~018U=;{q=SYg)+FpIjDvNBybYe;f>jEp66&#k?JKb25|>D!L7SX zVNd^2SQQ#Yrz7f*#0RA7Y0{7kxJz;uHqMc)&6%W%KnwIhvY4{3!qlwdRJ#*|B^-CL4PrrZ)A+83o03$-IkCpda7 zjK*cQCH#e%)EaCk_?>hc^YdU(w)x@pSj%zhmP@pY$Fs6y)1sqdHeEfA+r$r){$Tr**`6Ht|@7%uAhF zy2V*@6DxHCNP{2!OI^VQvSg8xfN<0%;q?7Hk;q?%XRyFZsmyAgDpk-ac=wSss^d%#F?U*Ux!q?$s~+31ja!OZ32ai=8K6;zg!c%LgTmkm!TU*!dDcIyT@5@eu%feaQ*ZE6{79Qm zseKgNf{|BnrrR%rozsNW+rW~KOA8}%=vscT7AW#Dc(OrILWk2G%nxCvqxonRc2q-9*Yyfa^X}eqL+uK0g1%L&{8QAdc38P z_9mNy6Mo#&$UNvf{>;@58_Pp+aU&YrHQwP)m&AwP`ETe&uK~u$JsGiG zdnS~^XvrbqgKX3ul{6F%_or{HZIvqFH-F!Zp~sn1Vr@^0D;B=K?Pv8S`_4K;=DvgN zjm6Q)Kezeya?$51YYs32g(`(M5Y~2?MLJ#k9W}UJd3wkh??6ey2>1mwqqN2C{)X&D zU!;At-G0_k*fxQ2-V6{W-&M-it8kv-Oa%xbD|bR=L6O8tB0gvWAo`A4wF7;?1i)l} zVi_5hU~fR|#hg;LUBT-NL#amuTDkN87S|Zcn(L2>s3{A~_B+@C@VbGl?R`9wsCIPl z#)nMXi`Eu4t4-lBbGbFC77jz|G8BciUBq^Xb)ihn`ZRqy*|ER@2swdW`G!jO@duxC z#P+AR^Q^5wx@eNbY!b-~c(!^9bt;RN;@WJ7k+m3v@$-UphkIy~{%5)pF56=JP`PN# zsn@Xyqr`t^@mD=o~Ht za_B5JIy3Rs(@2skzR+S!8X+WB}>f; zB$LE2C1!7d!50|0*v1zan&<}3Fr>tG#ZpNQI&eQ8{CtOOL=30<)Y_gl=Ilq4)@5InKzxeMKn_h3! zC_U#_Pe{JW(nMVgg3?L)*~_>ueJ0HQBlm#-L;$rSF=Fcl5qNeI_)9Say3fF;u!>Se zLKj!OuRO7)f&9S!GHos@B!7%uh+&h4a7w&`Ss|J|YT+R!a@74r4JCn2WAVL1$q@HtRR zG#i4cf+dYr#nWwL+7;gGu`I@dpHxJB;=pKx;>yt5ljetX{tw0E5QQ1`psT~1c`P-JJ{qXX0!K=yhrC#JXFmM0POv9Z--yh3AW!tlz z-IHzo_24R;(YnYc1|#TLueWQfa8fvEW7$*p(clbHqNnq{AmZRlZGZopSyFE( zDaQl+WC+Dg%cqM0J;Z1ar>6k)M*->om;13x_OW|pb-XXn$-$6?dDqrY$d zVD6EKynBH$>N^(y=A{J;09c89>UAZc-yNk|t-S&fMPrufwcXb{f@rWy;Fo~~thE2n z6Wv`p3Vh$u>s;hilu?{S61!W>PZ59dNR7u=VL1H1iR2eoeX#DY1me#@m(tmdth`J3 z&WhkZg}-D`e#kA;+6mr@!1){a78L2diHtUyM+sC5>ypugitNRX^2jJDJ7Rt%FPItr zp&!>J02(-_$34P#^DOe^g*TJ8^|UGjPRNAsg~%3-?k6LYstUG~OUZzzA^@z(SOp^A z@c#qW`yNMn-1Gpj)^~ZV%D^BWv_&HWi@*y?OVL~`y{@|jT$3KcFfi{T4|oCr&~JP+KJqO!L9Dvwu?o+ZoY0yb%j-1A8<%){ zyBUK$3?^7x+vROhO2%AsVG+lqK$9>4EE_oVAI~V-Gyd9>Pj8AuSU_hB z!e2mZf6bRqW4_&;Pra2(oLUWr9B=!vZi4 zRfM_6qqk83PMzM=Gm9y-(Fao*oF-ChIUYf;m|VHI8=)@|x#RR&^E zi0tn~rw`D=;4h-1u%yQg_aCB@Bk-3u6>g)>(7!KAePXuv4l3>v+^hoKp)8pb`NDnw zix3cn1RigT6Dn1%+7oy$z){0)j6ab9HZJ{2B(98<<&Wr{h_{%=bnu$MNNiHC_^u1hK1-(f#0wpF89@2#df zXoOl9F4L!nk=K5;kSB`pE#a-3RNGG3P zYR!+XT1nL}lBm}DDkHV-npzt341GbBOlC=-+fWwsG1!Ha-sy8BBpkQC3&s0=*I3Zi zjPa4!Mg=8PT8=X@zDAZ*p=h^$Z>_(o{V@AlPesvsVIq34&~XEq4~5d*E(fF{=iiQ= zj}tA}_q=D;`SG>tL9I4xLgA66j$x2-UAxQ_ZggG3!BqN@av!lW*m3iotH6(%?=jdp zMBhF`IG}H#WEv38P^H7LQA&{FxbgSKJ5mqf81yrX@+S_N^pNq6#p41-&Z`|;eH1Z`b99J&1{pZ;8<$@#YP|LWqo*^<;&s4*Pd2`t-n?!76f*6tNa zqpn{)ZDs1e)(s;TjiBbT;}DCGgPnFbqqnt>$3ccWrkUPt)C1T-Dk3f}A336pKv%3K zUO$3i7Yi?)2SdK2V2KUnj!Ys2WXX9#{B|kETzwnYOL!>Vn{$omDAd9upBoa)AvNu6 z0v;%JL?}k@QB0YO=3w9ow{eTfr~l2~^ZSMmwj?MS21%!lN67i3hTg(A7JB@UxU)rL z&=N`1i$&`{Eci7DxKS_XUkj39yk6sCgF`SlkmMN-2>oF}{_^ijAyRkW=$@F^mWrGT z5upTEMIA>!DNKmVo2{lMo=Fq9|2xsBv$wT}3og#q+GiCY9=Nm;x+0^UoI8H$s~I0? z<11_sZ!)IAhd!Z19X73wZ<^|BZC;D+l#WxrF$K7&KCR|*AWtga{5+@uI|>+7$rf*#LzkZV#_iEC1wPYYX=)bLm|MGlIbiVju|H|i%SlZhq3nVHk$o zThOzS;N?l6H^tLxceIoAo72MSCix2>iHb@WlJwhA5oDsLZAet848nhsBoQqp6;u*o zC5R%uHV1Z!Gl1|35&x1G4&^iHzfTAv^NksK4vvt3+EoT|miGO_5KhZdGzn)XF=m#@ zqSbO}AjvIaZM)+JT*OKG?v5aUDX5bSMT+t?<^{M55s-$>fX2wr`-wAf9RLBwcOgl- z_&Q&J*FBvCMDYs+5|_$TgWs4$bezeb=b!N<&%E5+9-e8J?(eG)bA=2Ov&NxZDL_Gy zCGzX&1U}Q0|BUV_^oC*>@sqTZp|jIYR8B`5V-)i(#UX0igZY19uk2>zFJZ3|?pHRY zf^h`q&hNrr$0+k(!k)mAVSr&~<_Q(jc(rpOh&VfaYcUqv(I~vL>F7~yHIj+}-BcpM zGNKlm#=JxwaYUvb?ry#_gkx;*e;HvdXt?8mk|3;WvRgB}tv~n|J>dYaEH%;dkcJ^W zWg`C(&NoSyu!Y`OnIWRN=xpiaY0oIPZR#70cLhLAKZM;cNsSmbk*=Fe*vJU5 zJlv+7H33V7v!^L4el39`aqYMM_?TcN=jL=|6$cxRnOH3xQT!sFJUi08;nf_@VeKSs zdwgDaI=Z%yfr9WIBR18QjzY}^5&Hi5`MBq7Z0#fx95jQp?;v6VNbVv7G#hapM@y@n z9b!BKJ++yWjKl7+%J}dYP~8~wa1%@Aq-_TD=zM;uw>7vo!g|rYX?b2=UaiVR#tv^G z?})Y}w=}r6a*`XJ9-G|kqH4Ln&EFK@HlcWUVtFC9RXn2-|2@P+b_LfzBN>8X8`ks= z9ZoOzrJo!zzzujSjpOGgADoPdt%A0RCHVm(B(waZF!CESQ^bOxmSBm z6AuZE0TAp)alR&Lwck@syQ{dg@ZvNUmhHGNyMUrW!H|) za64|Q`-IH=y;7C<_r8H~dK$T&e6QDCc9zW|`hQa?X8zdsKH*7vn%T5muf3sbz$(+k zKYF)pt+I-tzP}M3jOZ1jh@a%{ZOIqlAk5Bb&`7B%0SNFDUTv*q zl$&y0)z2O@jpgniw?|CF&!auflO`QF@t#W!bdyHm1gD~xh`%Lv91Zv1;Xf3l^GnIQ zVj*?*KMVazCI0{VK%%(*ux;M+)-3qWzRkd|IGjIHI~*yFVOs}};e1Tuax$(I9UE*GHR6b4UI2M;i94P)7Y#e8niG@&02Py85p&}Y8}RXPE**x0}#O1+|h zY=lp)+T~c`bSI~~uYsTl{kvwR0B}PTYaa5+EG751D9})8lVjot=v@)s_!=Gp>sPkZ z)!XiUhP&a1Cp#Wuy%^UywP%Me29QM8q%}9-G|!>$cYN4)%z^(T1V)MpD;FnxH5>sf ztC)vWvWXRAO*uyH9afzR6kEj(1oJy2TOLf z9QXXufZS2TkdTGqV=bt6VZz+H&qS~D35zYk>NMYfX}@1A8wK`4FTSASqG|}4LBwRY z#-73HDp6j<2!QsBS24HGlS(!<{T5{un-~SBHcyT16`9qjrnT^Z)jMk^N zz1(KxETPmI6?-|tW>aP2M}ramACu+%z&mTDlP0T4((J}ff}&nk(Ci-%$Wd>#>+#7V za&#qkwErCsC^iSH0eCRCgYG^K-38Uku9;?^W20#PcSO)ID<)TKhu@m(OAyy(@vLqUCN zo&Y{%kjcRHLvCq8n8Jd`?XPf#pYIP2>4&rD!$-Bi)vH(N8L$)o!HOtCu+2$nW41ey4MF!$N( zZ_GXRf(a_jl}fWf>D`kfpSA!Ar==84$FPHiw;iVdpW!fZVVm>U^Vp@PXelo^?M&&G z?9D`Pq^l(|Fz3U`3yJZS@vlGNJb8Xk$Ti;s9#`DLrTH)Znlp2!hx~=VYN$$`+MwxJ zb{u7;0*aX8@{B}Frk=fMJA7-r^{THC_KIvHid`z|pGLE*mx`M*0@aM8>=elp=S4?) ziVSf^{N`kqveUEm=3u@dgIfAA{4qQ_=ZS3zGr5{eXN^L=R7q#A$$5zl{TG&Dzm&z8M+Sr7)oi)OtX~6# z<2=l;OuXuRiZk|B!mxrA<;26((gy(&?HrhraySY5Z~lxF|8<8NIP<)_w1LUnMuk>= zzn%FjI`~1yDxb8Cz$ZBKS{%Aj)#gB5_$;`!SkmuSl{6Z2S}yzESq0UHc8~;`$a4RZ zW!;yOcDC;!l~fup1Pr=vUwpcfPWtr@jL)?aKmV+OW@nfaH0n)5hG`1<)V5hy>{8V# z-~d@{~VcdL%(R!`7traN4yA9E}sLndMFwS;a|fE%A@#w{*HHSwK}&2H^)1Bfx| zdcG3~=QlrwHQK;@Wu3}GZ~Z4QiK%piMkfS!2OlXb!`Qy#+9-Y>5AK7829n4z)9+2Q zL+TV70q!ZyGb5Wv9G*Vi5rd^WMP%;u+cE;H4ns))u$(orfapvlGd=ak!v-ZMFNPxn8nw6 zyJ>-$fI~#G|B5i+Y=N9N%}kd@F>naOzuYmp3B6Pl)iJC+)to{S z6asO0M3e+?^mV?*{4^Tr6`26oR<;%Y$z^>hRY+J7T9F3&4)kZ*&<$y3>OVq zFyAjEpCsTLUdohaeG30sZMlxfE*7@@a#KK|y?o*N5v-3EY&wWc_c#1P-Bf;9-+Bto zcHa&kZa(9MHT8zre62r6qTMM~!)e1`RL>HcUt{;<4ZBsB2Ap$>FvqP!(dmyhT-$br z=<>A!sX4z*?FWi7THCLM06CCxtUjNwg4fSG^4GHpUYsr`PRdcYF&JPkzUFOn4B{a3 zB#z=-{`@->fu?wT{+UdgA+Q+XZl_IPK^|(NEj) z+PJ-OR6ESd`eVwaI^6IRg2A>!k2Uwq5P|{?Huqs7fsN;w8?2@E5DosAfgV^#tjg06 zV-S7{IH|hKNY8bv&+~M1%cz%%v4Z**J)hO#PjSU!EFq4OQGnYaOGb6gz;A1fcuA$T zErR3BZtaE)56p2J?8H#GFnAP9n)#SBmuA@m5oI+6j&0rf`qp>Sq2A#k=C(~h74H!s z`BihkfMSWfl_&a{W+_^6#Gfm2Si2$l}{)7~@$uzSiQ1B>p36 z6mae!qK&lkk$$HA|M@6PGy)+P7)(5Z9dPczg@9V!hQ);h0^z#${;V0pM1rN|*dwo% zqTzBYkV{`SW=Evg((-6*YjS(LTA50B2_56+G5Cs<$uy0WeE_oW_~xhsJfQ7%dB7sT z+V1;}o&(oaw}A4;%I9<7i`|SK3E{ThmW$=%YC?W_r3IG?YA~)=_V#vnyIhY4C{(Xt zaJCQ^@C0Br)y{(FJ$5u_s+VnHri%bl9xVHu18S3Y!r2V8z6%p}IC#AVvvOX84f|Q$4F`lr zKv6?t)EuUfO4D>4XBjQRAC4Vd+|4i&a-t)hHzX(};CXI(++m>q?C>X*senK1CGxyoc^LjMD<#_Vk*@Yhams zz&Cyt7(4GhjU$=t7lQlalqWkddK`Zmo6i+A_3=WexYQjGF&Je_QswN(v`khTp1s&X$%`|G=e z$4m>H0dgv&moXKz{^1UUL1{yMz<__g#MT1H8+PL+RH;c@_L-r~ZN>z;*(CbH>a? z;sOz}mrn4DtUwt#_p5l);-W_=kM6LgD+D~~?%EAh0EmVC2xTMO;&i`zBeo|`$QOGk ze!Tn{JDRI3`QiZ_(1NfDOBOBr9fuO%2^wouY;@s}jhrutdkGmc6kT2|LRA>h);;q;bHX%)_-AmA&s;gT zUoIaRpOCJgt6IFeIsx~OzB<=~^2==mhSv#`XG*81$rm&lhgFV5VN!J3Z|yYG?JmUf z;jl~Z@;PZm9cDbJ0!DJz!#OY-2c_ETO{HgXexvvDb21cwdpP4EBFkM*qe2N4?AMX#xGhPt~W{qq8g>L;Sn!=)Gyw2||4 zYUqCutDrQUdT6uLVV!)$NK6-$=uxO(G=mzt(i;GJ_e(LWAkYBZ3pEl$vhE!91gQ<| zs~*V!GWaWONy3Va>;WaZzaC~>&H&1T5V;Ibmw5#%>_^f}EeBjl!$eH0eutJe2Q2d_ zc1+3}#JYRb3j{^`24ORUnm|aE?^eBYwR~fKA|42v327?a$ulaTRGXKht=X-fh}oNS zGtK>g)Krf$zs2fJAUvr{D{t;)WI#$6)(7D*$W2WCm0%U&;p<=F1j04fQi69*e7I`0 zo-l%U9^j1VoO7_xy-JzojruTi9pqNvJjJ?#cb*5~y`7e>{Ayj%FVqU?1pHI*xo0Cf z)})V{X)pPU?p;qCoK!0-XZ7orra3V5(+V`_#8i=lRc8C=QdB( zS8G~c$kj_gCp2gq@7A83FST=_+; zfTq%18YnNEcHb;pr$Shpt=D{px2Ko>lO#k-Px2^Rg;3>Q#yz@Q)cBEac0@e%!ISR1Jew4E1yRx`9GcQGVYzQ&c zZ9xbs9lV6&)FEWF7z#}76m`jf?!2`a_RW^%;_K@TxUzjmTZUJ2f+3%KVa>7S&jiCS zT-|`v%g={{N*}HsH#>bTMiaAeEvi5+H@i}hZr9;hB<+WD^t!b#=D7#H+xO27`X_~X zP?$4!FFU9)yM*5C_s~eqckbSH3SI^T4(=xW0@s;_enF1M756O=IpHHe`f77 zDtW%#XM7vrY$#lRDoH?AQ-kfuo+_}SCUJN%zxRj5N!_J7zy!WIpc-qO=-nPu$`B$i zRbm$CD(u*`?)+QW%rn2y(Sgr8FUto8MNEh0!ESod<&su5@t?Z>@(p(k9t7tcc@B;wWZTO4#2cP!b}Z*ip1cvw2sWcUzJofsO%@;e>y2G>gVChS1*X z!WU}5M8=;lV-FsB>!)?D+*9_MUc|8ZXnhSRmrI0t$Tn) zXl(Q|MpfpDsoU5kGA+=>f3=nfBGZ0NA>{s?xT@BNas6k0r3L2-gR5!h(Djd|Dp9z} ziTvvy)jjMyQ?1kIc3)xXuJ5-+yn2w6mb=pe@iZJ>vhVB6qokB8$}bd%zhzQ(Hd6}> z&H>J}lKdb*R<9~9H&Pz|r`U|ClY{_zv)!9Fm%0V!eEr*XcLkH;G#e4tv{|`L3)mz(v&mst8<${2uL*;B*OJK=18WhlFG4TI&$=-I@2v< z?E8cq5tY0%Bb*#RDM0{@@6oz9T!O_%?m8yWfwQKPOg0)dwD^)YZq@prfx@nVm-2|@ z^ml%Bdgs#VD&K&K>RnzuBZN>~v$jg0W%x}1_#9vC;)mM`j&Caxp`WJlO&53oUt{45 zp>WqDwbOwtT~rYxvem_4o?@~j+kId#^_gl6`*0NH#KNwn8VpGW#DST9&>0x-HFHZ1 zS`Ke+S%CMurK`6hm-BjTyPi;~O4qQMa(__;QpkM3``}=tc;?%3|DY;T!qfPl5SN_cgJKI zhoD|18B#uazdB^TBmmO=nXI%=|bl1i|+Ju8-!5?Nwo}2yowCS{#}bV z<$r=9`Q+a!rwLo}uAH{|j2Y}!LMZ&)*Zus~>dbpcme&8ENkXYrLoo6`9f8>wO+k0m_JFS?xUyC-({)^t! zOY54N8lI29^LO-QWhdYqFMLVSvUQVTa`Y7^sOwiwv8T|BGZHa@=Gpm9$4?M-?!P$N zMd2^zE;P^X9a`jSzTK(LnJO31ZnWwxJ+R}KO|UvCDcM7O@2!Q>fekf}%^<0f^(d+P z9kHr+o7)V<5{>p|Tsa{PGje(pK<3aYm0dVBdFcODvXCQ;gza`LWo= ziQPc%?SJJGq!b>dxwi9CZ{$W;$@aM2wJ5cCtewxEO13H1Va*>|v%7$coZNsUd4$`6 zV$x_0(9oIWdg5X)lbr|{;eM-|w(qWD6)-nhh#8fMS6|||&vsX>SmU?{>G1b>64Yxy zZ|X;}3ET{vF?k74Bta#r{qupR2|*Zf)sD9b!OU>k%JB(7**^{<4T0a=%qP}*vfkSy z0`AXqNgDsIb(J=4a!CS(nYr5G31busp1!Mt{kdT!ls>($BgNE1c60=Z$A&uGKQI6k z4wQz43PLquYC#OTy%#rdfKz#TAz(o>ot$3e88%yryhp@v0=qSfPRE3wUZ7^ymOI@) zWCC1i{aw@acQA)v5|Fr9Lj%A|f6}bd3otYQy|5#${eZ3W6y1jSJ2wL#!4mnY(-_%fAwlC*zFEe*- z0tZg{5*6A0l`>}vk2hA+^f!Es@dPCYjfUS~=l`<>gkG8Cz}5SZQ<;M@b3JQTCH?4F zjhbV$1oZ(w;g^~zkJSKYw&{DUO@dhIL_w?f!NT*XSQO9GvK&0dZ1Cyyy~`>vQ5r7y zO>We8Z?_Y|?nX+f4;&S_e~K!Tl`||OwqNJtW2ClXL3Be7XN|5u;Ol#9RU~`oQ zXwNGT5XR5Vb^KpP!?-i)<=y3LzD4IaU=%%^Sx!$4427o-z)N(eJj&Wrtd9x zir)`QHV%2UO1!x7qzUuYt$lR9@8)y)Y^dKJ`kIDZt`nDSk@MD7BoK zX{DQmTBpRZQi0NVuHA5LhM88YyogQ*o~$!kUl03pZJjycsC-66lQ}ZRao-F);VqYD zCmFsD@hmJxKnUvfDNHuiI#zuI=BC?v5VIBr)Vz%Q_4kt6blVhEEav-S$ynwCjeJPjvn8APXB+lIjY6t3)=8pGldV4yA&~m_zZj zBxRx@T-8aL1BFh(r@|&lRAtk^*$5&ypmI6zMmk>eayjU4aq(vk6n}2nm<49=WA&91 zVvEJN=5sKrbrmWP_G6$Y^57(;z~K(X-9lhzWpXx*a|b8oa&v*MSN1(nZ4(Pc^HG&Z z&khuZ*)e4{llgSpdWVf%H;*kNc##WE4coU*)tT6{8bpS@YzY=QA_9Cgv_ zu;W44-=^!U>O%N&rIwMbGEX6V9S@N;ZK**6{Lp5WtPYdKh|bSAjQv)c=I>3g&l+Of zDf3n$z`pn^vF>)ZY79{mSdmT2S;0a7N&17CTi1PfL|>x5+2hJZXS-%dVy^V(#nXDE zHAFeVDyV$|8@(DQ_Sv@(y3leI2Tx{kq>rnzP&a*=UT}eRX^^hl;~6bg6Ypp011Y_5LQ}i zFZ7_s-u+Cu3d{3QVXi~?5qQ?-EpZ+%(_e2>v?r!@K)t)hHx~~I%(xDFsNR@kp1Wb| z2WVlr{=Y5gYb@#C?yFVImMY847iH^>N!4m&EU}LoqC8)WYd_7gWN&!oRkmsrYWd_D z$}+7%5;VYCXlLUF9%4Vg;P?KoHHNr?TZKcrXAnpg-<|?wfmHf8+7*j+OfdgibqxIine$ALm)cW0?HD?raYeCo8Ie0=)8vm)3Le>e)Xe1G{#>58Yd_G=eX^zPx)XZv2||bB)k8 z!t2e<5X%AlBt(bPkZ16pqU~7s`+zxV@h>A{qdObxYbRdm*mxqryr4tIHu87*50rms zHlbh^6YPsZo>pw>$qtTeh*CqO-bfg&lF1MpLm zno6zu3YVc~uI+nf;dEd}bSI?>FAVjCm#zEl{TyE zZw$Vfi(LGpBi3H=c_pZW`#a~=+}z#S*qhR`iFMVx(WQ!l%q8!e$};;)=V&;lJ0k#TmyTR6x$koY zEVVM){o+z-;k8SON%=Ii!mRnjh+@p_*lkG&jE{8Th zG6M#iEGQbIr=%z<-jYv|h$&vism=aXJPDyl&QPxtH5xng^n+>6_-ATGtx$MkDs88o zMg8Cth}&q`$P>u-Iir+R+DfC*+u0?N6pJ{M<8Hfl5HV&I>?M2*L%zIB=G5Y9Q5e+} zJwdFqMRhLq?jn|z-JU#l2*4$0uvU zUrDl0)&cog7BL7)^M#+|5Eh)v$4H*6Eh3YmpRB_Rp3P$s7Vg86(mAkAYRVWSGz7R> z`(-6q27?yr6QL6j60D1dS)Qy-B2>1XE|GpPA070F$-%iMy@Ha2(2)`#HKU{(HcVmj-pH(9x6 zdz;y2LO6lI>M;8K+l~I{??f1UWrkV0uzZ3?MT2%~2>nMmN^zfIYN&}u#Z|<=xW&MfPmwK)3x zEA9_Y?0dFDLR)k|$fG!rFt}Sn1OXHG6`%msN>)_;=a@ag$YzkL!&@0|2?yNd4XrG; zXr(q|4ukJCU0enn}$NcaAqEMjD)qZ7=!2~V&Z5mTBZFg zp~*8UrQ?4H48%(9%D9=+9OvjI0dca8v+eelcq7k%ee{#+@ zPouM>6wLY9)!(&N)k+_{&e_b;a#l~E9)OG0c>;m{>4w-#j1;^o8W*RSI)nkHZJarO zK3_G(nFEJ8|9`E#Rajlw(l&~_ySqEV-6cTK;O_43?(PJFC%8KVcXx;21b1ENSxNWa zyZifZ&)s|mtT}7csPUH9(|Jx}{b$;hH&BGee_Jc?dNsuwhILId%PQYE;a0S~qiX-{ zv4VL=anM!n#tKN-eaEP3EQUF4Agm>7*t02H#+aJL^082;l?2LCWASh$#w^DJMqqXm z`C@R|*H4&PJDRHzQ}a)1*_#D}_9~mrrDqdu8x=>fUK?fmmAvOx+?AVGIXifX3$`7_ zElmj^BRNK}qXlQDv_|9-`-f+=zy2*rj6YR$!0LGI9xxIEMJZp`vZlqu^)9s*+BNjZ zMee`sJKQ-kUB67WhD&WK(Ag**H>4E}=tm(a*fbFG3W+4CSnvJvXtAdAerT=l%2Bo%7{FJpM zq|@oG0_CVy7iWajV;9J&Y0a^_tysgq`gjwyCg2wl@r%@W98me?E5So&`7OTE|(bLh0D|e-P?h`o~P^i$JhzyANoj9z<+QowS1+EQV1DW zJlGBcGDpI<421MQt^aBFVWkTtawFUyD+SZZN>w4WJo=T(Nn<<)3Cdnel*=u9+6?FB z>7bZ`4oX!K|J9-JIYWpMy@po_Rjx1@7yptGpgoJzo&J*$B3+5eI5XPn4EW`!cG~WJp33?Ju_~UP1CRNzYb#gDDp)sjRk$Bu zXa}(ou{2GZV=+!d@*!@LJ3C`b&%3g^DLzs7ouK(c zD$v1hpQz-qlT`ES{2h;F1l4lnlNrX0D+HGtox_(!Cm&3X15ardDwsgJat?gd z4|E@DGYK3v*AF@4|7Ycf*AKsebKYIzc^b2@Jdi35cg>Tq=AR5Zj&Y8zu#5g$l{(XS z-73ksG`ufIWq#d)DEWXm=qA+mRO?!JWmh;;#kc4x)waRCgulOsLoL-(IoMn&*?hme zNVJl-#6bUW+9{qrji^U)0QkFuHoooBF3|$)R{)>0>LZ2f3bDxJN-g`>X04ZUI}uu|5am zr$5pmuH9ud`X{m{fa+B%JDX4`;BoJ_0PzRh*+VS`?JcbZ-^;0Nz8}c`F4jWJ!)z`L z(b?wCd@HjIIEwxLr2Z*1@h|ac@>>u`_?=C48921ci!>fY99-V0&ipG((Ew?AvQGV#c0J8C}k z*|6kHX%q8IOg*LwDznANrXB0kzl8*dNxMr0Q>59f3tdIWfAU3G-ztNPvA(W=#OOumY-x!(UNjzdEB*y>&(#tt{}06Bw`0Xt z{tM&Wcl9vM3U{S|^|R|R?WooHV*E21gq!6~QHL>eZ+fYzunLphKXx52^iux{U;hu| zZuCDi)UB{+#)%^d1ma#2cda2GaE$WbX|I}W_Me=3~ zt?HEQ|5IqjMRsq?9qN#OCF}p()}ffQms;OyxWAIDe?jU0?^1lAhiUO|v75R40%RrB z5kg+9w`v+EQ1Tl2A~-KpNB(`v2CM(3Rz2o=-`eOOtprYC7VR;5oqn3_Z1QdC)dwDb zz^=d_z}qn6)Xl;LX5Vjph?aBTvg85a<@g^bh%@}Ix-Ja=o+!AT{{G1I{uZtOwte9L zgZFLjZi0BD;~8)w)c!W64jhwbcW%qSEcD!r+>Ct@ckq6_-M8DZKNh0cQff zckubOWcvga`xE*F+s6@~`w_eX?mK0C?@Euy-|h@q_@A5Y<1gZ>r@BU(A6&2Jh%a#M zFW$Z0?StXmpW>$kccT4y%bkM#U(qgZ{0Tm{-T3=a-TZ#~apRxU@=~!Q@P3n?`S4PG zpn7quuW+()<1bY0GxdJwdFK80Aacj|eh|qL49ER+aE0xu2dV$|T7iW029Txnd5fR2 z-j41PQtPwNU6=swW+=Km-0!A#hue8aIT`Ylw8!rhCO2sIcU?wNp)bHAQ&h%t=QjicY7;0(H1weY9_#FK*%vJbMc zSMMpNbSrjHkd3*<*Dq7gS>^!(QV!K5L3cbPW179+k`W}Ivau67dT=&z!tMm9|GXAA z>Q3mdCoznVr8#gHnwGylYVc+dB1E&@crF1DY7St|F~w`D(8uA@SdJ)y6RU0@P3&0) zxxx%}4`3}W6eq>PjQ)P=gFMdNY~QV)g|s+OgoYg3k)dil7+4b6Z}GvAtk3*3tGIcP*WC)3sVH@h39T{EB^AW5Q1pjtz5^cy9Vz}KS z27#~H!}Nyc=Frj+^6?W+Oiaa**G~dtA7dQOx|{$vBv-|n4zV)|Io_jrbDDoF@$k^i z_u&$?obm*nBK#r|{>XpjbWq>hv*oGG>qN}pl7lsLZ`kn7gZpzF312V|%=Csq3=9Z5 zVvPCz15SW0Sbzfy0f`%!(^f`Dz$wDuBUK1gg2Ocaf0iO1$)(#306P)R+)N1rqbl7w zX$J`GCZBU+kX!=Ma-(t!@8Q0skmUKq2uCM@Lp(&040D5GlZR&ngPb9n6$6E)kgF%* z3XDk*i4_`j6+Z@*w0BR1jJ@(>I=+4iC6bisZs;I(*DZ{cc-^oE-16M@*TArt(PY#_ zCWaki=m{lnNM#umMM1KHfqu{+ZwV9=@sQ{Uv2l@rd#HL9lKq^SEQ8?)7;MD9zBm|B z)9vH!?Ray^PRFgp9aa$@&tvv|ouE;jWaN88cV3yhPX0X_Rd#<(+mOz#8=gnaa0k>m z|Ezbp5G)84JR>^731P29IwbHQ(8rl&he(h`rgJsOg_kF8E81e zncKiGZ{#y%p@$S6CS=LWE^?nx4_%#;-74c=MExRNAPlwlv`Wd6$a8OtvSh;)EM3ISp*bC$HI?~vV9P;QGyLKh!Go!XF-KK$QMa{)a1^*~Cbt3DAf?y}zZ?i2RbB zI{=x@+(u4VYzc-qXPkrscXV7c9a~gMa6%3e_BIs?)9gh;YEEY<{fHoD$^HU4jMqHZ z9V~Iqp;#^!SdqWe@a0w11hkN5RYRgp&Of&1&r8sE6_XGYN~#_C5cxIs^uQZ=F^07m zQTr_a~@h#;7m&=el~k;S?YS!o$WgL z{+-(clM2lAv&i~PIZOFY-={iCgGG8N5^;|6QY#3uFK027IO6OlxoC3I;$ZQ ziHa%tZD|AsFip@aLlq)TVmqmPMPwQLl$%EnngSNBUllead}f8HQ%%Dp3*A_H$bmiJ z+}KCN7I;QD3W-&44X1a?%s3|2>bZc_?D=6GOmeXZT~QLVA7^iHK~T!kG@$8x%E;bl3$A(*TGr>tcD9y|{m#;6=U(W$9_RFir5o%0i22 z-s@b}`D`du`L(E%x2U7^Jjt+7)E@+~5dgy z*#{YAvrhE~u^FQ2M`(aQDZe(v(L}-%!qyRWSVvHFAF%1;{j-TMMb!&Jp%?t$%24@X zeQSFY=HGgQUGxA4Y+f($vm$8(f{w!>Y!Tc{aMak2;St3HGVx~y&{F;t7gB%4+QG{$ z(x6ytgQy-o?r{eMNvKknC{R6a1Yv|esIue1B!quMyutn)_3fbnM7iP&Yb#zI8xT<` zulkdVb`}R*kA{6&bj$3u7(iIGl=%td594&-0t|v!K|$r%&*!`57(tgQL12b5_jpG@ z{*CN%Ba;&;K&X!}Ad?n6&C4gxzONv5)53@w9YoJLw(2YJpn}afhFHQ?gTebir!Yvt z+tU+{*KI8euPE4`7y0R0djU-p#UvvTJ@fQw0HlE{BO8qNA!Y+YEDWDJaKvvCKL9J^ zB^(^muV8Lbb@B%>?K+^z6&E4b@^&8H=k(U<7qmdE}^<&+)n2yz? z?XM&R63-Fr9f8$NA;NLHui`oPTk6_Iha2SXh!%Z@@1|%bZEhKztIFhLjMuK>msYYM zXmG;P>x>De9M@K|K|v(bm^G%H_g1oDYqI~tRkA&LjxE$EAk{cSCO}kOd~%fORudQ| z$AFBw2O-1EDRc`jbIHp7o==8@J4PA7y1$E;o6J~6EK7@)==9o@&KaB+OnTN^tSEc7~_#;6jZCLg8hTN-E>-c5JGLGB59I)Wnd z;+E904KrMldUB@7Q2w&nJX`C*M7+bDh(64ZX`(^^@xU{qAW6qi!!I6m`8!6y({VB} ze*Zu|o8=V+tfuZr3RpLdLuX|OuSQXOdkG}usYXUqAre*f7U zHaU|;gNFp;L4Y;pD%LQlC$Ie`vH%IL$-N{yc`_CXu;NljG(VeR{@RtoS0QG5RJLGA!D!Z>r5UqzQ@vq6VCzYQyRO)o3)-k<_L8 zuX+fOA}q|fxzIy3!F^|xHs+3t3IJ3RgkRLVOHHhQla^8g*f&C`SIG+&kzn0td);P+ z*xk{DF^Lm;HZ+LKBQ3l>vf$;1D;I_iUZqb^4hpENb#^mIBWY2GNQ_Ir*;M%OFN0{- z(a|Mg1*lYjf=(_P7I|6>2J2L!jx zfOFpExT#oV43S;cySty>FiEgrYRh9M4>+U4!mOU>L20oke2*kC1=1HF>50kSk1P)M zK$~E`YSlh*S1~TBI&G2!I&K9%B0?y4EF+fJV>z;p`WWCbMD_8B1IecXoJQId3eQW?Y!L=lvD!KrzZ09ZaA>vK_O`Y|Z6uoLKP$;j@ z3Eky)p>ZcWkx~+N_uu(jO_)T1H~n&j(z*P4_imZeXw^`PMw0WW+X|-e3lj@9i6qBa z6Plt-=&`vn^9Upx>K{)O-Z&ah$!lm74V9hDO;aYsSUxS5x$Cn@C@davE4*=8zq0QcF9B#!4KHT`jcQ46JTRa%ij!V*a`7;X z`?%)VWS|uMVN=-IkVCNa2#|jVr?TMXBFPu**p^o?gWdsdvhKWBJg}IdBQY;G16qa9 zR3yvwMlUy8c>VJkXN(X@kYr_{%0Wab;-N^zVvo^6!~~wA-*jfi5z~gAf<#85lZOyw z1?(xE+HdyzH!zm#$^gY`ELXZdLt<>}L0U!VFW?6cn&~idyAk|P)toJ9AZ2YOheu2*P&(Vw${s;*BfzRUrcmZlr-R8Ii&${($91!^?;~RIVNs;G(|p(sz2=-nQ}sfe`)u`szC+;9mL{7#0_2u1z`#jObk%dUFFNQ z8MK}bvIrN3sPl6;mfx=CxO`xWx1@d0$Dyy6DGIPa-xHRmFTgPpk@2d%)T7=&iI@KC zRL1CYf7Eq8i%C_=I;on-Z|IX`&SFx{m$-6;s zu^VT^e_XC%UGPT;f6GCE4#BXSICcb{k{D=o-Gj|&kw12Xjw6w{0m|giGj_xV)X{EuCAy89 zWJ9$gSQRk|S&QKRMYTTGV7i>tzN|$Rpy$`o{FGb~c!gXAFIQ(m<>!!X1a#QQHINV` z<(+iBBI^B|bZX}N{56O~fxaC$<*-9QP_x^?v@i79m8OBCSfqNz(f$f45WI{*%L`wy zPH^x=diD@>w-dJO{6dIG^3s*UrQyc$p}+bEj+Oi-5lV!sdgc2p=_FLf?sA5RJL{3@ z0PC!GbYczbr6bt6UBOrsJdPcTnNK&L#oa{D=IhHHvEKWB`I0*q7jl62V@IM;yZ6KC z)AqK)BFDXB6ZvI)a~j{ZRE_!jhogIgLBSV5%Al8B#WTq1O6~a#dQX4%n5%kZL%aOd zf#TTdJW+*N$mNlwGyLWaXt@yU^bMGQN5S#Ck*kL0rG9U7=!&v6bdHDO`gT=q zOjbNk58)&P^0BVxu1$oSuc8McZDamU^I26(nzXTi`|?K8Gqxyr!Qhfm5vB)0>$r28 z5N@LM*oo1gfVFQ$W#|PUfWso@W*-m))z){xF?`&6shZs|2H%5tZ~91!cU}Bb55}(M z=1^&ht65R)6{hTq>3S@lLYt65-Vx*y=tDqH`GFq1KDID*#HOwO+Ey~+y@|4w@U5s_ zXJRLmQGSAEleo|Ui9Mdn))#1x)t;(4cBt7!YI7pMs|L}!!NJTLYlghgu!230&fvte zHV5RkkZX-(3sN@gvYgnVn+O7IT;E`Lw*7lT4nBIPxWg!R3)L%G6wDXm?l!=xN z6w5izB@61we*f;LG>@l|B0U#y!4f~nxYcp7DTfA(Kn-x4Q6XyNJEf2>%5cH`tC$gH z`sk!B<_;7Y#0~9c(nzq2LYTVtgz7ZL&COO8B1dO)w!6rP*$&cjE7Bp^*`%7+0KdaR ztG=9_*Wg63>gvi%%>XRk;qC)YUsTP000+Or!>E|(*j~jy#l%TT*n)9nOkMz)@;(KrJwElz6Ptc(POne%@i$Zp#{1eM8-O+I1@`al4*pHinfeIp|6I3FpT$& z7p0CVJ!^V3)_StoH5-^u5R6FPLhvaQ5X&tiE|5~q5&k~c2gf$(2RKWbenR*<^t^un z7Um9yBqk`q)y|aqOLh{)cBDM-3hf+%fo>TNaUDyQLAcm(PDagWsK1%Zz3l{{c2}MT zg0K3(NJ9F+vFbzF942QR=E4C=R%q7JK1?e~<4;q#w0Vi{x@!J`YpZ>)Q8t$}&4 zI3C9X--6t!@;vmT4$P+TV*VN6CTE++J8HB?w!rnR<3w9;er4 z5!+;XG4Z#^@%YmZTEX(7!}j$YSS4U)8?B;jkf-g=rXsWfP;)$L;_=WkJUNnfHxr{^ zD*4IwMe6W;?JaK*>RlM zQ7yn8bn#R1(i7$Q4Z*bFyXef$L?gzes4R>&@%63NI#xb{0j@^aguSD~V;EACU^fH) zMioi7WUvDTwmT&WcpH`pVLQ*B1%m;kMjPV==CouD3Cr$%*A2Cs&5aSDjvqQYwt>QS z!7cOhc(Wd!tqK~2r~1jYiGAj_ur%^;Y1~a8CcdBs-!C7wB53q}Jtx;oQss*SU5)(m z>2u@|e<&zH2hsNHMy{V)N=7a5=lC+ecY7o8c(ZK6?#3^1R9-JESplT{@S=9$VN_al zdn;+{ur3!V7DjK@7YKC|7uSrkY&B!ntHQt9ILoqRN-p3_<7 zFGjmxJ|e`8hcXinY(YhwuG9tkA7pJAs-x>nOnaVbYt;0XS5dVuigY|N=E-|4Er^$k z$1Zh+f|7O&gO^KCzQS>b8n;GU*7?HoOJz-(-my8nlW`C7)%q4IzlN!`Bg6$to=ck7 z<*1!t_p*K+>F_HU$amhJ+aKA`-kyiDIoOURnZWKVMO_9Km8>lO*q#lU7m0}-DXZ}p zTHoTUU9&n3t9aZ%10=P8X+Q9#F1hcY95@r;#hDy^@Vn4;v;tq++p~4F{9YI;6A^_S zNYPFkW<_%47v*iaZcvHP7Cpu3v=_Y#x1N+YNAp5p24vvMi!rG^nE(8C&Rj-cqjOpCTM*N1606o?r zVLIQm!m-Vx)$|KcPE@p^Dw)rAj|N~=s zUhH5Zv@|ofV@VyoMR+z)u$$8P$FzeItR1~cBR5d6nz56PW2<0b)?~k&s)ml;qkZ>Q z0fFPOW;O(7P6L!^CjpB%B_gY%I}P@*qE-8q2qu1F8XKbp>lfS0V_v8ib6!rmFDsyU zpC7-aG73+{;pO%6^o?NYf_c;)>wywwQcfQ^7fwa@!N}tah$m|3xnQ7xo4(D51UgT7 zVSxo0<=!&j9+qZ-%(NfI;6O;;&eww+=-uigpwV&-m(A^%VkGyTyzMh!D&X!A*<>^q zGbqlNK;wS{csk;UY+z3mL7{*{!ao;m>jZ)1+2k?v=Hdic>~vbNz788W>a%~TWsxpg z2gFb^>PfLO>D*!z&wWiHy@-}{E=Yll%Dj^}P$_BWjtMYP0mq1>yy(H2t3pV>Hz@SP z0~N0Qi0sCB1U9oLuM&;NN% zLZNA+5Ml zjyaP?esn1@pmfJ-jq2C1cauYthV4yAM1^@YRm|O>V${fw5}!$&bL!CUhG@9r*byQO z@mh~-Cb}G6Gi`NJri+4|72~Gu#zD%Pp^pHPBp;~ORoZACVeHgTWaZh7D5!2qBm8w6 z(Xy-*fYIWl9xNhArN-SH#w>ZNXFrsI9Z&&n-tXg0@L~jqE3=xWO(TzB4+lWfIy&F3 zUVrQ|u6dJpo3CY}%JF%1#%lHc$_5Eu{Oal_6_A2#p_NTUXQ(mv#6N?efg}wf4TvZj zfK5gl^{O~1YbXX z7Le*>595HVrH{NE376)5N=W-(X;w|d=>iH&&5e~IQyKaAL5a6i$K)pmz-th68%|xe zXK)N&>P*e0t`06~2uJh3CQIO(h(k^{*yN%QoY;5FMPrpPG|pu-bZA%83M>?%%E!hFuLKZgz5P5z!1F$_tD0 z_v0Pfy(Or^rj*8F9iJX3LlspqTbtEQi+gy3)T>*&{3M%R%KH{*6V%M8YkJ~whh_jt zACqBvYJ`cNPg`$yL}xwZy$!~^V*6P3{l`4vyEIqNNg9#q#rw+<#st?;G?!4%^EK7< z-bMs2`y=ctJWV@>S=azmU+(Ms*xmku2ZsxQR|bKvRX=G*{yZ#Zn;Kh1nV^VJhXyaG zMmRb>T$BaN+)?`gjGr6z;bx%^HcK7G+bxkT7}q`Aos0Z%Y)oA2a3wkrrC5}(lWn+$ zPD2HK^88yEH|6vtnRS>cn83yNFLHg}Q?L=R>>a-dKw8<`zqAn5d-Q)Hkw|#?u{$_( zxNyMLj&8yrdp8aqHYk|RjP6|Vi;9@rE0(lpj_JD}I4Kl;rD`yvo?xDIH$K05z*U|m zWp%p~_$uf+uEZ1=g$D=BV*1sP0dohQd6M@dZ9B<3f1>HlD4WKIxhJq4=LAyD3Osrn z28A4oTm#_^Zs9hRn@l0FBKbDIy51qAEIp&xs*+qRI|FUaUYU4>86{cm04LEI8L4Gn z1uoDGh$XRTi$W_%qutM8>|it=sxQL5?3_X zS@72!51724@Gg_*QHuHX?l6y0k1*gntx_E>xe95gKmIsI{z*PX*&^q@1wRD?sah6B zcSYwM_D$73lV(OL19yD*s7$BTt6+ow*Yf(=&KDyv3H?Y4j~z>csD^e)CcW=Br_){e zV~7cGoj&(z!k$@F^rQD*G~k-|B)L~Gf6@0A)_-DQhPLh%OBp5rlsUjma)2G-=;G*! znP%jH6yw*}2*rrZiWu352t~dA^>iMU<5oPJPOcv($oDlW zKMi2hFr+xxKIw?i))()JEDyzouJKN?i_+p>Z|e+39nU*njDQ1ABlUfmk_02xr6SkB2@GzYhF9YBxb`~2?hj{JIjHt+*k zioc6uxt?7uK%r=Hd~igQ1ac7R>*BjQ9^mDNvCYz1yt|kMd!^F}rcBUKs@)eA=bEM0 znIKHRBrPLwJ+JlkG+zN|?;~PHjU!VbZGzrX#2Bcr>Wu6F5ORJM7F?Dui0e;dE7>2W z;lY{LaPBab$sdi{jBm z6qxQw^}3%e#@r>^uyi>pX6_pVAWW%YsphjvhQ#m2WY%#VySL=k0Bm!XIGfhHXMJ7= zd-L!#%D*2;0i_|IU>u(zj5tWeHKq3JW3EY}!oGKxnuV*vVCZ5ws3qC>440X|XQt&< zw-Lsy-Lh#bjDPfSNsm1<`ACn9_ypbRfvOo%mZ055WdO8jR;<&)jVKk9Je^i_&^K(GGY&FwBEo6t-%N= z=o+5Ua$1@*u>=h&e7s2^Vei2wkdur_rtK3nZObgF<$%ghWB1l6!gvZvG}XqO@N$ zcj#Kvh-Ab0+j%J}AD@+7?WFBx;KZkdTPn-ANv*U@^4cs#Cxi9UVaudV0!WAIA2z&p zE=kVce+`;hiOq)$UhbIlRq9HZ1**df2`=NtZ=(7<%5A;Qr4#jU<-c=yJrRo0d?(ap zq3P)hUBh7{xw74O5TedHxPGAt+U^S=8I8bl^+ZAcP@5?CkB&~{n zU9h~AhLWuAfMTZBU`OIhT9Xf!?_Q|G4q-P3E#%%wzBrC|6`prd1OjmCpUTzE7G0s>VcAn6Fp-fQ>y9deI)&QxD_HG< z5pzz2LTaT0Pp(ak32Z6Zyh2$gHgmFE^&vt3fE_y}&W8B&J%qh^Oa@@=Fyx zUR#SiP}1JYnZ6*%C9Qmml-$yUa`gBiM}!h~Ys_>8r0a!(#}6P8X14XMpsvStycm;F*J4q}1jTD_cQHnfoo% zBnb_#do$^iiVS+YM0ovIv@vb=jD!A1H9`kQJb71T{1&`~5hRuXLD=kb5ONI_mp8Jy z?sD$0Y^eC6@jkua2$xh+#r)k77VKMFV1SiRo1*#}OSyMu@@B>UFb^t7olC?)!Di*| z&tmv9%?;15`LH_ZkKaV>0qUF{9EzT08+XXI-Bx)wpMPBGXXfQ9QgOLm=)Wr};q~N- zA^oa+^?${ev81Rp*nts7h=*hi2XW$eFJ$%EaRwpfwzRpY5VjyMPrqgmwg6MNX7#z= z1)-(~?VC2o%@DXU zdF+(M+Bba;XDNsP(A$r^k*2#T!X{zrie0}JR(GMBb=!~ewh)VwskrU|Ba}qdt5{EU zEuA418ouoU-@L9ZG(M5-dAueF50Ob?4H72_lL8LLu+xv^5W|D2t8hc5aWC3m2bT@? zu?jVR>aAxY21;6~*G3POkSv#V0IR@3(kDPU_x3SsFs!;}+fOEY?&(XB{9h@khJF<+ zJ-9paCs(zM>jc=|mib~;d%)GW>d!wN7X2LUKfY6l3ytmU9E{zyT+`KiW9nvv?T5h> zxR8+BQcGC3XRz@=6=wEp8YAEeNj{^=Y)9Hk<0Rn38s4*>*uD$TG)_LZiZj#4F;zK< z7U5fd;cMOYjWrLtxiX9m<(P;?w)XjKj&)(!K0D0txWI>3&5x1vL(ujqB6J|tmeT(& zxF=W(tLw36y27tc(DrRP6e^Q>z9Pvp2qXtO(BVF{9|cW_zIR5POCY3zIo5vxbnQ^0 zaYm)$PIaLgPh;6?4<#x`m=wzavZfy1Z<~va1v}s8+))qqku4eIM}l6LqGkq&rTFb= zAynKm!UGFc|CCjx7Z!T2A!TgZ!%9{LYuw@_3wolxr84+6c{+KNw{(4X@lk|Vh)Q>& z2Lm}Pvjebhwuz+hs@gq`vo58->xI^)TSmAJ?dFtP^hTz;4)0klt`=oPI{nkZ(4(VM zHz;*iy5jCZww`Nu!{o@uB2!`51zgwl zv?54bQkgy*liaW$5@XShdJ66QAi{B$hN3Dn&5*Ei`PoUr$BYJ zh3E;exc-}9czs>hcmaI=xB%pMx4_Sxlr#uiR^!F!eu)it3}g+L$`Qc2{qyYJ3kBBY zv2N6R&vux{x>@oP<-^k&f_~aqsN!TCkju2BCIWT*p+i7Op~w(xm3TuQhKklW=qMAUC@MdLbkK?{GU%ixgx){R zvbbj+2qKRsbNh(Xovekq&OSb~?!0=W=8O)Lic}t7(N=6kC6*{tcDf5C^jgI-H;J(s z59`>6q`Dq4k{~9sK$t;$Z^A_rTB`7&y!^UICW2GnRCVg-_Y_Kxh1V})1Q1aaWlwUB zR^(!{bo5UNrb}g$nj{SGNL2-CnqS&(QjF(3M2yU;xF{7r^GMrK$QLM5q-k^9?aw3= z9EMQEk}0SRQ;qha<@*nFN-F%CqxG59OuZcNNH(n8EN8l>#MxL=FWk-8R;6L(h0i+0 z30a1$?YiR*w|*jX@HIHXur13J0C$AKO09Gh`4nIe5d8d#{r*h!Uj5MW{Qfr71$fYZ zxkeUxKYBdB7K-OW4tu}6arbZO=w^^ zPj+JJ_yi$>(s@uV9>FIRh>oVilTiVl$guVyPiNfU+}>V?D08KD zV9w3`wTA!t?qbgE##|4|uj91nY4*sWr7rz__5Awapl{5N&-d%n{K3=cwyQt-%$PhGPQP1K7Q=xYkbz+*Eogda+FTB7{et3@vU73Rm?eOGzmh` z5j0F%qY@-0P{}Ec`SQ&`67a+@9M4_O}B6~N}5GaqeT)Hso|_{)d3os zT#4Ja{k|Q7c~|`4|0&NN6|E}ywddL^Z&~exmgBP<+|r1W-~IfeOqZ@cO-N<>RF6Y5 zDBES@PX?V;5YWi2g?^ZiJ+P!B4xfq!N?~P_Ts`sk@v)^_S0MYa&$rQc5Mz76j|&jx zhhz>k;vOhNp98TFi=atbjFKRKAt9@Sz%RCy1RnjYg$mTrB;b>DlSeVzh9zE*b}!L%Hx=wehEd9CQ28q21feIen)Y3B{?>T)A!K4`sC!a^cFYZ zuKgmeM5Z=UP(kGiYAc>s5jz&O6q}L&8?~4~L`2M_EMG7p;1S|*i?IVvv)XD?!R@qw zlYMw6aQ_zC;2CEonz`RVSiLmw*1(>(KW2Z%rzJ;+9)#nn#~(!}Gu!tqL>@Jm95qmE zJ>My^;X2~G(rA!u?^vL0&uBt=H(%z(&g-uqe@L$D(T+qMv zm%kEZ#P8^gBW>7LIJOyrO}MvUeH1IKF&w^Ak1mvjrobOCl&hB%Om+L_1pSfkWFjse zKZPaIV7ZJ@;ZR*1QrYpK6V>pf$)GC|@7v19X5!~jTMCvj9Y?L~vW#@Mc$tHw5>LPy1Toik?xT0?(XjH?nXcwL`v%0`o5mCJ)b|kyj%}F z_gXXe%-pkPZ^CdG7!bf8uvbG{7h?x~Q)7B-TO(sD1`9`98<6sju{Z)rhk}+vrTY0+ z>W@ZOt7X)flCkGV`mZ0jc4K>70m3nVMthj?!HTK|-aq)@7#d@O6!Mk$K&f zqu0b1HrAhBpH>)k+Mkxvnl8MrbYJEeJ73$gq_P3sI!gZeqs&t-yR&~BU zrggPHoDg=tmUaDX>vF&1oO*q+^?p88Wz2jY2-fX<4Nt@Jc}((o=-b+SI;-k>oqnl$ z-L>_(34eVu?Q(lOUujEgA}oK-5_lYNz0loie`rdw)!p3izTdI+zPa;xJ?VNXL$>zz zxC?IMyx`~f8Q%4Lv-R|IiO~CJ5;DVj`~BsZ?(@Xf%SP4f>FeX=9HH%tli8uq^9rN) z^_=ZyN87{IO`9&ikMrFv__@H=>)H9u#MxBWQ(xVoPy6Feuupqi$EWGK0 zc6!{sJP>w0eeao_dVP%MoYL)d`?i0W3VDp zS?=lSRn^tC%E#yKXVR3n%E3 z@&6dx;&1cvzCLbCW8^#EllO6ZI38#!d)ZIg5~#j;x$$Z9ctCW#Iek967VvyNq-5lG z?pbSdwdHrI^cUcJdG3Ssd_AZed42ePD^mXYgRn}s^Wp2DtAMr*Kb4DzFdksep{CJxt+i>jm6FGHF?g2&pla@==1%l0N(jNd{Cb#H8>ohIHJGoMAR+6T&A|HvV%s&~d%R_YOSMT&uRM1Z5{Ukt4Z;x?NZs0B(E!gKYf8&Z ziY#oCTrT@v^&flZxG&S?WG0qHTdrP5!)`5?W!0y*-QcolkGyFK^%?}@j;rhd1*M=1 zRS<>|94B0U)@npNdekp-eqH6%g~iE1>Xo$@=xBLRv9SkdWS2wTQ}Bx5;=;pd{dTFb zCXG;4-jQvNz5F_VwZ6BvaF#uRJ(r{s;1+QxT{c5iI*d9^u?ed3(5h(UORSNLfF^p} z-3zT4eREj4)4sg(lNHh?A1PZ6bs8GHxQJ6<^(s|RHlSYM(ndkc%X=o=5>c3?#+QiX zqUaI6EU0H)=z=l7JENBsU0V1c3g`GpcKKsFg#sG!8FKqubk!`~AG;dE8+?$6fAPd2~uNSaHioJ`sG3SpSx$*TSIZE>j%{PVV1&bb&tqsA2J<3U*^Y7SDsgdL+U6CQ;Wj>0{Y1-6xiOTBA* zFF`#Hjw3PdYyhN>B=gyyV17Rbg|j)^HRc!1bbI|R9G~TTu{mUy2UvA%J?{dxB;>ah)_uV0BKc2(6*i!T|SEGJK0EBbH(4(}oO zN(^eBVX8u@^Wr2+NyT#5@}6sGmK)?A$t#gz+~Y-+;oj(=DT$MRQI-f3HaB?endMBv zNL%zL6!NRFCWt~&v>>`C3!~v-0nfyBb~5agD(~lbC@^VAZ)5o9i_XX=>(F>+&vx-E`q%$ zcY`a)9v@|Q(LQ|LM;NX)9TLuOZ%F^JkdvcG%X`M=T%OGb**-1%(2@ecv9HUK19>q%RqODhy)aZR%QixhU(OIato5}Rw|~f#Y||wYa}o1rem7cf8tLwtuto`XH`!d z`boEVSWV)O@jyGSV}BmTy7bO=qHdAPbRVZ|QZ?MFVfdqLp~ZLQ)PnDL38uwF0+F?I zNsKzO9&VW&>UTx*)=iv6S@qZv){Z%p^iYO-s6NXUEIbp}!4)6!hLHIM8d4|&_Dn2~ zIWKH&50_++G==i^AkcD@UXA zrjcU*j-1SLUS&bo-Geu>aO)6^H0f_FzJVZr^bhYggdTKNDZ;Q@VEU$6zbNU~HvFl9 zRTvR}JIBKAjoWFrn}KszX@q7sCgM`RqYEHZxLWGi4Ghbk7Jf|yI{KIH^R~95r+iBK z#b=8RC-Th@E*RTE2>tV)6NUDA+(;OrFksT-)@OKAcF`}L#_AL1gql|HM?;9We24`b zXnJo@WSR&?KG-@?-mV+Ed(E3wvB;I7O_M0Z%Gv72Q`F0on1a@EiR95*Dz$pfn|-5Y zla&)Jqx$6D9;lc)%^#W$IlhY!DYv*7-k1E7fQ^SDO=+`F<*{-^X#=JqWG=ti11LD% zmUV1xJ>CC2pQz*XS@r(tWOmcMl|MlU2S}_U1N;0uZP?vDpzg*xPDcV-n8+ z`H14ZJdPQfNqk`tlN8!2Wo(_VdV0=GWP?~7A7}cbOtO{ZHpMa55tM=mv|BN*76bt= zQ)iufdi)e1mL-rM;x&iAlbr?qE0$eQUMbZD{M2W8t>2-^LDrbu;E_FrbU^dS+(s)W zA(K%<%?Uv9O%|wbdqnDu$f;(EeC#dt42QEHydWUVr6Z2kPxv4p z>`uOavS{eSb1BQgYM(1eCP%vC**z}^1dF!h7t8Al5ENiwwY*-?MCPqW=73)WX%sjlcVWqA&azIRQ(?2i+P#U4@cTX26mk<4SPpUjwr8){;J)cAu>WkWW=JjGpZb8Mi0uc`3Or`%{Wn#58o3Uy z4jQ;DIP~#h)jzmGqnNZ87M5dd;S1_Xg#JmAY8p{tS}MDxV5*5twaoGe>SJLV zDzib5)@qYb16InT;(=BlqOuM8<4Gz}?2TdQg!N#-;82vkvv)~p`D%Mf>3QvD^rew> z?d}~!(BJ0H`vyWWu-e7OCSiZ_i%rC8Zmuo}WTf*{Ck}&C{LuXwED5bSd00BMJ!W_` zq(0=mbV#%B_P##@LD*~#6`stBUKo@H%jIXkB=~H#5UId@2_hHS#b@(5mx2uh?hqyr z@)|(!sdP5rjZO-X%ne7Vq_ZYukJSDCKGg>Db99o449das+gS-IdEB9#&oDxIp)Cm+ z))gGy0p#Yb7|<%_tQ0Vgr_knJxB2#XG$AMuiOHHPIF_~F9=Xl0D-?2Z$L#dK8NL!Z zFB5|VBIgD56q+R67Ov9%6#cO3$%t3g%A$q+bKO?3!x=_!r>|JQx@2W$(>~I9+Qdj} zL3NGMLUtl3bCk9Wp%HFT`6GCY(g2{OA^yiob+wK+8s zTEU-y#JkFydE$~KzXm9mb7b(m_z`QoFGVBw*XAHV833*9@uRU!ZtzqQpEXy}?A75Mj8SVGMbSZ)JgJ)4`{ZgeVOU}i zW*SYaOWoTWc)``RYjUQtzVX zfSW@k;IUpRS+gwBNiY-pAaP2|q0sAtKFc-nAtT3HJTEtT>&`Y|3Lz6h9clYyD&s## zY0|5Y9Y<2G#^q{kMCPQ^n>ZzP~t$y*jqqGyHuqoNwS z8=y4o6=OvH&Y$I#aKR-zsc!0m*RKyTO2gs-1ucr@$ z7vA^!R2)c}6^~1u4^qQ0&L}ovk4F;|YiXRXF4w&{ZYB0{*ulq!{XILy(_+C(t2zd5 z#%GYSecdlbQHKPCRW{JpgUOxsB>jf(bTuNy6`zxRIj)eZN0%Z=z!7FpjgyH!Kx6WU zxR(p4A|4JPEoD*LkvfA6#QE}L3-WcqwuLQ0SdipD`?>n>9$$y+Xj!7fg+fBS`6RHR ztHzFSfaLB&40(>8nWKiv;ySJ%Z`)hB%kGy7WA3xPk=^|BDHm%AH^9gMREX%}4(HbG z)ocaDJthFlbnTk2)OhTZ#n-!HgRkOS5M$1ubB3Sk@f31gnalJ(#y1;=yi7ZR+ux*x z%k_S^iV<{q*1q9` zw|U?>3wd^=#3jSbn+Uf-ywqpxi?(vyN$|H@u9ALf+XH-G{-@|^_55A*Fv$NZdSv~eB9SN?L}(wHhQD@}l=j_hEjSi_xc$OI+vA7fL(3!H zhlkV$guuh-@PFIQp~4UA?TJS$G|MIjXT)}E0SOB%R=)qCxa5^{SwGS$V7*#dpGFM* z1}d9uQM+rULC)&%Dvn9gDYhq};4H6@l0JyQkAOKcQA&QW}pGv>tG=hCazbr>!+nr9hq8Ghk8Q2Eet7#TheJV;_ zInJs98bPC$m3)8ax5^qMKtHJL=dn^*ODm5J#}8#4b>r%f)rF(N zGexG-k?~lI=5+v%xQzj)`32d#HB6`Y3z>EA&vNMCjdpVrx zxHA>=NgWE&^!VMmKHkuMgs>cT5j0s!g}}&AU!%P|UY%2DMgok(9SZ&WN1ee*e3RL1 zFVCCYr`GPNp8@JkH3~6~D%8n~l4E@(U{QHi1ocyomT`Mn_k!tk*PNJe4wE#Hb3Eps zxE99k)4!q#t{c|PcQG@;$74c_ii$A0RaofIRmBo%);zuR37l`f^SCa`Z&hO>_<0(z zN<;WrH7wt*aB^8f0&kBMj$1#SAjYVXUlnV}dM|UhEF6KCR=+eHlwJ}i4MoB{H(w9E z?yOu+qbd3^r|gTv6J12b7ch88ZqtO%<{9o zRm-h{>=MwPkm>1RzF(3I4Gq${)L3X&(;*dVGaDz&i{M$pRJXL4Fqe|) zc^+ES6;5C}lN|DU7ER>i{9k8=?))JjBSFD=mf$aB)p8n*0A%G~3;@U~znJ`mZ1_0e z7=Wymhk6itcbUh$@h@ccavI>f1ePcH>)muQRbRl+5IeRTNGkg`1otB9 z%RT$rFC66KcKa0VD)4j}?b^h-!as_4=_Zab9PH@NdHFHD7ee*)D8iyTeW~hFgw2^56!qC{H*gCvxAf3CJ90+LoZ8HZLB$FQro1|QGg+%XnB1ZnPWWhl0A0> zQ@M|+vAr=VIvssH_khjGE;;38uuQ!+Gq#-f6hMZ}RUsukGf zBMM1ZtH+@@ZlZ5{r&sx$)k>lKh3FKx?mdW?x*df= zp;U`#EXG-d%?q7HzH>^sfVs3v&Wbdqz25iL-Pvy8uao-R)0y?s5fJ0|-A;8<$OzsX z9BiZs_=UKfMJs!(>olXk6MMvo3vaN@T8_?3&Vt;0*Y$XH;7T}nnAL0Q$Gkm4HyRr~ z*uilrOKkg@)`iIuj_&YZJJK0Sj+%r|rB99?%zt*$_K3jKPMCo3FAT4@vavEs-8ZhQ z{lK%c4F87qjAjJSc({Qo4L0cJq8hLo@Oj{WRtFpC(%!(_TsB)(Mi}M#)vlzNZ^S%B z#%1_s*u*&vH42Q9Lqm9_;~(uJo+{@`w7~#56rG{H7~;E(0pMeu;<48wQc{`^agaBy4hXu-aAF855V@#~9Pq7W=)$Yk1tU#oM?p#aguLIoUXZ+79p6=dvW**Xq^nd)f#tC!5~E%) zp3M&1>K#uEC|oT(c`fyB4v8v|{;%ua9{aw;yOED9IH%#^U0%-Dr?<~1(QDknvMEAi zLrVO9@6t0MzL%jw!GCq*i6lkQ`?rtnX>{j(LAWM%DoKcJ18cXvt?h)S7xaO##W6)b zTv)&|T^3W{{G!{R$etN_ClT}epO~U9z%W`jsINOdY6qP;{~q2Jrl~$=it1~Cy}MxC z{Ll7U`r4M_tIg1l7=(?JL82N>(m%2#bV`)kDGDQ`C`zKx184D8w=NrF06fvh#L3ypAdL7=BNF=fOYcwGMCVFGaSsKtG!m%`EIf@I0-Tz@TuVnxEs4RaM--r` zF>H$N(96X~Bx&KLvl6%Z`IIlkl`yEHjyT5xzGh~6H%IiSI=exvH6_Nc_{#@$sq5tz z*NaVoSHnhXi#4YiQr7XDNtmB%P*4=9bRPO4BwAWnq```yZ;TR=3-@FJcMW|d_x5w` zi$udCLLaP8tCtmbbiXZj?j~ylv%-{dQ@MW#RKaKdt39@MY-o{jE3NFK@Pi7rT+jf> zRQW?4FzyQ(atexr7YEX zqW^`K^u@8h&0@(ZyU00oO)s#z+wk`l2yKbJi> zbthYg4=;qp(~PK&1Jtu;89N!_UP)=NC5v9JS=Sw&7^xotrk%E zNbU?36_PRpiq@gaovl00&UTNxoK$W()S<+?_2e}<0=gi}S+^gC+9FkLKoY;RAFi5%WxKhxX|i#g zy+6B_*GQ||RWEiPR7#&(rS-`*X`CxsgMNgtX{|=QN5Hyye-p~cY+WQEzMZz>jZCJ_`m>L(FOrtLSAW4@cyQ3kl5q@)A$vBx>rIQ3g zf%f+22bm@Hki0@^Mx?jVa-t4&8dH@-k_JM)c!CHdkZ2?c`WoI%^hx}94cecW=uZG| zKt3>j2D~B2Or0>TN(%>bS-Y|USjKx+=;Qa;kfVT|*^t0!oBmgMgR!Sw}iV3(H~4Z#{M zTYeC|@6N1`G#W;&E_oPD*zig)E+c<<=^X8IQM`0qZL<6s_(CW!p{?8LU*{vuXYqJH z^A?=XGcmtf_oJN|+4$U$HA>KO0GAXRIba`)cnHcJx{Tf z-cOJIHT@%8q4Vr_yHd!ZmxA-+v(BvArB)0VVkobgv>bA3L$qBg*?9B_gZjuhxBIXL zFtnn7fKWwF>DCW6%OIu$?iRrjR5a-0zna%uq!XZKIfbz^01295GYEqu>_cVzGN~15S*~- zwA1{BUC{V)3-kMyIDS)RBc;6&9*<>=#e!Aw4$-US(c+Diak4d$pKx_TAz8t zzLaH8EGvCp!-uJhVruQF5&Nfw5}t-_*rxb=*$6pUu@_~l{1vE6@?-Ge3Ng55V?C=@ z69KykmZzg62KWNPGOa8P+_A4lqM!+!%MRbo8yrONzum=q!?H(f=ST5y{sE3z=IOon zM~f}MO;sNCHeKjPM~M4aA5|D3%&; zB|SPFnV=DBXY#eJP{)C9IwB{mXEQIpkAZ%PU^2&g`_mzxXiPmDh07by#fGQ4MVk@J z6|cxedND^CrSjnu80Guxr-DV2@@7tNY65Gde z(={{6Mrw@P#|wqMoz?6Qe%A2g3Nd7LNT0=Pa#2{nSZa!tpj^m}gZf)FD{sGj;|K)A z-Oy2&rk@wqv73yP9^<~S2akH#0h0=8k+^6X`I>P@u5j^RhNo zb_=SIE>wYVt3Q(sa{De|enhuIXJez|<$2>o*PA=sX*ub(`9XoLXrjxV09@q-S%lTP z)5Q$8To*)ll6B*ZBVv``-gwLX){u65{@o=7x{0d2sC821sb!4w$@}Eyp$Ok0q~{nb zfiYZx%md=%ov#Po@;~l-tmXZI%c2TG^kc=fW=g2Qw%$o8(Lm1t%ySw>!$Zt{$94Wr zcYW@Pt2%!BPZPj&^xbQI7rl1)%uc(+&W}P#zts}>IlWG*2cY^0reprl7Gg5%2Nq278J z_A7a+l|$tv`5RAwGwSjlRm={?v~%3bBb5x>kI#RkA^Digh0&thah(}s0XQRup9d}R z`m8X2UJTmc5vZieiBJZ*b(&GOg2_k9nzML>|B=UGO~wv-R85s%?pI$U02(uy90T^-+)5> z3~>{_usdu8Fi3!w9tsO6;Rw&w5K@s;^5(bnK&!Ih)pw0%cj>zTl{1S14q*Y#Ss?3tQ@sSA?S3%VW#)O7IEM>D}Qc??lpGaPHj4JI?@n0)tWzm$gy3*Tr@W-Wt7k($}LjM!7y4}{qS}4c;?eOKR(luzhfj} z!iY3KuqElaZmV>~t?!=WJ3LFO;hDdcB>_7@&e=_S$19WQtjl+hCAP~18Au8_N8QNq z5>rLlFI~MM_9?ivg>Nnx4S>@OD;c}&%PPrWU~5~dczaz45NO58nDR(PSylpr7K_XU zw(SZ@q;Yq@3iEN=FTg&eVQ;0I5V4B%jNU<2QccUejY%Aom>(e%Nf35YiM}Onrx5(xUuH8mCa)dU%S|CJHQ!Z5 zU}l)tUS#t`!~Ng0GGRwt8)NO-#cq;^jwm!SMmx^&_C2mg(>` zwvk4dHPAM7Tao0&9zY*p`mHEe@LjywoP-jQ(ZAOV6pcPyc^9jLnREx&logT_{@s2> z<>xECkwcR+>s=);IG_&@qLxHfL@1$4LM=c&t;A{s=NT9fFRVrJWx-sO%4i3mESJ|-a-9zY=gc&`PFX6}m?haTjC5mbP=j)3(K#$pMZtI{4~VQsP*p+c zhI6<0zOVk`EJhfHFitDGeB+wUmiuX!%@Dpwm2pkmx$s{itEtYb5!rUl{kCuZg99gl zKkL#zhgF}r%X{r$R2p7Y5cd4(@3RVMK;NoKBHb&?wk$E&(gAK(yh*fjmbEN#W86X$ z)RGQ;_5?}2jmS;3aJd0}n*bi9F0D$X0%QCw^Sc&c9Mv<^=4y>Y=XMjJL4nXOOR9HU zCPP*p(LUn8qJ9UW%RIHUCV82VlcE8wXVRv2)I9i~G*Vb0nuTH4LJOI>S z{n*W{ETshKA@=fw?tF%beuz?7jH8T@%XH66*B_a@LX6l79 z`6Q|ypY~Teu}j-*1f-LF3O)!O>fEMp#z4tUvck6Thn?wW<<&Qcs^HVccLCzw@U96- zHIC|hWE@`yx)o40k=fg>4F2kDh8z>uz%oJ=ejjr$Uh%{FUr=@KR9mih&0cEADcY{w zET_n=1f`AZ`#^oZl{oS>ize%LH|k9@NPbCS%hoP9Kw0I(NgFlans$@rJ09X=;+6)a zoY<{i?S11@Ej8pTS!ZUZc+n-1fvDEyzY<)ndx&j*I3U3_641EJ0G}hqb_nKFH9^?j zB%e*zqKLtvRusace^qd*wIi@c+{W*>F!-co-`!?(w}|Yn5(Krzr&yE~L>=g++_iYHuYG=fOg6 zTB?ezP?UaSA|tYZA*hfQ)R4p!YaU3;WRUMDWbg5eyMo(*PHZT(bXdZbMjWbX7@DXz ztk)NqTJxrae3ab?xaomO?lLVF7d%b{9aypxLW-~ayb^X~0W_F8re>fQ+Iik*HF;m?pu{Y!Qe zmBRlIRddkT9Ug(;cihR-ccRFF14UlitsWa?O149yA5{#l8h7zo0(^sLNSFIKD z?=u8l(hRJCl09gYe&nzV6Ve%)trj9>6Uzs1Hju+6=Onk3B|A??(w+l2c6P0ru~_UQ zEroTRF7ZZjda-2Y0d~AC5XtK8RCK^1S{(4=aPsU8ch%0WR~$5oLSN`c98CIMVrh_N zqbvz<8_|33=lVTi*Lzpn>*Y3A!mw^JYH0oFeCKbrIa2M28fa*PkCLZV=y2if6HWMd z5bdL#+^Ep}g?^i&#DXHxZdlYL5)L~>S_5fR5N6+CFIez2Y#FmBC(!s|Q6Z+K>AIIhmA?feNidC;25**f)saYM0zR z@N4fQxjBm>%=9#xP7N7-j@nNXNnC3-5@jT^@hRO6z=XtWDuZNjwc0H{PpDgXlaQrB zZ*gKiVlo$fQa+%#qd@;DRKD;OooC4RGc&cB?CTrtwO^e(2dpZI$uKxUq(9bH%qMR zm!p+BgSqo%wC8PZryDRxS~eFe!S5kRhO`TJ#VXW*BNb_2>YxopX0*xd^0XIO>L*q9 zB<9#AJHXe1vN8wF(zsYu+pf{tN$gMtJZ}Ysv z!|JZ}bSgIUcQHrUeFrS~!{2-KW!{5yg2nj$DCSf@6QaQFn>=99I`|!any$DmRw`sN zFnx`_bRkZIp?KfH0!yhazBI62UA#@KUQ_tSVUfY)`NJZk(Ey7Ioq;>7C04cf!!cHs z!4VB6LzmS;_#&+IA}qX0omR`hF`(yjKeX zKoCT}9r{A0E)IFj_m>mL?e&V>F-%=;eQ8`-f(9+B!(^l6ix4F+N)(AL@5G6i_emN) z2Z4ScID!cTqsJ`xeddPz?J)Q`$=%*pD?P^Di5&ZA;SB<-{gnI21vNEYt>?)+uM2yiU<*G$vyKNrbxUXf-HK3@THDp(+ zCt!3t1JZGx4GbKNpF0%p5N^kSjyi|Dpe=8VRGsoklOg!_*>64a)s zp0KdNsWRwYF0{hF$9~8)*R2?%&mImhP-@pm@_oaqBY0Nk+yaT&p(_s90BkZ-_<(Yk z2DA(jXe#+ay9QQ{p-oewtE4gSn$x6FgB-d<$*%W{(qCcWzrv0g)^W8KF#E*X28Ryl2!G*P(!(gu zi)dLkxQFN^OMu@&wfw9cz>KB~f~+sIXmGXmi$Gka{o$L$2&1Q1?ev>a6`_XWHPiTe z$qLHsN%02X64g3ZF1w5>#_tQK%OJ3wYG(IkE1-#& zyBHFksrYXNV;>8GyHfE1noG_9v}~mX64kQ~J5;p}&8{1-E49O~-{D%d4!j)p!<#~x z^7+qICeht&Y%PL(LAb5)!Hl1GF)m@!#ntINPcbe*lfLf^$hCtJ`0c%a1{=}=CA1^X zQkx&A-NxZTY*ZiKrrk!fo4CIz+W-?qr{4+3 z>Ua|xco42@^~;GR+K&vlNLnu46eDDRh0Xv@T{2UuL>;=smPqk{QNI8juS>wTYuk{$ zga6~ygin*(-Oin4Bgo!EQ134X;`#;d@%3OCb>hB4cNv0HIdqX-A$vosb<-x2J@#Cz zgTZ#H$Mr*Z0rovEN50Z$u(Dr5hV$!`0r4Piy@c!yUh`xUnFo!eg)f9Io#d`^9FDZx zRh*GLxg=ut@N*oQ9C=EOuPUA7PvyAf-@k3Vgbb8R|DY%i0ua5?i{e236TDxP$ z2^s8fj(k^+L;1}SZ<-O9UmS@miVOJ7k^2%d(%&56OCV$a%@KQ=kqAV$of0x|2F#62 zDb#;Ca_d#BBuVm{BR`bmK!0(hM4mkKH%I2urBHu!I&GfEz&c%yaIrcy zmYdHj3`x%hD~t&P!s>KiykpCghS4yG(GrJD?46`Zxe|>0LkBnj$M}aLA@bxnvG#5b zIAld}R)W+3k}LoYgD0w9?dLbZc9=~1!!rRegb!za?nh-xv4AK6h|;1Y$>N4*SJ%Iz zBuQb^J@VVgB|t{*{@2J$SbvPn_OFq{4F5IqEa^W+E*JV^{stGXD#PeX014o3g%@_C-h!h6{7TJLzcS}Dt7;3Rx;d^|ck&4WY`)bYG3 z>fvt{4-B<+XBUrf(T#Lspt7mbCDqJJBr%e4t+KU=CCdUCugN0L(Y#x-U%GL{{4mtS zAxg&M+iUynoCe2$iffL3=Y92vROzOfgjA?$M+4}Q5V>iI0d;0U*cS>#i&=Hf(afMq)uP`F>-Of2b#sSzq*bIw7?W`= zq>c%(^9GjN5Qh^;;MTPEhwTbQJ1KLx_I0UJ9o20}7Ym7U=!EowumZP4jstMWOj2I}gQZRsvmuut_Bw!K7hz-+<&N$Y4XeN&X%YrCZp zZYs?Pd1N)IH6hury6?}H51YGrgN(d1=96o>*DNYs%X|y67)8`C;J0JL&KTK{(&zNer4F(F+-t*mx=08!2g$1C}R~ z-3k|Bw>fk53AG;;&=|29r)+fjB_7qALcFG?vfJPT=;-PVI0CGc7?FEE|53FlQ|=in z^%6Q~i%CN;TBfMrIZL^t>nYWn9z`itihmGvHIrZkGE`mkL4@T z6rJ5l!!H@`RVlR*urL8?PiPO5p43~srD5sm`v*_9T-{p*L$;aH(z)Y}qW^)@E?1W( zXUNtnESsTn!4)}CZ}a{|d_=BWj#E-qtmDW6_p*;%q)j z4Vi{p8bkVu?kp48ihhG6_eIBwP7L$*!^br#R^SEz%2ac$??yha=)so?{IrVvVF=_t z_lpdZ&bU=Y8pFAT^enM9RFY8M)l)KeB6G%9}zDfSuN*=Wc&pxn9f(?99U}VpE&k!ie}S z$DVS!yJwW^pQ(`e+6t^}zJS&yO*bWHZG!20(VF&YLH81m??V@p`qsh`Xi&cx0d=!Pj(YAc+<0u6Il)}TS(<4WmVsu@HalXZ&U~Dtxp7ek}_l<;vB_w|}2JxirRdF|V(|VVHZz^=5*HTPP@%w)RD1+NaP0wDc>T_})eLMHY99?vZ2dmZ>D{?4*hc*EPhh z{#n>bH%2LaOFl~+zJ_g4Nr-aS6_;&ryb9N|LOmnU+p5mZ3Rh@SCDUf5`hmRU|C!ffE0u;uNbOSQgR?eD9pPbX_-n8zVLk;s=*Ct_SNLs?US*|B1<*(jY|Ad> zyHcFHpa;co+n@M>S-kVZIFK5iTn2kPel=C(#F zENJO!d<|nNEu&-eKD8J(wHN*xFtS-#(^3Lfk(=DKiGWOwkG|iZt@GY6Vw{ZW}xQWYzF1Rq^Xr%xQE4qtfFpmo-xh;Lb`6ZPn}7 z#7T6(IGNIG!y4CLn1w6dK%s94*D@bqM2-d8=vl@6z2_EuZdAsXWln;=+Qy$koH~8U zhi`Oc^l5MLcttvE#$$TS%>3+{1ih+8C0;=VTBJkJhw*2IDM0Jwqe-}L=b1`M5w;d1 z3tG)ZuEqM>D`vJzrRY33hoG4w`GweZ8M>Tu3cs99j5CoUb6VrFY>G*oFEoxpJIs0( zvNjY;QyVMj!{& zOe=m~3Gf7%A%ixKN}6QO8?=b%9QzFS2pW;L4)pC53u9xo3s%S=ch!32jKYfeiq8Nk zL6(rS@fq&&(%m^hzXJ;c`&fJCPqH?c%*qdLI1v7q{Ja zA>IO%5f@UJb;a}`nMOkkJFxeXe}!8lt-f%MmgKu;x2yxj#AWWe$HrYM87enP&@=$e zELmJEKigE(D|QuooHk#TTA?Wpbre^w8?~+jL9S9&uWNiX{p$=5BFH54*lE|aAJL&K zRoaFUvGqDiE7eaX8vi2i4c}zj((B%16Q}LVSzpx^g@*4ZI+kES)bah89p$haR*Djl z?ulp__2%D?3j~7ti{<#SbngH(mYWftqde4@(Il&EFhFnbhT3O17)|tz#Zh7-@gIrI z!kPhn5+Dj^7f}kN?oEuX1nRHiG`-o3-2(@2x5k z3JXn2K2Hf1neiK5UUc_}j|FsAk5CXu5sIUq3BHMFIcXb>?q;rv0Ntq6>%GrP2OpOc zd`j2)2YoL!yqEWA&A($r;RSS0v4z6&_M`ETd`$s@!@oYtke{0!A1JV_^o?01sW!XF z6@`n*gc|-+Cb<2Ka}9O4yBRppz03?}&|Q_!|D%A-sb70V36x=c0o5A z{24nXUAk7~oU+;K5`TU~`Eq-~$JNFb8>v`59)`+vu?4R6pUKdalk?j%z}Hq}`HXfe zQtE>3W%6snR4H4TVKQ^>CqQ-xRPQ|hNm3GvSiinFqFE?!7bY$$n4Y6eB{WpAW;kSq z;o{Dte&1PCWXW!pBeqU9M3RNPJjw~%9D*DbOH&AS+E z8k9)$Ocp9`{|bx5Y2S`y1q5=EvSx?mk9M?Q&)u6Y)?}w_el{F0RLci9t-AWw3)sf2 zEF>dF;6hR~8+;Pj{BPbPm3}!j>CWd{7HIS15N}8|<@_sLNmQP$OcVl6Xru0}FCk>@ zlnpA-WrfAS)fwQsRaPblSbwacw0EG74@%LQiNSK+?x9!y8NJ;fPUwPS)0=%c(*^&` zA>Na^@ocU%5;9TUni+=hFRmyA^AUr z$H5duQ;gHQd($t&{l#k6RxGTuE!ltcik%V36a}=xYE_!)m+9eR^*bvTcA#WuK1|{6 zJ~t)*ZNKQpm6E&LQdisN*`v!`U7B}=JNxO}QQZh4(b?4P3KAXvc#@k`_1?;1ga2&l`-)K%A>`~71vt>z9fJ-H)xp1Bo%@T* zPkCt)6v66Cbg1rR>3kU+2j8I)QJLDyJfl=R>M&sa;@v`A zX8H~a41O_KZcb%XMM(A?v-c-Eh@oy&%qTw`LhY*M@qAsI;bWX=zwXaBe{@Q8#9bEZ zj9>LYVIDDSoHoCa<$Sz)S-_;kd$Ypt2M{&#GxzA`yjT?HQ&YLy!|FM`%&fh6@W~n} z5D{nZsrPD_-~LBiu7+Nme4$>xUTUtsfQHH)OIS-GgGj6|zQTSlWA>*`a~?br7vaX7 z#DO)9^sf^a;)6PVqj2+Ddza+3b?do#A;5>|-|>QtcKg66PepWyx;R@~gPE36scK`^ z>{|z%pEI!q_^*7QUr|tRpZPdi#bZMytHuGbi<=`p*8L-MB_4k@$yh$BHHh)u}ki5=q)TQBTfS}=Sv#UdU?r9 znUW6(sGC><=Ql_*&XjxNnFA{8c!OXWw07Hgg9az_jXZ`>)OmVvF2ty z%&3NXxk-1*?d=w+iiZUGz)D|qZ(nLJ)-tCay}XC)dIP+?K_rr>>X;=lH59=Hy}Ult z{u%$~w*TlF7MMaE_Fc|iuQzfSu+ECZ~W=Z&AS@IA7K42W+}F)wZNi zXWpvZTK#^_O=njEy>(=4hCMBBy^FA0ktha~(#GhZN@2{X=1N`$O)^W+w6hy6Z+Bpd z>L*&w<&^kxTVu4pG!0JCTf}`eK*f^Ijx>o2d4pXnA-M25wyAsTwRZif3%aQmT4wn$ zTyq6bgs3~y^8OVd`xiDw4@?TFRy9}Zau_l>BBmdoRBj1S<#=9HI9QoS7S~6SfV(E1 zy+0oR`HmmTsD04kkI0-9QH;v^N1;)X6~dd&7pmfL@)n`;3bznKZ-4r7^0h(}UE3Sj zHen@#aX-tt8lIpB24+JFPfr{?v^|SpSWA7YY*Y^E^#^DuqT?DB)S~mXh-eEuw;1AAssSJ_lvuIt7GZliW_q zO>iO&FL3wg4J2r6p(gSPO@;7pvQ5gJW6PQ@hF2YOP0Gpglu6?9PBtu+IMxeDUDhR5 z;zRTLP0Dq3Yqkepf13kb!6_@9%H5mQ6Qi=K^FwDefLE`7QZBvjXJXVfPWJipPNK+M zVvVELH?Koyevvg65VfF{a zq+`YHA>PSk{4;Hlr347#LEY`M(bSr-$LA;0lNa}ro*&q!Xg>8vxB0gk7(2i;zH>Tw zuivZ+=g%WePfRmG$|AG26?dhouCOn&^DK4+$I2z71$1O_=_sR#l4zA3tSY>t?o>>_vlkUa^z$&Ig5u0fleS=)cIb!D)PLHPKvPU!19vz{;q`GVJ- zt%VfW!4TCW5{~f11S9L=HiVGu903uZNK=I`@WZe z4U_$h%a>1&dMTZ?)tGp(hffnFh{J_8p^YUy^u_(BXXII_Rql-S^l_=NFUP{JM=I8J|zgd*my6lVzfHLEqrhA`?c~|n?~hwj8*My3B35- z*%wv(bQ!0?)h=+^{oz5#?bo{gQ-$eZSe9H4gw(I;lqYj?J=|W9TG*}ais*%+lBlLr zJXYVMk%Aq|bK~X5t~Bo_P+J1o-~^7=If*r^<@Kw*))?Q1`WgrXRFU%34 zE$HXC*cXl_1{bVKZ~m1u1H^2yGONgn4X4N`mj!3p@qX?>2+h?8tPHva{;Yx*_4u43 zr<0GT-TLj|kV_K~1lDm(Zj0gLTAoHXiLfN%dM2l6XR>WnB{~sG4BcGx{T;j=$6exk zATonHMj_hwQyR2R7FCIvG40dxhtnTwa|&-iku=%#J?j#t55+a~&DOBt8l-SFVoGIc zmiO-)(lUPbbe;j{l)0PTIo0;(cPcn3tZ_O0$m5xr#Wy00YgiIhtu!$Iso=P<=4Ujm zKqUb`puXw|Now#%f`sH)DEc!k$Ib#^8S`B1sXF)58}5}q|5(Sb?!f&*D=&PZ!-b_K zy?J&f^kW=sOV&L@bK9beqWK3n_euE%2R%KX#}+D98xuEn7r(5X?+%WE13h|Wp4`b{ zMRWhsT{xdS;(-Y#zF`S(?)IKRmYjzmBwY5s#>p(#QV#ipy5xNH4#5GWQyx6;lB}m7 zrS;xY&b?o>2H)baV;j#!ex?VVY&+O~Guo2@!BmtPH^rozU}#0ZaL$zg?z-Us5#pA1h{ z;#_Vyc{~%d*au{B^-ChF62#{}8J6Lz`)FN2I)Nrfm$oCjn3HPO)TcE_n(-$?uq1I% zhIxM(DEYYDg5+4XKgyCBov)P5;OJO0M9HTM@ju*TfOvkj%a<5k@%X^Zf`07LbG19Q zz(xPL`rzMJco7}t$PrU&$d{fAqzlg-pAC+Y0k&_IodV19E%39B>KL5&#MxEHbpIg}P> z(!CTc?e zdH{e;Pd10(|0^?n1m}EsnpF6jF9Zli)N?nHe0XX9wUV4B1TjFqPC)`<1g&@mq%kFD z$?`^f7tiDgY%Y(5&|)S8lZ0H0dI?ooRBi{Jw=W3|{+l9HNU2%A^5I9Ca+u3t)AP|E^ff%S~~vzvrD@@ z`n&X5coar^j5&Dhnrjp>kUMkR(75i>316Vo+{Y5WjJrfG7&0};EQ}c@D2tv?xuz(W z6M_v?UbVRUiJxA!xCfZfXSBGxC*XY7;w)e3cf9f&!?W2liEloUPb*Q{a=ClFJ6m5N z_zdW`zrJeHZSe2DSjfhzaMHtL8u__-2`7hOMbbcv8tf8-rFqk~?Z3l|sLO%E zpw9RB%)f5p!VdzFr4r(YRETYPQJ6`CDI5AIsXPO6;a|F*iWXvX>A^0jf5QPAk1nJ^*XQKa$J;EtErK;kLd`4DiOeU*{(q$H?xm}$9-p!KUNc{P zWNjIqJQimod~|Luzs0l?$UfaoE(?A-;}1hISPBh~!5)Yw4%UIN{o&GZ{ z3c6fU0$!Ly2wcnGAx5D}&cyf?Ep6jVk+(<8?%&;^__cUm;wGn^EN1hez#iUW&2b|A zLD(f*9dfAu0sxBJoF))h+o7{EwhsXDTgF4zpYx#Nwxto0^y=uYjin@0kRN9}{FwWI zz?S_3B`5b6bO8G`sI3o+Z=aE2@A}vvJH%po#zV`Wflo+!2NnTAWAC<>G=_4x9%qiL z@INzCQUBk7xO%Yo3L1HPBFfN98tEfZ*AW{@JzWD~jA?uqt`VjZ;W+o(7iFP39`AP~ zu`!A*_mBk}`y(I4T^&3be(P5rGrBK2)|DIYBa4>)M;YfJt=zgI{Nowa((NW*dke|J zP>sP#q>}x?EYR8g-(m4C2pzk0A$45_9lNdH_+V!G{h6MFWWSVl0yFk*cc=Ig1^3N) zQq=L!gp91!b(&cSoTvAb^bhs)#9uk#!kQ(VKim|hyaH+|p_Zl)+cwe;f`$jgFQgNf zu#C#G!w3Fs%*AV-VwuKHy16(?qiSM?QsqbeZ=Q9FrM=_m%Jll)@$rePz5Um*u^hvr z`_Iub{!@+HN z=8toHsN;Vo^SG;vCPkY7;4Pcx4*|gQrASZv13X>z*L}d@ubpqRzHL4adwUx|?SCDE zS|gz>PAvzh(R4-iLahVjx)CRl!DC~0eb?LDta)bVLe2|vL(4C~mY2+5O7Uw55`_~l zG?!8tUfkp@OK#rtvoHz+w&qNP=_Z(=Y=~cD9Ivhf+5m?soLJT6EfxHZ+|!7c`BP+L zb^c2Gdf7DgarYCT_R=Q6)<7jF@G!d~>LZ=ve5Ft39k@Xb^psj1^kD02*)(vE$k* z{lrwEgCA4V2U2F2FQ)s=$`1}9oTg@qTpOtmg$P%pPgDHEMspJvCuAkuwz9|vXxiMr zV>(rFpRvu13NC(UeKGwpFL1c~P-XZb!y*fYNl;N&U;$gdWvjRj51MxgkPV`D`IKIp zw5V%QeEmD!>`LNPBc(VZPNAGU=ne4`Tg&RtCq>y&#lWqarfybr^*hk23ZiV!Gm?7T zzX~a{G}_Ll=>9t}m$8wNimpHQ!)P=uZX^K^&sYUp?<(tPKE>A1(3-eIq&&WSjFRwQkDsp+9wu=C+fPY%SVzj1_zIX=?t&eAlkDa zyh&4a{z2{!XIvkS%^kDZw+}C~=$GSKqQ{(MM;v8G9G37(#_hg87mwN)*^shi)Lo|6 z)uwZol(M8V=X|Kc&KNzH-WxrX_)MLEUCR5rdRFFKX6#EWlb9U;yI^ttEm#SZUeTHF z8*Zv#s_mwFCfzjAeisej-*pg0NXE~}7*~1b$$a{+0pC!YR^(3g_|zCXm;0YA08Ix-ASqUs~GQ|axWS9U9e9N!m!2Wge;e4 zD8`N>Kn)4!i0p=3l_$b`;WTtv5);PlIufaz4Ja@7Iw$!R6yJnnN3m;Gf9kRp2BPrpxM!LN=^P!hu%e>wb9 zw8h|L2n2Aa2mozT=79}N2KixmgN7{Or~GzcsV7xF9kR#r3&op zmP;b4s=WvIpNv9m_{AjV{`$HkmcskGq~y#R?K-wW=KJ^AS^sxG*(4SVG0DJlD5lgCF?sqNCSMXBj^B(~$8Nn)?J*q|CyRJbdA{C?| zowDF^_t8kKF40m+45nd@ArwiVku5UV|F18j)8vjAJ9)Qxgy)vHIVL01q7=q# z1=Jml{Hl72mVdA7_|r-zts`&!X$xGo$~s6zE1Wuhmx)OFQQJsBx8#R|dr2I85YL>2 ziu!}Tu~^vD-LLTT-qV49c{NN*nt}D}$RPhR0H-OfW&ACV@L#nG?^eZ8JXx}uy681J zw&qWVBFjEop)OOQM)^kaa|K$1q(W&bcfaz7Ht|jL5oq^gXm!hkI(F=`NmcJgYKPBp zrP+xk*@;DsToxI5E)V7@StBdTCdHJyMU=)x*0M_`h1&c}#s$onNoik4L_TA9s33T# zKxpjeINz%8WgG9AU&&@WADuxV!>)w_x^s{GoHH}2x6DrEmd~KTe>dLOcQ0WvDfCV4 zTmbIg|96|dbv3)VG#XRAyXSuwjjfbc!m*AF^E(Idmth-9&2Qsd`&IXV#FO0XZQgdbb_ponpR=>dOO~|-u5SSzT9pLX-cX3z8=@q zb-qvyBhb==Qx0OO`JRt#`yT$ACb;OMW-TA%x-er@xO1~DC0`7%^c z0LagbtNy1K=yv_c^H{>$U8~znr;?ymbeH-58d=!)rHoX`g0y*7#Ci%Th-q`S|M>~A6(nadT7MqFHIA`#7~+!IN4L}6AOEDLqAhSH$#Ql!3nK- zeVbe$e++1$igiHrU-S72s@t2|p+^r>xB@FA)aba7nKrqf;pqn}VOK>IFwR9Zy4X{g z04Khe1sNOBjkS&yGR_wxh#BfYz>JpROW;qBu*(alKvwiPbKj%EBG)q~)xSjSc!J9luHlev@1U$Le2v+Zn=Qt zSR^_c5YZJXy*<0Wi} zUV-G`Cgd2r)vqsfZ&oG{s)XCq@S}$?6a`8()AyQrfBHaBn7yEVNq$<6V`74pa0B-g zK@hQa(>#J&&_is4T417mghE@m$%u`dp~oOC>JM0Uc_&<;@YoHMWuWas;jaBv3Yg%E zl1TC$;<*{|kWi=SLb1*uYmvNtTCh|7dX;D&TTHJhuxfR>GGangErGvdMIsaT?2sc< zTh1&;AugeW7BpT&4M5s@1ksSR4B!7dJ`G8GO_(l%Mgc(N5{YuX#6&qz5s_AC0*Gix z+QQw!S}<}Wy0t#UI!qqOFdB&kalj)o&4V~GVUhMgT&?a0e(8$tF&y%FP&0HHsQ}z$ zyp|*^T)N31{t=1(%qO@E(=ook6;74lu>EHW7*GN(9V}c1eoqEHE&yBF4Ah{4d(IJ> zOKA6sJj|Fj>fGSo*AXBi#UeUJE6!l_We690s?j~LO{(uZ#m2(~rY}33{MtLaBSCF* zQxOt6=u5-dmH4M5 z`l=YFFNv7MA$gHJfa1lV9pNr5T)rc%f=D4syJV@8|Fy{@-n%7 zA8Uek6uYc&_@LE)U)Gx}l_^rU{Dg}oHXSawGjZ+mhT&GzOauR;a>=r$f3@95=FEst zD4@g(^n8T0itTzUPKP8ilQM$s|4_-{IsuIDW*vtK|KkWqFZq?gppOVbb3y!|VNeIg zEDtqwCNI;*ae2Eu2X8Rcv7^C4SjY09pR`>RD9!jF6pN|2n<5k0>|-io+5h=JZY2kDeMu@d9Z}WbzXw6xmXC*X z{d)HA8sv#^YdAj?H#Mg)e#s|QA?tN&x&}bPNi&AJat3&R5a7hA#SDR9-P(*Lfg}Ur zF?wqOx&~YP=DNv;PBuv0;BZ0g&q%JS!VJ(4$*u6vi~2N&Szf%JMeFZCU|tarXI@Vh@KBI8 zwS0E)(2&+n36=w*(Yw8%&KJ3sgJ;aLv3VH@cX!Na6d632UdP8ksj?^+mB)N!b1`s| zwdmX;*26vcA5qCtIWG9~vA(otG2ks2vUKl3$Jiu&iVH7$hy|YP6;?yT)?&RHl!k^t z@t1<1=eSnhz58p|ahZho8G1$7v zB;U#ndgqGPR1vtwI+mWX*Rr%3nbrj$U=UaZcSyo)2J2;!7Wa7s!YiX@_{fzESpc$( z0kxj98G90EHPR;xSbo|Dek~bs49TXytq~n_39>yh(k4~_ZNyg~M+JajfY6~%ynoVx zBDq?;T{zte3E^3LC*TawHpzZ|=jo){x(A)^E^d4=s(;27ol zfB8Fuv~niG!sQh&^1Png0wG5e5Aiw%BEE^|S_Xcn0&p=GV$v@t)+%u`LJ`03RIxDB zU~$2)^n604NZ#`H%e+=uJFlCN=n??RSE1h_RL*4j&H2YTR7?w)2rPJkabnbTOkJ1= zph&&d!4}*#WPg!+C7Cq$;SW=)M!nr#)W-uounhTW_7?2Fi5^p?w>08uF77E;D_u?l z4~S7?hF44W0JZfQGyr-442*I5J4?hKr+ z`DFH7uW+GYnaj!}C}ODEv916L9v>zkZMZ#2|3pu5+W_KGcv`*Md3Vq$h=fi~M!{Pp zx(6u5001yMiW+)Ttmb(!M8S-QWFh0I*w0XkaBYo45fD1=g?B2tC_vJRjORa-p+w1s zaP4k8pMmM{ZdECSiti9+s~X3`K1MwA zliXWK0inPMe&nT#UAEXMql1PC=u+;osjI1KGy)iJZS~9>C7mFw5XGQpS@?S>F(80N zGGu0!hr@m0B-eB08s{9aRH=@aW!FhvqcxNEL#9hTg z=EOt28xz~FoAWCoJ7S>Du|TPG6~C_U@!UTE%mtogsCWlKzTt`p(cr~q+`97s<+s5p zu(Xt@9k6gX3>h4|P;g#sJ}_Y|UOYAP)SSrdTp}nq;QK^Ee0zc(BReLz3fS{u2Hti~ z2n*E@vCh727*k7B0_|>s)5Jt`K#$iM=&}3?WP$bA-4za`CaNanc`v%0c7^jYz}nB84|31><9Oj40rOIO}onRuuz& z8;Nf8&HG^vz%0aYl`w-%BaG7AErC({tP1bmu@;tQ;>R-tOAtTSM8-RU2}=?z0$x$D zy6e+13h2N6I4dorkgII0-+o-T0b}-qsKPvqi~Uy`=mc5r8*FN|9d=WJznrg;;kf*G z?&IngbOaapxLN_LJvXqZ3MyDN`a$1IS9sTEc~XUHxJ6XgK~cUc9k z5U+1lk^_vk3Raj1*C(yQ=zUKnG_dZr9XA2akavf#l6KOy+v_9|c4}ufAqErQ++P+M0Cn550FB_;p-;G$Mww13Bqo673BGnwpWPfo zfCUm!h32B)o!(f6{7Z&G4$vCqZwjJ&LNzA-dy?t*90*nIucz$hPc)}SQ<)-qw3N=OTv)mI! z<`4Y>vL&V5KiA*r->PJlZ?&irix%OM^l5SeZ`n(Y@C&F~us+fO|h|F0PlaO#I z1p0D_%uAV`J5S&VU=Uekv=v8e#L682554oGg)dgx(33kf zZnjbm&EJ45oU=iEeXq8UTRA4zo6Bqw=6c@?!>l0p~sVV zn}&{^ED^59tdb<7x&#>*ML=7HFsARq?}7DwO`cT&kt~&elYS)U(xVX8{q&g*r&5RkSv0XsBXwpeF7m78;~)c5_5TR;JKp1(Bt(KN^8)oa z4)I?E`W*`Pul@87ZVCAJPsLQMlNE^liyWa7h2shR^v^287(gadC$KauqMYKyb{OQa z9M3ABAc8nq6A*AkPyjpmfDJ3|j@pFo!UQA;(;xQxp8%7b>K!OJ1bmOLwhV_+A3o` z@|_yK7>txv9y*nIU(tdo+JaMRz(e6MTjW%--%k_YgNcGD3L(ok(u1ExNB`cJ7FsGe zxv2X6XWw|3j(U2DEd&4<*chvtzNU!0G9$zm!%D+6>gtO*ubh<`A&iJc(GFYz~wLQn=t-m^@jA$eLe-IPlNyEzJj1AJXylLSbBXJNW_v!zu#on3qywZ zZ1;!-AO%(U#*3C_L1fkrXG}s_0TE`QE^A%~PH#Ve8zbFupO33?VD|_nu3+Ti$I-m8 zd5!(=o^h~73}XeGn6d)$YXrzA`XX?MO@Bj-&ss^n33E#SklQw9C8V^&sC1iMmH`8J zJyfztU{V5J3c(g&(}%fow8@Yh3fSL13PLk$Hz6T8aEBt|1P19Jf7SdY%P^5QAg5f+`pdsmR%=xR`J#td7lsb1$`cht3rL; zt7&Mx+;30vLY`kc&?8wq_?kahs^%O%|8Hp;6t8p`sr{TyJ&##5=u36!|A9XarXF9rk-vfkTzhse^xImuV$?Tr_S zD5uTQPo57Q>0PbBxu|aT45eFbNTAkIRJ9t?Q4@rAMy~Sv}8lpF<>4 zXxtO?T*jEG6nVYV^Ei%4nv*l4q7T;EmbKw(9yKneflnt(+De1vy?p2Yns+Fd-xmdMu(GU4ynh179J-o*5pEV zhi3RSu}hZd!MeZ8@0XAT2=U#n1FA;rcJ|ekcNSrGy9bR+0L!`I3HD>nBk|+cQPi(a zkJY_+hMn!LWc5BE6WcGh-~BeV{d!{w3t+1xiXP}p!y0##rtUFLL1?)b5a3rZv}pxP z^k-$+4PwIA+uf^5CXhUNp@a4IZ#2rG4tXa&i+g+Y4AkVm;u+N0U3K6Qa}>IKm+p@D zfKr=Cim7SSY)QL>Msua*lZ!s+IVLT7+uZBz?C|;Q*~)e`-P8mL3VGHp0=hlli3tD8 z_EZR#8R{DT3K9eG>gY~y;>NK3gI;f2uIF=;VXz!q7A9Egkd5NWn(%AZ0*m?q}y50enyZ+qe0cdr6%FFmxpi_zs9 zI)T0F3TbA_ZVn7O-R&@xPP@ID_I2T_CWwJ1IMSNzk-_4?OgJGa!b$$(U|cwjnxD{C z%7`R-o5{&o?e9M(7>YfQCdF-g@3_E~H*&!?BX%6V49XdUFNm*SKizDnE_T%?^=jyG zli_kVqfciPaDcU7PuB&cywoG(M`se0$Pi8lV~Wjkr9W6KK#{<+S{%tTSszrh-{Y}& zSQSUrcMKh)TFn5Ne9`APNzP^eZaG_iwS5X{PI5)c6FHZgfOI))6$Spyuaw8ttk|!D z#}y+H#EePXQ0&?bS$o@jhkZShmJvPOi-k{PVbd6|AN}U|-YD=6zDehVE=WgokPX)f zO~`Alu4ak~iSnyv^E?oFi6*Ktc!Q+%lr(6&q%|!oOqdVR8En9IQxbF$NXBN08nQsn zVu~AKOZy}1&*&UZ7*JM^tWr3PRD{pP@r1#ZGEUP&mNckUbUF}W8lB{Wdg(?nXs{p!#|ey8AzV7BaBIIwEK$u^^o;PuF~ zP-D$6p09n;_&T(@$xnJUagsCNnr4GkJ16PKnz03U+Kx`F%=51ceu$1)?%C@?#pG8b zJ`1C&fhV1`nED*Cq>ltC`OEa|wdCBVmN5Di%+s{_-j)c^NRBPQepCJY_H3&!T4geh z+SQ&Wj@s^%8tO;P3oFbI0VktOy|b9?g>uj#aBQVVzVcX<4eFEdmO# z)2qB;VO8ANeg+M4f35O?{hF{Q!#1s*zjeIc;}@&5<3B4y{(^1?Gw6V5T>FCI7R{b8 zddg<^CT17uYVh>Q*dtWM3mXqKYNg+(mZ9thvxg~jxcyBuLS(@PI&c@g zlC;#+qEfcTN4A!ChJa4kHc7Fc*iCPu91MkZzcVq|=zxD-2b|wTB;dhXur5T`p-vBn zHZyCF%CB2(+peO^z(H|c}(Kz>u?i~nQl>bxb zs?_r1(>{u9iLwi9lb7X+FC_l9duT2Pf2->H@>$-#%F3?*8y}3bRn>K|PztAx1>8QcCFTwnYfS34C@YXjEKdnUtiic25$-Xe37~b=f-jv zFbTsS(ZP71K=e0#k5v5j_HU&F7_heH_l?orZo#{~o_MK%CNV`(Cg+>F3xu3IZ+0g2 zm-XdW2g#`UtmN`#Ab$1&hH>-46I3Hxymwp47E6eUu}yXI;uT|DpSid|L+-zNt8?LR zcU~38TYXYOTZKSeGfiRFW{t=RPEF z=TDZvtRunZM*R|Qq^sA+%dVa>ivB^mBN)UN3y^U(HgM&X{Yo}E`ivjksxoNYen z)Nj;4rJ)nOA@U~%H)t#xzw%6f62MpG=^XN7&;6)dsZ_ zFMyR}=$rXPBW6O6vlmz4Ry_n>cZfSrg%~xuShZ04WW*@Jxf-h`&PQ=-xPkmljUFIH zYZ456Oo)6AP7L%>#E?xjR`nXrAoJ5^gS`GHok&=YID^iyJT}0;G*G8C3Ki99x#{!V z>=!{3WZ`V83IVS;xLHCCD%p`cK4hu>qXmfhZcuE6-jV?;y+x5J_6N}@p#G#$QL(5iIOYyQ<+3Ao_#~n= zVn(jD5YSlpL)^6j5e1Q-I_?!w@HW+lEN~^*8#BrFec~NnkmI80KP=i4?}!=|lu5b_ zHE+H;Io@2}oOG_Wx4&*otiie2mxot3=M5VEz{r3E4!w20eaHf~_7*N#J;}qE z9q)1xX3njcJg9HcpE1xWImt?Z|JtaAR%&VWllnB_7EihSyCFBJF%*(25Vk36S-zhZ zh-C7QBM|>)+5yfYbyQfRJ?cg6iZg+Z)(#$6o)TDg%#E*Aj>THh-?=SsaMb2B_fO-% z<^!mO?%Y&d7j7PI1cBy1d69JeT&Fx#0V>Urk-)og{zv4+8B1w2>RW=ckT6Fc*U1dU@HHo2U!7@)mxL$zc zx$s?BHD4q{+1-?6M^(O^*|l?HTFKKKRBIKulDXKn%i5~%ZoJNCi~8BwKhVgFl&lWw z5NO$ikqu!`Nz4lfWLf8FKxDwOS*4H@tO+z_}#ityB z_56qBo3gUEk+WSH`ODQXY#1NmPN7{1eUoyV+ULbL#3oytf!4g;V=Am=Z`~P(R$CDR zuRDFo(L)PJLbTvm75R%xq@V_eFICPM=NsZHB^gh+?3Dck{5!v@3tM_^IcS`om=wYY zbl?)Te*E0z35<^7&WH@^Tm5$&F5PsxT9Z*mVMfa=WGuFgs!f}=zo@WeOYG0k9@6Nlxq%;)_ zvATh9>YS|97-gZ9!#v_UQJ~DnB!my7vFywd3HUL%nWEw8r{8>*n&UbY4Xh*XSTE~A zw$LGTxDB3oi~8dCYT-(RFXWNz^@py_Wrkbm)76~F8i=nH24{b89Y}v9S@)aTL&ZMC zRIIA!X02Sj=FmKWbK|)W_&EU={fN87nJ$A%z$3((k$hck@v_0QKcIR0n&t{0{gaH) zwIuP(*uEL5V^3$A)qF}$QHtKe9-Ss%qVNMv`4Tza>$n-L$+wCv{o5f&*rvm1uT6CY zMX6O~^tO?)k384K;ql%`1#O zbP6c#;y5dcip6|G@)KM-~*SQf;!~wa6f3ch{$jfi;(=9hbfB>rIDm z;ZoCerGPZwX3|7 zfG*vlEG@!_Ez@b$#6ITBEPGWD{`SZ?!JcC00bBsRUoVOa8tGOd#H`Paz4WyFQ=3e$ z0(B{7shk+4+F*2wRJ?V&gj@qMI1Z^S+$Umv7)spvZF@p~yj0-!+rYFrOVVI9%AJ_U*@rPbNPb86iE3| z0$$9S8Clz`UiLOyD^3X;@4PT;EXGX1<7Jn5_}pt^vBW7vPy#Ys7%!<=Vfg`wD3rHO z)durqeP`R~!3@kn`DS84U25zz0(C4SA=)O@LP^ox^CYh=$*%@Tri0mb(08uan=HXei@COh1=4XM4^+oGp*c&3#I6Qdn6kjB4{%x<7LclM65(0+b6Q+k4=oJVQMcnHc+li z=c{3}IF(~arnnEX5S_X98atS*anWH^nraIXU5fPz+g8L&(Hqa-xNFsjO`Z^43012u zh>#oji%A|oBost^j1{anU^pnoDzzNhC!xrAhCeDu4<+H^tbMUV8E|KUa+YJG>9?u>I@L5Kyn zxj8^r+#5pqXyKYcES2u{aALss9fI-1_HNh&cH1MPDk+3fu_4uj$fE2fJ-bK$=cky( zRE!uSv+p(yZd}Ag&P{ROXN{)_5CT5RY@0ypgZVbM>w6PHD@}ZXf5dZhN?lTLQM(A` zaGlH{VX$F?7f7}}>yU>!1^YsXU@&=x zsiD*3Bc?}oP{Zq|gjyKQ7#oMH?_f=dRAUeVjydD62ULRb45L5>yAKLXgON#F!GnC# zN+ubE5b|@B0@-@p4Q;c0p*=qSqgr(yVhAWFn-C5hB(no)P`4eCK|dTUePN*f!>vq6 zl^P>V$O#K?9F%4E#rM7XAzZ`Qq|X$+-|F0HSum>eqo$fDHC0PK1Jh`Sw}G|ai}Xce zOqYo1K*HB1Pa3&(GOYiK1g9+{ONXZvR!Kr}gLj``O|C0OV-u+^;rHZTbI1CkR!Sfz zLq1Nc9!?})?uPM54ebJUKwjn6@;Q8JPi{_QYj;63h{gB&+!n(EH}s5)y369lK#04` z;&64ykRsB9l@0X(HJO~yf>blMOcE0^NXZk#eAH0MP#jdnklB~=gW%VUX|?kyHdtsE z+NTc=VMoYVCouE(UmirWHDJ`3a-|#}EPqM{-S$u^ZC2?}l4PQ9OM`djo{-Tl$a?D6 z_xOOu0_*|slkRollm-jCv7;jBr>|pPs)~osMI6FR}}HJosx{1$IWLq#tN2k zG_)t``ITF#4oivW#Y%qodB)dG1nkyA2Xj9|mdQZJj7Exyo&{GW(NUgz{{=yVAsFOa zD!AAIa1_I9{q+dYIjMTN3}YuN#8R~=Ic-z`YvnMwBxSEBt8Tu$mgX$2VlI%iRFky1 zm|ILpS!VJL0o@!|sxA&_n#5=kLQ-C+zxppI;1M#H7SbTItZz@49|H)t9L9}|z#cbX z35dH|Gqo?Fi2Y~7B4GDSIgU~0&@=k$8j_S8W(4}K$U_B=LmBOs@9POi1RJF%pYovi zP@eON`7h6m?)8y&OD{?V5{W~FGE0gf-zuOidl*|jO}=guQillFbM*{+9X<=Z!W&Zs zAyjX`01ea#$4dca-i@oJ^55_QL7|#T*@6gSTWN*30_Q^Ul?NWq>O$r^S@l&h#eWvZ z)rHPwt|!KH!1ksA>KVy|ZIO0j0)m-p9CO6>JBl**Grk|C&hX~^CiZl= zV3Sd3$L(z@*N~h9+E4^XS>^TlqMteI(7yEih@~JN?uqA4cMQQ0yLfehz9XHkTlD8* zmkmKdyxq5xqY_^r#8n|~hX-i2h+>uW|0lHvO8AFv4yRU%0)G7@rf0d*`|o#o-l?wn zn0WxFy^s8}dBrn;)8+@G9(`%I6H&eX=~?OY{b$@fZ+*A?_HhCT3X|-_LC`e+hIo=y;&XRC7r=5jWHdTKZbcN-1%BJ_4 zcMA9OL(R&Omdy_}kH$>#9>ysqS?8;FG)_4xE6-xu^bw1cYkh^k`r&Xe%J7RHj&|eI z#kBcj7crPvXmLpL1VKCxB5`M>U<$#{ig?=SEuKQ^kH()?;I#L_u!)yg^{1*&3#{~- za0*pVpcn~y0Hmu6|G{r3V6KGrxySh+@Xt&NworZiLS-XDq!H+q#~2*PjO<}d$tlE?8 zUdiSu!lk9%6z}&>F)Yj#&Nk^5Dt_MSv;x~P7e?E38x_7Qe9BR%0K1bKzh>Ns2;-dE zOx3R_TQu#n7;nLbYV?9K##1xdG3J*t*{*8x?y{w0TdDLz@2o~IHV3#;UOu__-PP=| z=Y-ci@Y7?uK^nd#7zohA*e@sYP12=j5v7E!B7VGndv26|f68Iojw|-EzY5$AdWA+B zI$V~MT%@DH?IcG=S{hzyg6w7nhs+@iN-k3Ez~3AhsW$S_;bdtbr=Uqx7MN4bG>9Wm z`(<%;(4bm1kw?z27$Q=6<2WTzRG`kl59hg$A+b8Up9yLd_LclBP~gT<(Bor*H=_!C zY1q@gScYZHlv4UUW)g{NjEW64NW%(&pE-yJgzruzGsk{brp;ER}!94a*W1}0^9 zr$H2pOfnm*;g>2UomizB=h;UoTeu2;$t4bhDiw7xGq|2~_Mz5a{CwvBa^}we|J&dF{l)LU`}Ft7%L;}6$o_~kEe?%sdc{L6VBxSt*#|Lu>jZeCx%efp34E$XdP-;-=?lPO-mmC znT?BK;bF^0CF>T@%2B=ZDn*>6F%ri|8yi(zbTLuHLmCTl9Aq&N#6J@IXxu|FkHtF> z>$*7C#yA$=Kx}K{8i{FbJgZ|_6~~$w_UY#Ox3c2PzaQ$o_~Sq{4UH@;!jMS9VH*m| zFnFJQj?%i*?9a!)TzJ9wZ-;-rdfw5v_{WV#CV#yA_oBt)*3McwXyu`W4};_Kzh|x8 zw)Cf`1%*hx<_s=Ab-=Fhle>+|wGL1~q`Xy(PzqKEQN}dL?8{k&HOhKijq=0=BG2K1 zmA{A{bD&OW3R$HgWMbE+P=)NosRtKrMyqsd{rgM`B-#*R&InNbHeP|!BpF4LxQhA=F4DDru) zPEE~>x$GYSwjG=hvPGsbupPxL+7vF^FrIVka`Tp2BPs(r;|ar2qhW(#V=+S!BlQL< z=a6xk>7TDIPDL;O*vT&B?quGVbNMf0HaZ5tPL*Rqw#YQ5ZAXz2Pm4|Joh-t4g`_Ez z8Jr1(`l;%P+UrDXTxRg+FRR+^KQ~g35--lS_>Q6iH_D>aPIU#AG*GvVH>;jETA*uC zHHRFF*PDx!Y1%a1c!b4KFJuQy+j+Bbi>s(wmu}iSkFj44q=)-x!y>!sbpQA5oVx$( zWdOQA{B~!w7`jvlBDQ*&A+}MfJ*+8Zq^Jv5jVL0-m!jH4&OFjI#w^$*+#J{x%nZE= zW?S>ezi&PC`HvO2hc0fTxLYQcz0`&Qw+iaWc7M_fFb!{ZEGKxsuH_(TZMV`LV*@hHb)xA8tMJJ{VA796hO_9bc6 zsM@m1hg*y3)B)jvaTqzY(Hx~7X(Xt&Gv7S$*g*+N8Vp@^&k27;C}=A7+|M9%-F@M? z6YWY4J?Z(fuXZCrUh0ZtTUFIu>pd3dO>X?Ahw~7~2Z|vL9 euZKSU;^!CtJOA(h00030{{sO3#`#|(%L@P?7(ki; diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 07bafc43..52caf952 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -122,6 +122,51 @@ def named_fault(module, fault, invariant, tlc): print(f"COUNTEREXAMPLE model: {module}/{fault} ({invariant})") +def spec_mutation(module, spec, label, pattern, replacement, prop, tlc): + """Run MODULE with one guard of SPEC rewritten (for models without a + Fault constant), checking TypeOK and the single action property PROP, + and require PROP to be violated.""" + import re + with tempfile.TemporaryDirectory(prefix="bad-spec-") as directory: + work = Path(directory) + for source in HERE.glob("*.tla"): + shutil.copy(source, work) + text = (work / spec).read_text() + mutated, count = re.subn(pattern, replacement, text, flags=re.S) + if count != 1: + raise AssertionError(f"{spec}/{label}: mutation did not apply") + (work / spec).write_text(mutated) + lines = [] + # Keep SPECIFICATION, CONSTANTS and CONSTRAINT (which precede the + # checked invariants and properties in the cfg). + for line in (HERE / f"{module}.cfg").read_text().splitlines(): + if line.startswith(("INVARIANT", "PROPERT")): + break + lines.append(line) + lines += ["INVARIANT TypeOK", f"PROPERTY {prop}"] + (work / f"{module}.cfg").write_text("\n".join(lines) + "\n") + result = subprocess.run(shlex.split(tlc) + ["-workers", "1", "-deadlock", "-cleanup", module], + cwd=work, capture_output=True, text=True) + log = result.stdout + result.stderr + if f"property {prop} is violated" not in log: + raise AssertionError(f"{spec}/{label}: {prop} not violated\n{log}") + print(f"COUNTEREXAMPLE model: {spec}/{label} ({prop})") + + +# S1 (otp-exit-signals.md): v0.7's startup behaviours against Supervision.tla, +# which has no Fault constant: (label, pattern, replacement). +SUPERVISION_S1_CONTROLS = [ + # v0.7 retried a failed first start of a Permanent/Transient child. + ("FirstStartRetry", + r' init /\\ \(Restarts\(m, r\) \\/ \(~ready /\\ m # "Temporary"\)\)', + r' Restarts(m, r) \\/ (~ready /\\ init /\\ m # "Temporary")'), + # v0.7 decided at once on a ready child's exit during the initial start. + ("NoDeferral", + r'(Decide\(c\) ==\s*/\\ pc\[c\] = "decide")\s*/\\ ~\(open /\\ ~sready /\\ readyGen\[c\]\)', + r'\1'), +] + + # Rank 11 design models: (module, fault, invariant it must violate). def named_temporal(module, fault, prop, tlc, constants=None): """Run MODULE with Fault = FAULT, TypeOK and the single temporal property @@ -325,6 +370,23 @@ def fail_instead_of_retry(events): model_fault("MCCohortsAll", fault, args.tlc) # Needs a second generation within one round (MaxGen 3). model_fault("MCCohortsAllSupersession", "InPlaceRetry", args.tlc) + model_fault("MCCohortsAll", "FirstStartRetry", args.tlc, invariant="Safety") + for label, pattern, replacement in SUPERVISION_S1_CONTROLS: + spec_mutation("MCSupervision", "Supervision.tla", label, pattern, replacement, + "RestartFollowsPolicy", args.tlc) + deferred = next(t for t in traces if t["name"] == "ready_exit_deferred") + def decide_at_once(events): + # v0.7: no deferral; the restart decision is taken during startup, + # where the deferral was recorded (before SupStartDone). + at = next(i for i, e in enumerate(events) if e["a"] == "DeferExit") + child = events[at]["c"] + undefer = next(i for i, e in enumerate(events) if e["a"] == "Undefer") + decide = next(i for i, e in enumerate(events) + if i > undefer and e["a"] == "Decide" and e["c"] == child) + events[at] = events.pop(decide) + events.pop(undefer) + require_rejected(deferred, decide_at_once, + "ready child's exit decided during the initial start", args.tlc) model_fault("MCCoordinator", "ReleaseEarly", args.tlc) model_fault("MCCoordinator", "ForgetFailure", args.tlc) model_fault("MCCoordinator", "ForgetSignal", args.tlc, invariant="JoinAfterAllSignalled") From 43aba082c70f12ed69c6e246b0cd137d507752bb Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 02:10:25 -0300 Subject: [PATCH 086/208] feat(reply): ActorStopped carries the final reason, NoProc after death (rank 11 x 5e) - ActorError::ActorStopped(ExitReason) and new ActorError::NoProc (decision 4 of otp-exit-signals.md). - The reply registry is sealed with the final reason at the death point, inside the exit-table critical section (ActorCleanup::with_replies, link::close_links_at_death); callers read it from the slot's owner. - send/request after the death point report NoProc; sends and requests while exiting are admitted and answered at death. - Exit pending: a token dropped in a panicking turn or while exiting, and an undispatched queued request, stay open until the death point (or the caller's deadline) instead of being answered Stopped at once. - Threads-mode request slots now carry their owner registry. - DeferredReplies.tla: pendA, StoppedAtDeath, control EarlyStopped; trace control for an early Stopped discharge. - metrics: requests_noproc_total. - Tests: otp_exit_signals caller_errors (tasks and threads); existing assertions tightened to the exact reason or NoProc. --- CHANGELOG.md | 23 +- concurrency/src/actor_cleanup.rs | 25 +- concurrency/src/error.rs | 30 +- concurrency/src/link.rs | 15 + concurrency/src/metrics.rs | 4 +- concurrency/src/reply/mod.rs | 66 ++-- concurrency/src/reply/registry.rs | 175 ++++++--- concurrency/src/reply/turn.rs | 6 +- concurrency/src/response.rs | 8 +- concurrency/src/tasks/actor.rs | 65 +++- concurrency/src/threads/actor.rs | 65 ++-- concurrency/tests/actor_scenarios.rs | 6 +- concurrency/tests/cross_backend.rs | 2 +- concurrency/tests/deferred_replies.rs | 35 +- concurrency/tests/deferred_sim.rs | 2 +- concurrency/tests/delivery_simulation.rs | 2 +- concurrency/tests/failpoints.rs | 12 +- concurrency/tests/failure_windows.rs | 4 +- concurrency/tests/metrics.rs | 11 +- concurrency/tests/metrics_deferred.rs | 7 +- concurrency/tests/otp_exit_signals.rs | 340 ++++++++++++++++++ concurrency/tests/reply_trace.rs | 17 +- concurrency/tests/semantics_gaps.rs | 13 +- concurrency/tests/supervisor_policies.rs | 2 +- docs/GUIDE.md | 25 +- docs/MIGRATION.md | 31 +- docs/TESTING.md | 5 +- docs/design/deferred-replies.md | 13 + docs/design/semantics.md | 57 +-- .../capabilities/src/bin/deferred_replies.rs | 9 +- .../src/bin/streams_and_recipients.rs | 4 +- testing/src/probe.rs | 4 +- testing/src/shared.rs | 10 +- testing/tests/tasks_kit.rs | 11 +- verification/tla/DeferredReplies.tla | 78 ++-- verification/tla/README.md | 30 +- verification/tla/TraceReplies.tla | 5 +- verification/tla/negative_checks.py | 10 + 38 files changed, 981 insertions(+), 246 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ef85f6f..32664ac3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,8 +45,22 @@ `Supervision.tla` (no restart before the tree is ready, deferral actions `DeferExit`/`Undefer`, traced) and `Cohorts.tla` (round 0 once per child, `FirstStartOnce`) follow, with the v0.7 behaviours as negative controls. - Remaining: rank 11 reason-bearing `ActorStopped`/`NoProc` errors, - lifecycle-trace mapping to `Lifecycle.tla`, Loom and simulator coverage. + Caller errors (rank 11 x 5e, decision 4): `ActorError::ActorStopped` + carries the actor's final `ExitReason`, the same one `wait_exit`, `Down` + and links see (`Killed` after a kill, the signal's reason after an + untrapped exit signal, `Panic(..)` after a crash); new + `ActorError::NoProc` for a `send` or `request` after the death point (the + registry is sealed with the final reason inside the death point's + exit-table critical section); a send or request made while the actor exits + is admitted and answered at the death point. Exit errors wait for the + death point: a request whose handler panicked, a token dropped while the + owner exits and an undispatched queued request stay open ("exit pending") + and are answered with the final reason, or by the caller's deadline first. + `DeferredReplies.tla` models it (`pendA`, invariant `StoppedAtDeath`, + control `EarlyStopped`; one more reply trace control); metrics + `requests_noproc_total`. + Remaining: lifecycle-trace mapping to `Lifecycle.tla`, Loom and simulator + coverage, GUIDE and examples. - **Breaking (0.8), in progress:** deferred replies and pipe-to-self (ROADMAP rank 11), implementing @@ -77,8 +91,9 @@ and returns `()`; `#[actor]` generates it, and `#[deferred_handler]` methods receive the obligation and may `defer()` it into a storable `ReplyTo` (bounded by `Actor::MAX_OUTSTANDING`, `Overloaded` at the bound). Dropped - obligations answer `ReplyDropped` (`ActorStopped` in a panicking turn or - while exiting); at death every open token answers `ActorStopped`. Request + obligations answer `ReplyDropped` (in a panicking turn or while exiting + they are answered `ActorStopped` at death); at death every open token + answers `ActorStopped`. Request deadlines are fixed at send (`Deadline`, `request_raw_until`; `request_raw` gains the 5 s default); a per-actor deadline heap and service expire stored tokens without a polling caller. `PendingReplies` is a bounded token diff --git a/concurrency/src/actor_cleanup.rs b/concurrency/src/actor_cleanup.rs index 69e1526a..39a8986c 100644 --- a/concurrency/src/actor_cleanup.rs +++ b/concurrency/src/actor_cleanup.rs @@ -27,6 +27,8 @@ pub(crate) struct ActorCleanup { notices: Option, cancel: C, failures: Option>, + /// The reply registry, sealed with the final reason at D. + replies: Option>, } impl ActorCleanup { pub(crate) fn new( @@ -48,8 +50,19 @@ impl ActorCleanup { notices: None, cancel, failures: None, + replies: None, } } + /// Seal `replies` with the final reason at D, inside the exit-table + /// critical section: callers then get `ActorStopped(final reason)` and + /// later sends `NoProc`. + pub(crate) fn with_replies( + mut self, + replies: std::sync::Arc, + ) -> Self { + self.replies = Some(replies); + self + } pub(crate) fn with_mailbox_failures( mut self, failures: std::sync::Arc, @@ -68,7 +81,17 @@ impl ActorCleanup { self.close(reason) } fn close(&mut self, reason: ExitReason) -> ExitReason { - self.notices = Some(crate::link::close_links(self.id, &self.links, &reason)); + let replies = self.replies.as_deref(); + self.notices = Some(crate::link::close_links_at_death( + self.id, + &self.links, + &reason, + |reason| { + if let Some(replies) = replies { + replies.seal(reason); + } + }, + )); self.reason = Some(reason.clone()); reason } diff --git a/concurrency/src/error.rs b/concurrency/src/error.rs index 7852978f..ed133ee8 100644 --- a/concurrency/src/error.rs +++ b/concurrency/src/error.rs @@ -73,16 +73,25 @@ pub(crate) fn panic_message(panic: &(dyn std::any::Any + Send)) -> String { /// Errors that can occur when communicating with an actor. /// -/// Non-exhaustive: later releases add reasons (for example the exit reason of -/// a dead actor, ROADMAP rank 5e). Match with a wildcard arm. +/// Non-exhaustive: later releases may add errors. Match with a wildcard arm. #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum ActorError { - /// The actor has stopped or its mailbox channel is closed. + /// The actor died before answering, with its final exit reason (OTP: + /// `gen_server:call` exits with `{Reason, {gen_server, call, ..}}`). /// - /// Returned by `send()` and `request()` when the actor is no longer running. - #[error("Actor stopped")] - ActorStopped, + /// A request that was queued, deferred or being handled when the actor + /// died is answered at its death point with the reason that `wait_exit`, + /// `Down` and linked actors see: `Killed` after a kill, the signal's + /// reason after an untrapped exit signal, `Panic(..)` after a crash. + #[error("Actor stopped: {0}")] + ActorStopped(ExitReason), + /// The actor no longer existed when the message or request was sent: it + /// was past its death point (OTP `noproc`). A send or request made while + /// the actor is still exiting is admitted instead, and a request is then + /// answered `ActorStopped(reason)` at its death. + #[error("noproc")] + NoProc, /// A request exceeded the timeout duration (default: 5 seconds). /// /// Returned by `request()` and `request_with_timeout()` when the actor @@ -101,14 +110,14 @@ pub enum ActorError { impl From> for ActorError { fn from(_value: spawned_rt::threads::mpsc::SendError) -> Self { - Self::ActorStopped + Self::NoProc } } #[cfg(feature = "async")] impl From> for ActorError { fn from(_value: spawned_rt::tasks::mpsc::SendError) -> Self { - Self::ActorStopped + Self::NoProc } } @@ -118,8 +127,9 @@ mod tests { #[test] fn test_error_into_std_error() { - let error: &dyn std::error::Error = &ActorError::ActorStopped; - assert_eq!(error.to_string(), "Actor stopped"); + let error: &dyn std::error::Error = &ActorError::ActorStopped(ExitReason::Killed); + assert_eq!(error.to_string(), "Actor stopped: killed"); + assert_eq!(ActorError::NoProc.to_string(), "noproc"); } #[test] diff --git a/concurrency/src/link.rs b/concurrency/src/link.rs index af4f1674..cbe07cc6 100644 --- a/concurrency/src/link.rs +++ b/concurrency/src/link.rs @@ -466,11 +466,26 @@ impl LinkNotices { /// Close the exit table at D: set the final reason and take one notice per /// link. Takes only this actor's own lock. Terminal closure is idempotent. +#[cfg(test)] pub(crate) fn close_links(own_id: ActorId, own: &LinkTable, reason: &ExitReason) -> LinkNotices { + close_links_at_death(own_id, own, reason, |_| {}) +} + +/// The exit-table part of the death point D. `at_death` runs inside the same +/// critical section (it must not run user code): the reply registry seals +/// the final reason there, so a registration that finds the table closed +/// (`NoProc`) and a send that finds the registry sealed (`NoProc`) agree. +pub(crate) fn close_links_at_death( + own_id: ActorId, + own: &LinkTable, + reason: &ExitReason, + at_death: impl FnOnce(&ExitReason), +) -> LinkNotices { let mut state = own.lock().unwrap_or_else(|p| p.into_inner()); if state.exited.is_some() { return LinkNotices::default(); } + at_death(reason); state.exited = Some(reason.clone()); LinkNotices { from: Some(own_id), diff --git a/concurrency/src/metrics.rs b/concurrency/src/metrics.rs index b2b5d40f..af4a3e09 100644 --- a/concurrency/src/metrics.rs +++ b/concurrency/src/metrics.rs @@ -195,6 +195,7 @@ counters!( requests_replied_total, requests_timed_out_total, requests_actor_stopped_total, + requests_noproc_total, requests_reply_dropped_total, requests_overloaded_total, requests_cancelled_total, @@ -499,7 +500,8 @@ impl RequestObservation { match result { Ok(_) => &COUNTERS.requests_replied_total, Err(ActorError::RequestTimeout) => &COUNTERS.requests_timed_out_total, - Err(ActorError::ActorStopped) => &COUNTERS.requests_actor_stopped_total, + Err(ActorError::ActorStopped(_)) => &COUNTERS.requests_actor_stopped_total, + Err(ActorError::NoProc) => &COUNTERS.requests_noproc_total, Err(ActorError::ReplyDropped) => &COUNTERS.requests_reply_dropped_total, Err(ActorError::Overloaded) => &COUNTERS.requests_overloaded_total, } diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 256abc9b..77e47bc1 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -123,12 +123,25 @@ fn record(_delivery: Delivery) { crate::instrumentation::reply(_delivery == Delivery::Delivered); } -fn terminal_error(terminal: Terminal) -> ActorError { +/// The caller's error for a terminal slot. `Stopped` carries the owner's +/// final reason, which its registry stores at the death point before any of +/// its slots becomes `STOPPED`. +/// +/// A slot without an owner (a test double's `reply_pair`) has no actor whose +/// reason it could carry: it reports `NoProc`, as OTP's call to a process +/// that does not exist. `Abandoned` and an already taken `Replied` cannot +/// reach a caller (the receiver is gone, or consumed); they map the same way. +fn terminal_error(terminal: Terminal, slot: &Slot) -> ActorError { match terminal { Terminal::Dropped => ActorError::ReplyDropped, Terminal::Overloaded => ActorError::Overloaded, Terminal::TimedOut => ActorError::RequestTimeout, - Terminal::Stopped | Terminal::Abandoned | Terminal::Replied => ActorError::ActorStopped, + Terminal::Stopped | Terminal::Abandoned | Terminal::Replied => ActorError::ActorStopped( + slot.owner + .as_ref() + .and_then(|owner| owner.final_reason().cloned()) + .unwrap_or(crate::error::ExitReason::NoProc), + ), } } @@ -136,7 +149,7 @@ fn terminal_error(terminal: Terminal) -> ActorError { /// /// `owner` is the receiving instance's registry: an awaited raw receiver /// puts its deadline in that owner's heap instead of arming its own timer. -pub(crate) fn reply_slot( +pub(crate) fn reply_slot( deadline: Deadline, owner: Option<&Arc>, ) -> (SlotHandle, ReplyReceiver) { @@ -153,10 +166,11 @@ pub(crate) fn reply_slot( // --------------------------------------------------------------------------- /// The envelope's hold on its request's slot. Dropped undispatched (mailbox -/// disposal at death, a rejected enqueue), it answers `ActorStopped`. -pub(crate) struct SlotHandle(Option>>); +/// disposal at death, a rejected enqueue), its request is answered +/// `ActorStopped` with the owner's final reason at the death point. +pub(crate) struct SlotHandle(Option>>); -impl SlotHandle { +impl SlotHandle { /// No caller: the message was sent, not requested. pub(crate) fn detached() -> Self { Self(None) @@ -166,10 +180,15 @@ impl SlotHandle { } } -impl Drop for SlotHandle { +impl Drop for SlotHandle { fn drop(&mut self) { if let Some(slot) = self.0.take() { - slot.close(); + match slot.owner.clone() { + Some(owner) => owner.dispose(slot), + None => { + slot.close(); + } + } } } } @@ -197,8 +216,8 @@ pub enum DeferError { /// /// Answer it with [`reply`](Self::reply), hand it out of the turn with /// [`defer`](Self::defer), or drop it: a request whose `PendingReply` is -/// dropped is answered `ReplyDropped` when the turn returns, or -/// `ActorStopped` when it panics. A message reached by `send` has a detached +/// dropped is answered `ReplyDropped` when the turn returns, or, when it +/// panics, `ActorStopped` with the final reason at the actor's death point. A message reached by `send` has a detached /// `PendingReply`: `reply` drops the value and `defer` returns /// [`DeferError::NoCaller`]. pub struct PendingReply<'t, R: Send + 'static> { @@ -300,7 +319,8 @@ impl<'t, R: Send + 'static> PendingReply<'t, R> { self.slot.as_ref().is_none_or(|s| s.is_closed(&now_ticks)) } - /// Answer `ActorStopped` (test doubles standing in for an absent actor). + /// Answer `ActorStopped` (test doubles standing in for an absent actor: + /// without an owning actor the reason is `NoProc`). #[doc(hidden)] pub fn __discharge_stopped(mut self) { if let Some(slot) = self.slot.take() { @@ -334,10 +354,11 @@ impl Drop for PendingReply<'_, R> { /// A deferred reply obligation: answer it from a later turn, a job, or /// `stopped()`. Not `Clone`; `Send + 'static`. /// -/// Dropping it answers the caller `ReplyDropped` (or `ActorStopped` while the -/// owner is exiting). The owner's death answers every open token -/// `ActorStopped`, wherever it is. -#[must_use = "dropping a ReplyTo answers ReplyDropped (or ActorStopped while the owner exits)"] +/// Dropping it answers the caller `ReplyDropped`; while the owner is exiting +/// it is held instead and answered at the death point. The owner's death +/// answers every open token `ActorStopped` with the final exit reason, +/// wherever it is. +#[must_use = "dropping a ReplyTo answers ReplyDropped (or ActorStopped at death while the owner exits)"] pub struct ReplyTo { slot: Option>>, registry: Arc, @@ -465,8 +486,8 @@ impl Drop for ReplyTo { /// The caller's side of a request: `.await` it in `tasks` mode (it is /// [`IntoFuture`](std::future::IntoFuture)), or call [`recv`](Self::recv) in /// `threads` mode. Yields `Ok(reply)`, or `RequestTimeout` at the deadline -/// fixed at send, `ReplyDropped`, `Overloaded`, or `ActorStopped` when the -/// owner died. +/// fixed at send, `ReplyDropped`, `Overloaded`, or `ActorStopped(reason)` +/// with the owner's final exit reason when it died first. /// /// One pointer wide: the deadline timer lives in the awaiting future /// ([`ReplyFuture`]), not in the receiver, so receivers are cheap to store @@ -529,12 +550,16 @@ impl ReplyReceiver { .slot .take() .expect("ReplyReceiver used after completion"); - slot.recv_blocking(&now_ticks).map_err(terminal_error) + slot.recv_blocking(&now_ticks) + .map_err(|terminal| terminal_error(terminal, &slot)) } /// Non-blocking check: `None` while the request is still open. pub fn try_recv(&mut self) -> Option> { - let result = self.slot().try_take(&now_ticks)?.map_err(terminal_error); + let slot = self.slot(); + let result = slot + .try_take(&now_ticks)? + .map_err(|terminal| terminal_error(terminal, slot)); self.slot = None; Some(result) } @@ -591,8 +616,9 @@ impl std::future::Future for ReplyFuture { loop { let slot = this.rx.slot(); if let Poll::Ready(result) = slot.poll(cx.waker()) { + let result = result.map_err(|terminal| terminal_error(terminal, slot)); this.rx.slot = None; - return Poll::Ready(result.map_err(terminal_error)); + return Poll::Ready(result); } let deadline = slot.deadline(); if deadline == NEVER { diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index e2cbed2b..6d7260da 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -5,10 +5,20 @@ //! //! | phase | a dropped reply obligation | //! |---|---| -//! | `InTurn` | recorded on the turn; committed at turn end: `Dropped` if the handler returned, `Stopped` if it panicked | +//! | `InTurn` | recorded on the turn; committed at turn end: `Dropped` if the handler returned, exit pending if it panicked | //! | `Running` | `Dropped` at once | -//! | `Exiting` | `Stopped` at once | -//! | `Dead` | no-op: the close already answered it | +//! | `Exiting` | exit pending | +//! | `Dead` | answered `Stopped` at once (the final reason is known) | +//! +//! **Exit pending** (docs/design/otp-exit-signals.md, "Required changes to +//! rank 11"): the slot stays `OPEN` and listed, and the death point answers +//! it with the owner's final reason, so callers, `Down` and `wait_exit` see +//! one reason. Only the caller's own deadline (or abandonment) completes it +//! earlier; nobody holds a token that could still reply. +//! +//! The final reason is stored once, at the death point ([`Registry::seal`]), +//! before any slot of this owner becomes `STOPPED`; a caller that reads a +//! `STOPPED` slot reads it from here ([`Registry::final_reason`]). //! //! No rule uses `thread::panicking()`. The phase is one atomic so a turn costs //! two uncontended atomic stores; the turn list is locked only when a drop @@ -16,7 +26,7 @@ use super::deadline::{self, DeadlineHeap, Expirable}; use super::slot::{CreditPool, Slot, Terminal, NEVER}; -use crate::ActorId; +use crate::{ActorId, ExitReason}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, OnceLock, Weak}; @@ -25,13 +35,6 @@ pub const DEFAULT_MAX_OUTSTANDING: usize = 1024; use super::turn::{Outcome, TurnPhase, DEAD, EXITING}; -fn terminal(outcome: Outcome) -> Terminal { - match outcome { - Outcome::Dropped => Terminal::Dropped, - Outcome::Stopped => Terminal::Stopped, - } -} - /// A slot seen without its reply type. pub(crate) trait Closeable: Send + Sync { fn close(&self) -> bool; @@ -79,6 +82,9 @@ pub(crate) struct Registry { lists: Mutex, /// When exit began: the cleanup deadline is armed from here. exit_began: Mutex>, + /// The owner's final exit reason, set once at its death point. Set means + /// the mailbox refuses admission (`NoProc`). + final_reason: OnceLock, #[cfg(all(spawned_sim, feature = "async"))] sim: deadline::SimService, } @@ -101,6 +107,7 @@ impl Registry { ..Lists::default() }), exit_began: Mutex::new(None), + final_reason: OnceLock::new(), #[cfg(all(spawned_sim, feature = "async"))] sim: deadline::SimService::default(), }) @@ -225,14 +232,65 @@ impl Registry { } let slot: Arc = slot; if let Some((slot, outcome)) = self.turn.dropped(slot) { - slot.terminate(terminal(outcome)); + self.commit(vec![slot], outcome); } } - fn commit(drops: Vec>, outcome: Outcome) { + /// Answer `drops`: `Dropped` at once, or (`Stopped`) exit pending until + /// the death point. + fn commit(&self, drops: Vec>, outcome: Outcome) { for slot in drops { - slot.terminate(terminal(outcome)); + match outcome { + Outcome::Dropped => { + slot.terminate(Terminal::Dropped); + } + Outcome::Stopped => self.hold(slot), + } + } + } + + /// Exit pending: keep `slot` open and listed so the death point answers + /// it with the final reason. Past the death point, answer it now. + fn hold(&self, slot: Arc) { + let mut lists = lock(&self.lists); + // `close` sets `DEAD` before it takes the list under this lock, so a + // slot listed here is either taken by that close or closed below. + if self.is_dead() { + drop(lists); + slot.close(); + } else { + lists.tokens.push(Arc::downgrade(&slot)); + drop(lists); } + // The caller's receiver may have been the other strong reference: + // dropping the last one runs no user code (the slot holds no value). + drop(slot); + } + + /// An undispatched envelope dropped its request (mailbox disposal, a + /// rejected enqueue): answered at the death point with the final reason. + pub(crate) fn dispose(&self, slot: Arc>) { + if !slot.is_terminal() { + self.hold(slot); + } + } + + /// The death point D (called inside the owner's exit-table critical + /// section): fix the final reason callers will see and refuse admission. + /// Idempotent: the first reason stays. + pub(crate) fn seal(&self, reason: &ExitReason) { + let _ = self.final_reason.set(reason.clone()); + } + + /// The owner's final reason, once it reached its death point. + pub(crate) fn final_reason(&self) -> Option<&ExitReason> { + self.final_reason.get() + } + + /// Whether the owner is past its death point: sends and requests report + /// `NoProc`. + pub(crate) fn refuses_admission(&self) -> bool { + self.final_reason.get().is_some() } /// The run loop is about to dispatch a handler. @@ -243,11 +301,11 @@ impl Registry { /// The handler returned (`panicked == false`) or unwound. pub(crate) fn end_turn(&self, panicked: bool) { let (drops, outcome) = self.turn.end_turn(panicked); - Self::commit(drops, outcome); + self.commit(drops, outcome); } - /// Exit begins: later drops answer `Stopped`, and the current turn's - /// recorded drops commit as `Stopped`. + /// Exit begins: later drops, and the current turn's recorded drops, are + /// exit pending until the death point. pub(crate) fn exiting(&self) { if self.turn.load() == DEAD { return; @@ -258,7 +316,7 @@ impl Registry { began.get_or_insert_with(spawned_rt::time::Instant::now); first }; - Self::commit(self.turn.exiting(), Outcome::Stopped); + self.commit(self.turn.exiting(), Outcome::Stopped); if first { crate::lifecycle_trace::reply(self.owner.raw(), "Exit", 0, ""); } @@ -274,11 +332,19 @@ impl Registry { self.turn.load() >= EXITING } - /// Death: every open token listed becomes `STOPPED`, wherever it is. - /// Returns how many were closed. + /// Death: every open token listed, and every exit-pending slot, becomes + /// `STOPPED` with the final reason, wherever it is. Returns how many were + /// closed. + /// + /// The run loops [`seal`](Self::seal) the reason at the death point + /// first. A registry closed without it (its lifecycle was dropped by the + /// runtime before the death point, G1) answers `Killed`. pub(crate) fn close(&self) -> usize { + self.seal(&ExitReason::Killed); crate::lifecycle_trace::reply(self.owner.raw(), "Death", 0, ""); - Self::commit(self.turn.close(), Outcome::Stopped); + for slot in self.turn.close() { + slot.close(); + } let tokens = std::mem::take(&mut lock(&self.lists).tokens); let closed = tokens .into_iter() @@ -343,40 +409,59 @@ mod tests { slot } + fn terminal_of(slot: &Slot) -> Option { + Terminal::of(crate::reply::slot::state(slot.word())) + } + #[test] - fn turn_drops_commit_dropped_or_stopped() { + fn turn_drops_commit_dropped_or_wait_for_death() { let registry = Registry::new(ActorId::next(), 8); - for (panicked, expected) in [(false, Terminal::Dropped), (true, Terminal::Stopped)] { - let slot = deferred(®istry, NEVER); - registry.begin_turn(); - registry.dropped(slot.clone()); - assert!( - !slot.is_terminal(), - "an in-turn drop waits for the turn end" - ); - registry.end_turn(panicked); - assert_eq!( - Terminal::of(crate::reply::slot::state(slot.word())), - Some(expected) - ); - } - // Between turns a drop answers at once; while exiting, Stopped. + // A turn that returns answers its drops `Dropped`. let slot = deferred(®istry, NEVER); + registry.begin_turn(); registry.dropped(slot.clone()); - assert_eq!( - Terminal::of(crate::reply::slot::state(slot.word())), - Some(Terminal::Dropped) + assert!( + !slot.is_terminal(), + "an in-turn drop waits for the turn end" ); - registry.exiting(); + registry.end_turn(false); + assert_eq!(terminal_of(&slot), Some(Terminal::Dropped)); + // Between turns a drop answers at once. let slot = deferred(®istry, NEVER); registry.dropped(slot.clone()); - assert_eq!( - Terminal::of(crate::reply::slot::state(slot.word())), - Some(Terminal::Stopped) - ); + assert_eq!(terminal_of(&slot), Some(Terminal::Dropped)); + // A panicking turn's drops, and drops while exiting, are exit + // pending: open until the death point answers them with the final + // reason. + let unwound = deferred(®istry, NEVER); + registry.begin_turn(); + registry.dropped(unwound.clone()); + registry.end_turn(true); + registry.exiting(); + let exiting = deferred(®istry, NEVER); + registry.dropped(exiting.clone()); + assert!(!unwound.is_terminal() && !exiting.is_terminal()); + registry.seal(&ExitReason::Shutdown); + assert_eq!(registry.close(), 2); + assert_eq!(terminal_of(&unwound), Some(Terminal::Stopped)); + assert_eq!(terminal_of(&exiting), Some(Terminal::Stopped)); + assert_eq!(registry.final_reason(), Some(&ExitReason::Shutdown)); + // Past the death point a drop answers at once. + let late = Arc::new(Slot::::new(NEVER)); + registry.dropped(late.clone()); + assert_eq!(terminal_of(&late), Some(Terminal::Stopped)); assert_eq!(registry.outstanding(), 0); } + #[test] + fn unsealed_close_answers_killed() { + let registry = Registry::new(ActorId::next(), 8); + let open = deferred(®istry, NEVER); + assert_eq!(registry.close(), 1); + assert_eq!(terminal_of(&open), Some(Terminal::Stopped)); + assert_eq!(registry.final_reason(), Some(&ExitReason::Killed)); + } + #[test] fn close_stops_every_listed_open_token_once() { let registry = Registry::new(ActorId::next(), 8); diff --git a/concurrency/src/reply/turn.rs b/concurrency/src/reply/turn.rs index 5bac605e..f5256f7b 100644 --- a/concurrency/src/reply/turn.rs +++ b/concurrency/src/reply/turn.rs @@ -5,7 +5,11 @@ //! |---|---| //! | `InTurn` | recorded; committed at turn end: `Dropped` if the handler returned, `Stopped` if it panicked | //! | `Running` | `Dropped` at once | -//! | `Exiting` / `Dead` | `Stopped` at once | +//! | `Exiting` / `Dead` | `Stopped` | +//! +//! `Stopped` means "answered with the owner's final reason": the registry +//! holds such a slot open (exit pending) until the death point, or closes it +//! at once when the owner is already dead. //! //! Shared with the Loom model (`tests/registry_turn_loom.rs`), which supplies //! the synchronization types through `super::sync`. The protocol never runs diff --git a/concurrency/src/response.rs b/concurrency/src/response.rs index 01979325..2d09af4d 100644 --- a/concurrency/src/response.rs +++ b/concurrency/src/response.rs @@ -217,14 +217,18 @@ mod tests { #[test] fn ready_err_is_err() { - let r: Response = Response::ready(Err(ActorError::ActorStopped)); + let r: Response = Response::ready(Err(ActorError::ActorStopped( + crate::error::ExitReason::Normal, + ))); assert!(r.is_err()); } #[test] #[should_panic(expected = "ActorStopped")] fn ready_err_unwrap_panics() { - let r: Response = Response::ready(Err(ActorError::ActorStopped)); + let r: Response = Response::ready(Err(ActorError::ActorStopped( + crate::error::ExitReason::Normal, + ))); r.unwrap(); } diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 97981a14..b700d7ca 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -195,9 +195,14 @@ where } } +/// Admit a message. Past the death point (the registry is sealed, or the +/// mailbox is gone) it reports `NoProc`; before it, including while the +/// actor exits, it is queued, and a request is answered at the death point +/// if it is not handled. fn enqueue( sender: &mpsc::Sender + Send>>, _mailbox: &crate::instrumentation::Mailbox, + replies: &Registry, msg: M, slot: SlotHandle, ) -> Result<(), ActorError> @@ -211,6 +216,11 @@ where #[cfg(feature = "metrics")] observation: Some(_mailbox.enqueued()), }); + if replies.refuses_admission() { + #[cfg(feature = "metrics")] + envelope.reject(); + return Err(ActorError::NoProc); + } match sender.send(envelope) { Ok(()) => { #[cfg(feature = "metrics")] @@ -222,7 +232,7 @@ where { _error.0.reject(); } - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) } } } @@ -339,17 +349,15 @@ fn signal_target( let held_tx = sender.clone(); let send_exit: SendExitFn = Arc::new(move |exit: Exit| { let envelope: Box + Send> = Box::new(ExitEnvelope { exit }); - exit_tx.send(envelope).map_err(|_| ActorError::ActorStopped) + exit_tx.send(envelope).map_err(|_| ActorError::NoProc) }); let send_parent_exit: SendExitFn = Arc::new(move |exit: Exit| { let envelope: Box + Send> = Box::new(ParentExitEnvelope { exit }); - parent_tx - .send(envelope) - .map_err(|_| ActorError::ActorStopped) + parent_tx.send(envelope).map_err(|_| ActorError::NoProc) }); let send_held: SendHeldFn = Arc::new(move |exit: Exit, parent: bool| { let envelope: Box + Send> = Box::new(HeldEnvelope { exit, parent }); - held_tx.send(envelope).map_err(|_| ActorError::ActorStopped) + held_tx.send(envelope).map_err(|_| ActorError::NoProc) }); Arc::new(SignalTarget { id, @@ -487,6 +495,7 @@ impl Context { enqueue( &self.shared.sender, &self.shared.mailbox, + &self.shared.replies, msg, SlotHandle::detached(), ) @@ -515,7 +524,13 @@ impl Context { M: Message, { let (slot, rx) = reply_slot(deadline, Some(&self.shared.replies)); - enqueue(&self.shared.sender, &self.shared.mailbox, msg, slot)?; + enqueue( + &self.shared.sender, + &self.shared.mailbox, + &self.shared.replies, + msg, + slot, + )?; Ok(rx) } @@ -956,6 +971,7 @@ impl ActorRef { enqueue( &self.shared.sender, &self.shared.mailbox, + &self.shared.replies, msg, SlotHandle::detached(), ) @@ -984,7 +1000,13 @@ impl ActorRef { M: Message, { let (slot, rx) = reply_slot(deadline, Some(&self.shared.replies)); - enqueue(&self.shared.sender, &self.shared.mailbox, msg, slot)?; + enqueue( + &self.shared.sender, + &self.shared.mailbox, + &self.shared.replies, + msg, + slot, + )?; Ok(rx) } @@ -1260,7 +1282,8 @@ impl ActorRef { completion_tx, move || cleanup_token.cancel(), ) - .with_mailbox_failures(actor_ref.shared.sender.failures()); + .with_mailbox_failures(actor_ref.shared.sender.failures()) + .with_replies(actor_ref.shared.replies.clone()); let startup = crate::actor_cleanup::ActorStartup::new((actor, ctx, rx, cancellation_token), cleanup); let job_owner = actor_ref.shared.clone(); @@ -1691,7 +1714,8 @@ fn test_context( }; let noop: fn() = || {}; let cleanup = - crate::actor_cleanup::ActorCleanup::new(id, links, request, monitors, completion_tx, noop); + crate::actor_cleanup::ActorCleanup::new(id, links, request, monitors, completion_tx, noop) + .with_replies(ctx.shared.replies.clone()); (ctx, cleanup) } @@ -3705,10 +3729,8 @@ mod api_coverage { recipient.send(Ask(1)).unwrap(); actor.child_handle().stop(); actor.join().await; - assert!(matches!( - recipient.send(Ask(1)), - Err(ActorError::ActorStopped) - )); + // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). + assert!(matches!(recipient.send(Ask(1)), Err(ActorError::NoProc))); }); } @@ -3719,13 +3741,14 @@ mod api_coverage { let recipient = actor.recipient::(); actor.child_handle().stop(); actor.join().await; + // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). assert!(matches!( actor.request_with_timeout(Ask(1), WAIT).await, - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) )); assert!(matches!( request(&*recipient, Ask(1), WAIT).await, - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) )); }); } @@ -3742,8 +3765,14 @@ mod api_coverage { assert!(futures::poll!(direct.as_mut()).is_pending()); assert!(futures::poll!(erased.as_mut()).is_pending()); release.send(()).unwrap(); - assert!(matches!(direct.await, Err(ActorError::ActorStopped))); - assert!(matches!(erased.await, Err(ActorError::ActorStopped))); + assert!(matches!( + direct.await, + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); + assert!(matches!( + erased.await, + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); }); } diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 694cbafe..cc54eb7b 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -118,9 +118,14 @@ where } } +/// Admit a message. Past the death point (the registry is sealed, or the +/// mailbox is gone) it reports `NoProc`; before it, including while the +/// actor exits, it is queued, and a request is answered at the death point +/// if it is not handled. fn enqueue( sender: &mpsc::Sender>, _mailbox: &crate::instrumentation::Mailbox, + replies: &Registry, msg: M, slot: SlotHandle, ) -> Result<(), ActorError> @@ -134,6 +139,11 @@ where #[cfg(feature = "metrics")] observation: Some(_mailbox.enqueued()), }); + if replies.refuses_admission() { + #[cfg(feature = "metrics")] + envelope.reject(); + return Err(ActorError::NoProc); + } match sender.send(MailboxItem::Message(envelope)) { Ok(()) => { #[cfg(feature = "metrics")] @@ -147,7 +157,7 @@ where envelope.reject(); } } - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) } } } @@ -213,19 +223,19 @@ fn signal_target( let envelope: Box + Send> = Box::new(ExitEnvelope { exit }); exit_tx .send(MailboxItem::Message(envelope)) - .map_err(|_| ActorError::ActorStopped) + .map_err(|_| ActorError::NoProc) }); let send_parent_exit: SendExitFn = Arc::new(move |exit: Exit| { let envelope: Box + Send> = Box::new(ParentExitEnvelope { exit }); parent_tx .send(MailboxItem::Message(envelope)) - .map_err(|_| ActorError::ActorStopped) + .map_err(|_| ActorError::NoProc) }); let send_held: SendHeldFn = Arc::new(move |exit: Exit, parent: bool| { let envelope: Box + Send> = Box::new(HeldEnvelope { exit, parent }); held_tx .send(MailboxItem::Message(envelope)) - .map_err(|_| ActorError::ActorStopped) + .map_err(|_| ActorError::NoProc) }); Arc::new(SignalTarget { id, @@ -381,7 +391,13 @@ impl Context { A: Handler, M: Message, { - enqueue(&self.sender, &self.mailbox, msg, SlotHandle::detached()) + enqueue( + &self.sender, + &self.mailbox, + &self.replies, + msg, + SlotHandle::detached(), + ) } /// Send a request and get its [`ReplyReceiver`], with the default @@ -405,8 +421,8 @@ impl Context { A: Handler, M: Message, { - let (slot, rx) = reply_slot(deadline, None); - enqueue(&self.sender, &self.mailbox, msg, slot)?; + let (slot, rx) = reply_slot(deadline, Some(&self.replies)); + enqueue(&self.sender, &self.mailbox, &self.replies, msg, slot)?; Ok(rx) } @@ -767,7 +783,13 @@ impl ActorRef { A: Handler, M: Message, { - enqueue(&self.sender, &self.mailbox, msg, SlotHandle::detached()) + enqueue( + &self.sender, + &self.mailbox, + &self.replies, + msg, + SlotHandle::detached(), + ) } /// Send a request and get its [`ReplyReceiver`], with the default @@ -791,8 +813,8 @@ impl ActorRef { A: Handler, M: Message, { - let (slot, rx) = reply_slot(deadline, None); - enqueue(&self.sender, &self.mailbox, msg, slot)?; + let (slot, rx) = reply_slot(deadline, Some(&self.replies)); + enqueue(&self.sender, &self.mailbox, &self.replies, msg, slot)?; Ok(rx) } @@ -984,7 +1006,8 @@ impl ActorRef { completion_tx, move || cleanup_token.cancel(), ) - .with_mailbox_failures(actor_ref.sender.failures()); + .with_mailbox_failures(actor_ref.sender.failures()) + .with_replies(actor_ref.replies.clone()); let startup = crate::actor_cleanup::ActorStartup::new((actor, ctx, rx, cancellation_token), cleanup); let replies = actor_ref.replies.clone(); @@ -2331,7 +2354,10 @@ mod stop_wakeup_regression { release.send(()).unwrap(); first.recv().unwrap(); for reply in replies { - assert!(matches!(reply.recv(), Err(ActorError::ActorStopped))); + assert!(matches!( + reply.recv(), + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); } assert_eq!(actor.wait_exit(), ExitReason::Normal); assert_eq!(handled.load(Ordering::SeqCst), 1); @@ -2423,10 +2449,8 @@ mod api_coverage { recipient.send(Ask(1)).unwrap(); actor.child_handle().stop(); actor.join(); - assert!(matches!( - recipient.send(Ask(1)), - Err(ActorError::ActorStopped) - )); + // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). + assert!(matches!(recipient.send(Ask(1)), Err(ActorError::NoProc))); } #[test] @@ -2435,13 +2459,14 @@ mod api_coverage { let recipient = actor.recipient::(); actor.child_handle().stop(); actor.join(); + // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). assert!(matches!( actor.request_with_timeout(Ask(1), WAIT), - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) )); assert!(matches!( request(&*recipient, Ask(1), WAIT), - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) )); } @@ -2450,13 +2475,13 @@ mod api_coverage { let actor = Echo.start(); assert!(matches!( actor.request_with_timeout(Boom, WAIT), - Err(ActorError::ActorStopped) + Err(ActorError::ActorStopped(ExitReason::Panic(_))) )); let other = Echo.start(); let recipient = other.recipient::(); assert!(matches!( request(&*recipient, Boom, WAIT), - Err(ActorError::ActorStopped) + Err(ActorError::ActorStopped(ExitReason::Panic(_))) )); } diff --git a/concurrency/tests/actor_scenarios.rs b/concurrency/tests/actor_scenarios.rs index 482500bd..60fc4977 100644 --- a/concurrency/tests/actor_scenarios.rs +++ b/concurrency/tests/actor_scenarios.rs @@ -303,7 +303,9 @@ fn scenario(backend: u8, count: usize, operations: &[(u8, u8)], endings: &[u8]) assert_eq!(reply, Reply::Work(race_id)); assert!(logs[i].lock().unwrap().contains(&race_id)); } - Err(ActorError::ActorStopped) => {} + Err(ActorError::ActorStopped(_)) => {} + // Sent after the death point (v0.8, decision 4). + Err(ActorError::NoProc) => {} Err(other) => panic!("request failed to resolve during shutdown: {other:?}"), } let actual = logs[i].lock().unwrap(); @@ -313,7 +315,7 @@ fn scenario(backend: u8, count: usize, operations: &[(u8, u8)], endings: &[u8]) ); assert!(matches!( actor.request(&rt, Command::Work(u64::MAX)), - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) )); expected_down.push((monitor, reason)); } diff --git a/concurrency/tests/cross_backend.rs b/concurrency/tests/cross_backend.rs index 7ca223ca..e894a417 100644 --- a/concurrency/tests/cross_backend.rs +++ b/concurrency/tests/cross_backend.rs @@ -804,7 +804,7 @@ fn end(env: &Env, actor: &Ref, ending: Ending) -> ExitReason { Ending::Stop => assert_eq!(actor.request(Cmd::Stop).unwrap(), Reply::Unit), Ending::Panic => assert!(matches!( actor.request(Cmd::Boom), - Err(ActorError::ActorStopped) + Err(ActorError::ActorStopped(ExitReason::Panic(_))) )), Ending::Shutdown => child.shutdown(), } diff --git a/concurrency/tests/deferred_replies.rs b/concurrency/tests/deferred_replies.rs index 258f293e..0cb1a955 100644 --- a/concurrency/tests/deferred_replies.rs +++ b/concurrency/tests/deferred_replies.rs @@ -1,7 +1,9 @@ //! Deferred replies (docs/design/deferred-replies.md): every discharge path of //! a request's reply slot, in `tasks` mode (also under `spawned_sim`) and in //! `threads` mode. -use spawned_concurrency::{message::Message, ActorError, Deadline, DeferError, Delivery, ReplyTo}; +use spawned_concurrency::{ + message::Message, ActorError, Deadline, DeferError, Delivery, ExitReason, ReplyTo, +}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -208,8 +210,9 @@ mod tasks_mode { for defer_first in [false, true] { let (actor, _) = start(); let result = actor.request(Explode { defer_first }).await; + // v0.8: answered at the death point with the final reason. assert!( - matches!(result, Err(ActorError::ActorStopped)), + matches!(result, Err(ActorError::ActorStopped(ExitReason::Panic(_)))), "{result:?}" ); actor.join().await; @@ -330,8 +333,9 @@ mod tasks_mode { std::thread::sleep(Duration::from_millis(5)); } let result = caller.join().unwrap(); + // A runtime-dropped actor ends Killed (G1); its callers see it. assert!( - matches!(result, Err(ActorError::ActorStopped)), + matches!(result, Err(ActorError::ActorStopped(ExitReason::Killed))), "{result:?}" ); drop(actor); @@ -354,9 +358,18 @@ mod tasks_mode { actor.request(Stop { answer: false }).await.unwrap(); actor.request_raw_until(Park(8), Deadline::Never) }; - assert!(matches!(rx.await, Err(ActorError::ActorStopped))); - if let Ok(queued) = queued_behind_stop { - assert!(matches!(queued.await, Err(ActorError::ActorStopped))); + assert!(matches!( + rx.await, + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); + // Sent while exiting it is admitted and answered at D; sent after D + // it reports NoProc. + match queued_behind_stop { + Ok(queued) => assert!(matches!( + queued.await, + Err(ActorError::ActorStopped(ExitReason::Normal)) + )), + Err(error) => assert!(matches!(error, ActorError::NoProc), "{error:?}"), } actor.join().await; } @@ -588,9 +601,10 @@ mod threads_mode { actor.request(Stop { answer: true }).unwrap(); assert_eq!(f.recv().unwrap(), 1007); actor.join(); + // Past the death point: NoProc (v0.8, decision 4). assert!(matches!( actor.request(Outstanding), - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) )); } @@ -621,14 +635,17 @@ mod threads_mode { let (actor, _) = start(); assert!(matches!( actor.request(Explode { defer_first }), - Err(ActorError::ActorStopped) + Err(ActorError::ActorStopped(ExitReason::Panic(_))) )); actor.join(); } let (actor, _) = start(); let parked = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); actor.request(Stop { answer: false }).unwrap(); - assert!(matches!(parked.recv(), Err(ActorError::ActorStopped))); + assert!(matches!( + parked.recv(), + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); actor.join(); } } diff --git a/concurrency/tests/deferred_sim.rs b/concurrency/tests/deferred_sim.rs index 38c747d4..cac04a19 100644 --- a/concurrency/tests/deferred_sim.rs +++ b/concurrency/tests/deferred_sim.rs @@ -217,7 +217,7 @@ async fn reply_vs_expiry_vs_death_vs_drop(seed: u64) { Some(Err(e)) => { assert!(!delivered, "{e:?} after a delivery"); assert!( - matches!(e, ActorError::RequestTimeout | ActorError::ActorStopped), + matches!(e, ActorError::RequestTimeout | ActorError::ActorStopped(_)), "{e:?}" ); } diff --git a/concurrency/tests/delivery_simulation.rs b/concurrency/tests/delivery_simulation.rs index ccf30dbb..b1415e52 100644 --- a/concurrency/tests/delivery_simulation.rs +++ b/concurrency/tests/delivery_simulation.rs @@ -146,7 +146,7 @@ impl<'a> World<'a> { if let Some(reply) = pending.take() { assert!(matches!( reply.wait(self.rt, LIMIT), - Err(ActorError::ActorStopped) + Err(ActorError::ActorStopped(ExitReason::Normal)) )); } } diff --git a/concurrency/tests/failpoints.rs b/concurrency/tests/failpoints.rs index 31f138a3..1700b87b 100644 --- a/concurrency/tests/failpoints.rs +++ b/concurrency/tests/failpoints.rs @@ -219,11 +219,6 @@ async fn factory_panic_is_a_failed_start_not_a_controller_failure() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn seeded_probabilistic_factory_failures_match_generation_count() { let _scenario = FailScenario::setup(); - failpoints::configure( - "child.factory", - FailPoint::new(Action::Error).probability(0.5), - ) - .unwrap(); let system = ActorSystem::new().unwrap(); let group = system.async_group("fp-probability", 1).unwrap(); let started = Started::default(); @@ -237,6 +232,13 @@ async fn seeded_probabilistic_factory_failures_match_generation_count() { .unwrap(); let tree = tree.start().await.unwrap(); let mut actor = child.wait_ready().await.unwrap(); + // Armed after the initial start: since S1 any failed first start fails + // `start()`, so only replacements may fail (and are retried). + failpoints::configure( + "child.factory", + FailPoint::new(Action::Error).probability(0.5), + ) + .unwrap(); for _ in 0..8 { let generation = child.snapshot().generation; actor.send(Crash).unwrap(); diff --git a/concurrency/tests/failure_windows.rs b/concurrency/tests/failure_windows.rs index bd5d907a..0580af37 100644 --- a/concurrency/tests/failure_windows.rs +++ b/concurrency/tests/failure_windows.rs @@ -52,7 +52,7 @@ fn controlled_failures_reach_every_required_window_on_every_backend() { assert_eq!(actor.wait(&rt), ExitReason::Normal); assert!(matches!( queued.wait(&rt, LIMIT), - Err(ActorError::ActorStopped) + Err(ActorError::ActorStopped(ExitReason::Normal)) )); assert!(effects.lock().unwrap().is_empty()); reached.insert(REQUIRED[0]); @@ -111,7 +111,7 @@ fn controlled_failures_reach_every_required_window_on_every_backend() { control.release(); assert!(matches!( response.wait(&rt, LIMIT), - Err(ActorError::ActorStopped) + Err(ActorError::ActorStopped(ExitReason::Panic(_))) )); assert!(matches!(actor.wait(&rt), ExitReason::Panic(_))); assert_eq!(*effects.lock().unwrap(), vec![9]); diff --git a/concurrency/tests/metrics.rs b/concurrency/tests/metrics.rs index e38f5fca..032cfef8 100644 --- a/concurrency/tests/metrics.rs +++ b/concurrency/tests/metrics.rs @@ -147,7 +147,7 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { spawned_concurrency::Deadline::after(Duration::from_secs(2)) )) .await, - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) )); let (release, gate) = oneshot::channel(); @@ -158,7 +158,10 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { assert!(futures::poll!(pending.as_mut()).is_pending()); actor.context().stop(); release.send(()).unwrap(); - assert!(matches!(pending.await, Err(ActorError::ActorStopped))); + assert!(matches!( + pending.await, + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); assert!(raw.await.is_err()); actor.join().await; assert_eq!(actor.mailbox_depth(), 0); @@ -166,7 +169,7 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { let actor = Probe { gate: None }.start(); assert!(matches!( actor.request(Crash).await, - Err(ActorError::ActorStopped) + Err(ActorError::ActorStopped(ExitReason::Panic(_))) )); assert!(matches!(actor.wait_exit().await, ExitReason::Panic(_))); // Managed restart/shutdown must conserve the same aggregate accounting. @@ -314,6 +317,8 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { s.requests_replied_total + s.requests_timed_out_total + s.requests_actor_stopped_total + // Requests to an actor past its death point (rank 11 x 5e). + + s.requests_noproc_total + s.requests_cancelled_total ); assert!(s.actors_exited_shutdown_total >= 1); diff --git a/concurrency/tests/metrics_deferred.rs b/concurrency/tests/metrics_deferred.rs index c0ca92d0..163faa4b 100644 --- a/concurrency/tests/metrics_deferred.rs +++ b/concurrency/tests/metrics_deferred.rs @@ -10,7 +10,7 @@ use spawned_concurrency::{ Actor, ActorRef, ActorStart, Context, Deadline, Handler, JobCtx, JobOptions, JobOutcome, PendingReply, ReplyTo, }, - ActorError, + ActorError, ExitReason, }; use spawned_rt::tasks::{oneshot, sleep}; use std::time::Duration; @@ -212,7 +212,10 @@ async fn deferred_and_job_families_count_each_outcome_once() { let late = actor.request_raw(Hold).unwrap(); sleep(Duration::from_millis(20)).await; actor.context().stop(); - assert!(matches!(late.await, Err(ActorError::ActorStopped))); + assert!(matches!( + late.await, + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); assert!(matches!(stopped.await, Ok(1))); actor.join().await; diff --git a/concurrency/tests/otp_exit_signals.rs b/concurrency/tests/otp_exit_signals.rs index f030c24a..0a0597b2 100644 --- a/concurrency/tests/otp_exit_signals.rs +++ b/concurrency/tests/otp_exit_signals.rs @@ -53,6 +53,10 @@ mod tasks_mode { tokio::sync::oneshot::Receiver<()>, )>, pub panic_in_stopped: bool, + /// Deferred tokens stored by `Hold`. + pub held: Vec>, + /// stopped() drops the stored tokens before waiting on `in_stopped`. + pub drop_held_in_stopped: bool, } impl Node { pub fn new(seen: &Seen, trap: bool) -> Self { @@ -61,6 +65,8 @@ mod tasks_mode { trap, in_stopped: None, panic_in_stopped: false, + held: Vec::new(), + drop_held_in_stopped: false, } } } @@ -72,6 +78,9 @@ mod tasks_mode { } async fn stopped(&mut self, _: &Context) { self.seen.stopped.store(true, Ordering::SeqCst); + if self.drop_held_in_stopped { + self.held.clear(); + } if let Some((entered, release)) = self.in_stopped.take() { let _ = entered.send(()); let _ = release.await; @@ -121,6 +130,52 @@ mod tasks_mode { reply.reply(()); } } + /// Defer the request and store its token. + struct Hold; + impl Message for Hold { + type Result = u8; + } + impl Handler for Node { + async fn handle( + &mut self, + _: Hold, + reply: spawned_concurrency::reply::PendingReply<'_, u8>, + _: &Context, + ) { + self.held.push(reply.defer().unwrap()); + } + } + /// Panic in a request handler (the request's obligation unwinds). + struct Boom; + impl Message for Boom { + type Result = u8; + } + impl Handler for Node { + async fn handle( + &mut self, + _: Boom, + _reply: spawned_concurrency::reply::PendingReply<'_, u8>, + _: &Context, + ) { + panic!("boom in a request handler"); + } + } + /// Block the actor in a handler until released. + struct Block(tokio::sync::oneshot::Receiver<()>); + impl Message for Block { + type Result = (); + } + impl Handler for Node { + async fn handle( + &mut self, + msg: Block, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { + let _ = msg.0.await; + reply.reply(()); + } + } /// Run `f` inside one of `actor`'s handlers. pub async fn on(actor: &ActorRef, f: impl FnOnce(&Context) + Send + 'static) { actor.request(Do(Box::new(f))).await.unwrap(); @@ -559,6 +614,161 @@ mod tasks_mode { assert!(seen.stopped()); } + // Rank 11 x 5e ("Required changes to rank 11"): callers, `Down` and + // `wait_exit` see one reason. Each test pins one rule of the design. + mod caller_errors { + use super::*; + use spawned_concurrency::tasks::ActorRef; + use spawned_concurrency::ActorError; + + async fn answered( + rx: spawned_concurrency::reply::ReplyReceiver, + ) -> Result + where + R: Send + 'static, + { + tokio::time::timeout(LIMIT, rx) + .await + .expect("the request was never answered") + } + + /// Block `actor` in a handler; returns the release. + async fn block(actor: &ActorRef) -> tokio::sync::oneshot::Sender<()> { + let (release, gate) = tokio::sync::oneshot::channel(); + actor.send(Block(gate)).unwrap(); + release + } + + fn stopped_with(result: Result, reason: &ExitReason) { + match result { + Err(ActorError::ActorStopped(got)) => assert_eq!(&got, reason), + other => panic!("expected ActorStopped({reason:?}), got {other:?}"), + } + } + + // A request queued, or a token stored, at D is answered with the + // final reason: Killed after a kill (stopped() skipped). + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn queued_and_stored_requests_get_killed_after_a_kill() { + let seen = Seen::default(); + let actor = Node::new(&seen, false).start(); + let stored = actor.request_raw(Hold).unwrap(); + settle(&actor).await; + let release = block(&actor).await; + let queued = actor.request_raw(Ping).unwrap(); + let child = actor.child_handle(); + child.kill(); + release.send(()).unwrap(); + assert_eq!(ended(&child).await, ExitReason::Killed); + stopped_with(answered(queued).await, &ExitReason::Killed); + stopped_with(answered(stored).await, &ExitReason::Killed); + } + + // An untrapped exit signal: queued requests get the signal's reason. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn queued_request_gets_the_untrapped_signals_reason() { + let seen = Seen::default(); + let actor = Node::new(&seen, false).start(); + settle(&actor).await; + let release = block(&actor).await; + let queued = actor.request_raw(Ping).unwrap(); + let child = actor.child_handle(); + let reason = ExitReason::Panic("signalled".into()); + child.exit(reason.clone()); + release.send(()).unwrap(); + assert_eq!(ended(&child).await, reason); + stopped_with(answered(queued).await, &reason); + } + + // A request whose handler panicked is not answered at once: it waits + // for D (stopped() runs first) and gets the final reason. A kill + // during stopped() makes that reason Killed for the caller too. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn panicked_request_is_answered_at_death_with_the_final_reason() { + for kill in [false, true] { + let seen = Seen::default(); + let (entered_tx, entered) = tokio::sync::oneshot::channel(); + let (release, release_rx) = tokio::sync::oneshot::channel(); + let mut node = Node::new(&seen, false); + node.in_stopped = Some((entered_tx, release_rx)); + let actor = node.start(); + settle(&actor).await; + let mut crashed = actor.request_raw(Boom).unwrap(); + entered.await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + crashed.try_recv().is_none(), + "the panicked request was answered before D" + ); + let child = actor.child_handle(); + if kill { + child.kill(); + } + release.send(()).unwrap(); + let reason = ended(&child).await; + if kill { + assert_eq!(reason, ExitReason::Killed); + } else { + assert!(matches!(reason, ExitReason::Panic(_)), "{reason:?}"); + } + stopped_with(answered(crashed).await, &reason); + } + } + + // A token dropped while the owner exits is held until D, then + // answered with the final reason. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn token_dropped_while_exiting_is_held_until_death() { + let seen = Seen::default(); + let (entered_tx, entered) = tokio::sync::oneshot::channel(); + let (release, release_rx) = tokio::sync::oneshot::channel(); + let mut node = Node::new(&seen, false); + node.in_stopped = Some((entered_tx, release_rx)); + node.drop_held_in_stopped = true; + let actor = node.start(); + let mut stored = actor.request_raw(Hold).unwrap(); + settle(&actor).await; + let child = actor.child_handle(); + child.stop_with_reason(ExitReason::Shutdown); + entered.await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + stored.try_recv().is_none(), + "a token dropped in stopped() was answered before D" + ); + release.send(()).unwrap(); + assert_eq!(ended(&child).await, ExitReason::Shutdown); + stopped_with(answered(stored).await, &ExitReason::Shutdown); + } + + // A request sent while the actor exits (a slow stopped() of a + // trapping actor) is admitted and answered at D; after D, send and + // request report NoProc. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn admitted_while_exiting_noproc_after_death() { + let seen = Seen::default(); + let (entered_tx, entered) = tokio::sync::oneshot::channel(); + let (release, release_rx) = tokio::sync::oneshot::channel(); + let mut node = Node::new(&seen, true); + node.in_stopped = Some((entered_tx, release_rx)); + let actor = node.start(); + settle(&actor).await; + let child = actor.child_handle(); + child.stop_with_reason(ExitReason::Shutdown); + entered.await.unwrap(); + let admitted = actor.request_raw(Ping).expect("admitted while exiting"); + actor.send(Ping).expect("a send while exiting is admitted"); + release.send(()).unwrap(); + assert_eq!(ended(&child).await, ExitReason::Shutdown); + stopped_with(answered(admitted).await, &ExitReason::Shutdown); + assert!(matches!(actor.send(Ping), Err(ActorError::NoProc))); + assert!(matches!(actor.request(Ping).await, Err(ActorError::NoProc))); + assert!(matches!(actor.request_raw(Ping), Err(ActorError::NoProc))); + let recipient = actor.recipient::(); + assert!(matches!(recipient.send(Ping), Err(ActorError::NoProc))); + } + } + mod startup { use super::*; use spawned_concurrency::supervision::{ @@ -845,4 +1055,134 @@ mod threads_mode { peer.child_handle().stop(); assert_eq!(ended(&peer.child_handle()), ExitReason::Normal); } + + // Rank 11 x 5e in threads mode: the final reason at D, held exit errors, + // admission while exiting and NoProc after D. + mod caller_errors { + use super::*; + use spawned_concurrency::reply::{PendingReply, ReplyReceiver, ReplyTo}; + use spawned_concurrency::ActorError; + use std::sync::mpsc; + + #[derive(Default)] + struct Server { + held: Vec>, + /// stopped() drops the held tokens, reports entry, then waits. + in_stopped: Option<(mpsc::Sender<()>, mpsc::Receiver<()>)>, + } + impl Actor for Server { + fn stopped(&mut self, _: &Context) { + self.held.clear(); + if let Some((entered, release)) = self.in_stopped.take() { + entered.send(()).unwrap(); + let _ = release.recv(); + } + } + } + impl Handler for Server { + fn handle(&mut self, _: Ping, reply: PendingReply<'_, u8>, _: &Context) { + reply.reply(7); + } + } + struct Hold; + impl Message for Hold { + type Result = u8; + } + impl Handler for Server { + fn handle(&mut self, _: Hold, reply: PendingReply<'_, u8>, _: &Context) { + self.held.push(reply.defer().unwrap()); + } + } + struct Boom; + impl Message for Boom { + type Result = u8; + } + impl Handler for Server { + fn handle(&mut self, _: Boom, _reply: PendingReply<'_, u8>, _: &Context) { + panic!("boom in a request handler"); + } + } + struct Block(mpsc::Receiver<()>); + impl Message for Block { + type Result = (); + } + impl Handler for Server { + fn handle(&mut self, msg: Block, reply: PendingReply<'_, ()>, _: &Context) { + let _ = msg.0.recv(); + reply.reply(()); + } + } + + fn stopped_with(rx: ReplyReceiver, reason: &ExitReason) { + match rx.recv() { + Err(ActorError::ActorStopped(got)) => assert_eq!(&got, reason), + other => panic!("expected ActorStopped({reason:?}), got {other:?}"), + } + } + + fn gated() -> (Server, mpsc::Receiver<()>, mpsc::Sender<()>) { + let (entered_tx, entered) = mpsc::channel(); + let (release, release_rx) = mpsc::channel(); + let server = Server { + in_stopped: Some((entered_tx, release_rx)), + ..Server::default() + }; + (server, entered, release) + } + + #[test] + fn queued_and_stored_requests_get_killed_after_a_kill() { + let actor = Server::default().start(); + let stored = actor.request_raw(Hold).unwrap(); + assert_eq!(actor.request(Ping).unwrap(), 7); + let (release, gate) = mpsc::channel(); + actor.send(Block(gate)).unwrap(); + let queued = actor.request_raw(Ping).unwrap(); + actor.child_handle().kill(); + release.send(()).unwrap(); + assert_eq!(ended(&actor.child_handle()), ExitReason::Killed); + stopped_with(queued, &ExitReason::Killed); + stopped_with(stored, &ExitReason::Killed); + } + + #[test] + fn panicked_request_and_dropped_token_wait_for_death() { + let (server, entered, release) = gated(); + let actor = server.start(); + let mut stored = actor.request_raw(Hold).unwrap(); + let mut crashed = actor.request_raw(Boom).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + std::thread::sleep(Duration::from_millis(50)); + assert!( + crashed.try_recv().is_none(), + "panicked request answered before D" + ); + assert!( + stored.try_recv().is_none(), + "dropped token answered before D" + ); + release.send(()).unwrap(); + let reason = ended(&actor.child_handle()); + assert!(matches!(reason, ExitReason::Panic(_)), "{reason:?}"); + stopped_with(crashed, &reason); + stopped_with(stored, &reason); + } + + #[test] + fn admitted_while_exiting_noproc_after_death() { + let (server, entered, release) = gated(); + let actor = server.start(); + assert_eq!(actor.request(Ping).unwrap(), 7); + actor.child_handle().stop_with_reason(ExitReason::Shutdown); + entered.recv_timeout(LIMIT).unwrap(); + let admitted = actor.request_raw(Ping).expect("admitted while exiting"); + actor.send(Ping).expect("a send while exiting is admitted"); + release.send(()).unwrap(); + assert_eq!(ended(&actor.child_handle()), ExitReason::Shutdown); + stopped_with(admitted, &ExitReason::Shutdown); + assert!(matches!(actor.send(Ping), Err(ActorError::NoProc))); + assert!(matches!(actor.request(Ping), Err(ActorError::NoProc))); + assert!(matches!(actor.request_raw(Ping), Err(ActorError::NoProc))); + } + } } diff --git a/concurrency/tests/reply_trace.rs b/concurrency/tests/reply_trace.rs index 0c2940e4..9f869705 100644 --- a/concurrency/tests/reply_trace.rs +++ b/concurrency/tests/reply_trace.rs @@ -9,7 +9,7 @@ use spawned_concurrency::{ lifecycle_trace, message::Message, tasks::{Actor, ActorRef, ActorStart, Context, Deadline, Handler, PendingReply, ReplyTo}, - ActorError, + ActorError, ExitReason, }; use spawned_rt::tasks::sleep; use std::{path::PathBuf, time::Duration}; @@ -66,7 +66,8 @@ impl Handler for Owner { async fn handle(&mut self, _: Ignore, _reply: PendingReply<'_, u32>, _: &Context) {} } -/// Deferred, then the turn panics: the token answers `ActorStopped`. +/// Deferred, then the turn panics: the token is held until the death point, +/// which answers it `ActorStopped(Panic(..))`. struct Boom; impl Message for Boom { type Result = u32; @@ -248,7 +249,10 @@ async fn death_closes_stored_tokens() { let owner = id(&actor); actor.context().stop(); let got = held.await; - assert!(matches!(got, Err(ActorError::ActorStopped)), "{got:?}"); + assert!( + matches!(got, Err(ActorError::ActorStopped(ExitReason::Normal))), + "{got:?}" + ); actor.join().await; let events = write("death", owner); assert!( @@ -260,8 +264,13 @@ async fn death_closes_stored_tokens() { #[tokio::test] async fn panic_in_turn_answers_stopped() { let actor = start(); + // The token unwound by the panic is exit pending: answered at the death + // point with the final reason, not by a discharge of its own. let got = actor.request(Boom).await; - assert!(matches!(got, Err(ActorError::ActorStopped)), "{got:?}"); + assert!( + matches!(got, Err(ActorError::ActorStopped(ExitReason::Panic(_)))), + "{got:?}" + ); actor.join().await; sleep(Duration::from_millis(10)).await; write("panic", id(&actor)); diff --git a/concurrency/tests/semantics_gaps.rs b/concurrency/tests/semantics_gaps.rs index c596ef75..5ba4c0e9 100644 --- a/concurrency/tests/semantics_gaps.rs +++ b/concurrency/tests/semantics_gaps.rs @@ -202,7 +202,7 @@ mod native { let actor = Panicker.start(); let result = actor.request_with_timeout(Boom, LIMIT); assert!( - matches!(result, Err(ActorError::ActorStopped)), + matches!(result, Err(ActorError::ActorStopped(ExitReason::Panic(_)))), "{result:?}" ); assert!(matches!(actor.wait_exit(), ExitReason::Panic(_))); @@ -211,7 +211,7 @@ mod native { let recipient: Recipient = erased.recipient(); let result = request(&*recipient, Boom, LIMIT); assert!( - matches!(result, Err(ActorError::ActorStopped)), + matches!(result, Err(ActorError::ActorStopped(ExitReason::Panic(_)))), "{result:?}" ); @@ -219,7 +219,10 @@ mod native { let receiver = raw .request_raw_until(Boom, spawned_concurrency::Deadline::after(LIMIT)) .unwrap(); - assert!(matches!(receiver.recv(), Err(ActorError::ActorStopped))); + assert!(matches!( + receiver.recv(), + Err(ActorError::ActorStopped(ExitReason::Panic(_))) + )); } } @@ -560,7 +563,7 @@ mod tasks { for actor in actors { let result = actor.request_with_timeout(Boom, LIMIT).await; assert!( - matches!(result, Err(ActorError::ActorStopped)), + matches!(result, Err(ActorError::ActorStopped(ExitReason::Panic(_)))), "{result:?}" ); let reason = timeout(LIMIT, actor.wait_exit()).await.unwrap(); @@ -571,7 +574,7 @@ mod tasks { let recipient: Recipient = erased.recipient(); let result = request(&*recipient, Boom, LIMIT).await; assert!( - matches!(result, Err(ActorError::ActorStopped)), + matches!(result, Err(ActorError::ActorStopped(ExitReason::Panic(_)))), "{result:?}" ); diff --git a/concurrency/tests/supervisor_policies.rs b/concurrency/tests/supervisor_policies.rs index 96ff440e..29caf049 100644 --- a/concurrency/tests/supervisor_policies.rs +++ b/concurrency/tests/supervisor_policies.rs @@ -267,7 +267,7 @@ async fn scenario( Action::Panic => { assert!(matches!( actor.request_with_timeout(Crash, LIMIT).await, - Err(ActorError::ActorStopped) + Err(ActorError::ActorStopped(ExitReason::Panic(_))) )); let reason = actor.wait_exit().await; assert!(matches!(reason, ExitReason::Panic(_))); diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 7a09b32a..a4642af9 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -69,7 +69,7 @@ All three phases are wrapped in `catch_unwind`: - **Panic in a handler** — the current message is lost. The actor stops and `stopped()` is called. - **Panic in `stopped()`** — the panic is logged and `join()` still returns, but the exit reason becomes `Panic("panic in stopped(): ...")` unless it was already abnormal. -In all cases, `join()` will eventually return and subsequent `send()`/`request()` calls return `Err(ActorStopped)`. +In all cases, `join()` will eventually return and subsequent `send()`/`request()` calls return `Err(NoProc)`. ### Stopping an Actor @@ -77,7 +77,7 @@ Call `ctx.stop()` from inside a handler or lifecycle hook. This cancels the acto From outside, use the actor's `ChildHandle` (`actor_ref.child_handle()`): `stop()` exits `Normal`, `shutdown()` exits `Shutdown`, and `stop_with_reason(reason)` picks the reason. All are cooperative: the running handler finishes and `stopped()` runs. They ignore `trap_exit`. `ActorRef` itself has no stop method; an explicit shutdown message in your protocol also works. -Stopping discards messages still queued: once the stop is observed no further message is handled, and queued requests fail with `ActorStopped`. +Stopping discards messages still queued: once the stop is observed no further message is handled, and queued requests fail with `ActorStopped(reason)`, the actor's final exit reason. --- @@ -556,9 +556,10 @@ Rules (the [design](design/deferred-replies.md) has the full contract): - A `ReplyTo` is `Send + 'static` and answers exactly once: `token.reply(v)` returns a `Delivery` (`Delivered`, `CallerGone`, `Expired`, `OwnerExited`); `token.overloaded()` answers `Err(Overloaded)`. -- Dropping an unanswered token answers `ReplyDropped` (or `ActorStopped` if the - turn panicked or the actor is exiting). `stopped()` may still answer stored - tokens; at the actor's death every open token answers `ActorStopped`. +- Dropping an unanswered token answers `ReplyDropped`. If the turn panicked or + the actor is exiting, it is held instead and answered at the actor's death + with its final reason. `stopped()` may still answer stored tokens; at the + actor's death every open token answers `ActorStopped(reason)`. - The deadline is the caller's, fixed at send. `token.is_closed()` tells you the caller can no longer be answered; the actor's deadline heap expires stored tokens on time and returns their credits. @@ -629,14 +630,18 @@ impl Message for GetCount { All actor communication can fail with `ActorError`: ```rust +#[non_exhaustive] pub enum ActorError { - ActorStopped, // The actor has stopped or its mailbox is closed - RequestTimeout, // A request exceeded the timeout (default 5s) + ActorStopped(ExitReason), // The actor died before replying, with this final reason + NoProc, // The actor was already gone when sent to (OTP noproc) + RequestTimeout, // A request exceeded its deadline (default 5s) + ReplyDropped, // The handler dropped the request unanswered + Overloaded, // The actor rejected the request at a bound } ``` -- `send()` returns `Err(ActorStopped)` once the actor's mailbox has closed. While the actor is stopping (after a stop request, including during `stopped()`), `send()` still returns `Ok(())` and the message is dropped without being handled: `Ok` means accepted, not handled. -- `request()` returns `Err(ActorStopped)` if the actor stops before replying, or `Err(RequestTimeout)` if the reply doesn't arrive in time. +- `send()` returns `Err(NoProc)` once the actor is past its death point. While the actor is stopping (after a stop request, including during `stopped()`), `send()` still returns `Ok(())` and the message is dropped without being handled: `Ok` means accepted, not handled. +- `request()` returns `Err(ActorStopped(reason))` if the actor dies before replying, with the reason `wait_exit()` and monitors report (`Killed`, `Panic(..)`, the exit signal's reason, ...), `Err(NoProc)` if it was already gone, or `Err(RequestTimeout)` if the reply doesn't arrive in time. A request whose handler panicked is answered when the actor dies (after `stopped()`), not at once. - `request_with_timeout()` lets you specify a custom timeout. --- @@ -1311,7 +1316,7 @@ out.expect_no_message(Duration::from_millis(20)).await; reply types still fail loudly. `allow_send::()` acknowledges one type only. - A message without a rule makes the mock panic with the type name and the - rule to add. The caller sees `ActorStopped`; `received()` lists it as + rule to add. The caller sees `ActorStopped(Panic(..))`; `received()` lists it as `Outcome::Unexpected` and `assert_no_unexpected()` fails the test. - `expect::(timeout)` returns the oldest not-yet-expected `M` (other types stay queued); `expect_no_message(window)`, `count::()`, `take_all::()`, diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 419b1ccf..09d82b41 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -98,6 +98,29 @@ table is the full list). What to edit: `start()` and `init_network` panics on `.expect("Network starts")`. Fix in ethrex: make the bootnode ping best-effort (log and continue) so the factory does not fail on it, or retry `start()` at application level. +- **Caller errors carry the reason (with rank 11, decision 4).** + `ActorError::ActorStopped` is now `ActorStopped(ExitReason)`: the actor's + final exit reason, the one `wait_exit`, `Down` and linked actors see + (`Killed` after `kill()`, the signal's reason after an untrapped exit + signal, `Panic(..)` after a crash, `Normal`/`Shutdown` after a stop). A + `send` or `request` to an actor past its death point returns the new + `ActorError::NoProc` instead; one sent while the actor is still exiting + (for example during its `stopped()`) is admitted and answered at the death + point. Patterns change from `Err(ActorError::ActorStopped)` to + `Err(ActorError::ActorStopped(_))` (or a specific reason); code that + treated `ActorStopped` as "the actor is gone" for sends should also handle + `NoProc`. A request whose handler panicked, or a deferred token dropped + while the actor exits, is no longer answered at once: it is answered at the + death point (after `stopped()`) with the final reason, or by the caller's + own deadline first. With `metrics`, rejected requests count in the new + `requests_noproc_total`. **ethrex**: `peer_handler.rs:1251` matches + `ActorError` exhaustively on purpose, so it no longer compiles: add arms + for `ActorStopped(_)` and `NoProc`; two tests that construct + `ActorError::ActorStopped` must pass a reason (for example + `ActorStopped(ExitReason::Normal)`). **ethlambda**: one test mock + constructs `ActorStopped`; pass a reason the same way. A `spawned-test` + `Probe` closed with `close()` now returns `NoProc`, and a probe without a + reply answers `ActorStopped(NoProc)`. ### Deferred replies (rank 11, targets 0.8, in progress) @@ -123,7 +146,8 @@ impl Handler for Counter { ``` A handler that returns without answering a request now gives the caller -`ReplyDropped` (a panic still gives `ActorStopped`). Sends reach the same +`ReplyDropped` (a panic still gives `ActorStopped`, with the final reason at +the actor's death point). Sends reach the same `handle` with a detached `PendingReply`. To call a manual handler directly in a test, use `spawned_test::reply_pair()`. @@ -146,8 +170,9 @@ that builds a `Snapshot` with a struct literal must add them. **Lifecycle.** At death the registry closes (open deferred tokens answer `ActorStopped`), then the mailbox is disposed, then actor state is dropped: -a state destructor that sends to its own actor now fails with `ActorStopped` -instead of enqueueing into a mailbox about to be dropped. +a state destructor that sends to its own actor now fails (with `NoProc` +once the exit-signals change above is in) instead of enqueueing into a +mailbox about to be dropped. ## 0.7.3 diff --git a/docs/TESTING.md b/docs/TESTING.md index 1ea47649..10b6f2bd 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -278,10 +278,11 @@ handler panic. The fixed regression case exercises every termination mode. - A request fence observes every earlier message from the same sender in order, exactly once. Dropping a reply receiver does not cancel queued work. -- A request racing shutdown either returns its matching reply or ActorStopped. +- A request racing shutdown returns its matching reply, `ActorStopped` or + `NoProc`. Accepted sends alone do not guarantee execution across termination. - Termination resolves within a five-second test watchdog. Post-exit requests - return ActorStopped. + return `NoProc`. - Each uncancelled monitor observes the matching exit reason, including when registration races termination. Unknown or duplicate observed notifications fail the test. Clean shutdown and injected handler panic remain distinct. diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index 7dc9c290..c38391ec 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -345,6 +345,13 @@ No rule uses `thread::panicking()`. A drop that lands on the turn list from another task while a turn runs is committed at that turn's end. The delay is at most one turn, and the outcome only depends on whether that turn panicked. +**Refined by rank 5e** ([otp-exit-signals.md](otp-exit-signals.md#required-changes-to-rank-11-221-design-239-implementation), +implemented in #240): a `Stopped` outcome is no longer answered at once. The +slot becomes *exit pending*: it stays open in the registry and the death +point answers it `ActorStopped` with the owner's final exit reason (or the +caller's deadline comes first). An undispatched queued request is held the +same way, and a send or request after the death point reports `NoProc`. + ## 5. Owner lifecycle Each actor instance has a reply registry (its open deferred tokens and the @@ -425,6 +432,12 @@ the actor's job set; it is not detached. non-exhaustive enum; - a request to an actor that has already died reports `NoProc`. +Both are implemented in #240 as `ActorError::ActorStopped(ExitReason)` and +`ActorError::NoProc`, with the death boundary of +[otp-exit-signals.md](otp-exit-signals.md#phases-of-an-exit): the registry is +sealed with the final reason at the death point, and exit errors wait for +it. + The slot's terminal kinds are defined so that rank 5e only adds the reason. ## 6. Delegation diff --git a/docs/design/semantics.md b/docs/design/semantics.md index fd460cd6..98a04c4a 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -64,10 +64,12 @@ duplicated or retried. While the actor keeps running it is handled exactly once. Checked by: [`actor_scenarios`][actor_scenarios], [`payload_stress`][payload_stress] (FIFO delivery or complete destruction), [`request_histories`][request_histories]. -**D3. Accepted is not handled.** `send` returns `Err(ActorStopped)` only once -the mailbox is closed; the rejected message is dropped. `Ok(())` means -accepted: between a stop request and mailbox closure (including during -`stopped()`), sends still succeed and the message is then discarded. +**D3. Accepted is not handled.** `send` returns `Err(NoProc)` only once the +actor is past its death point (the mailbox refuses admission); the rejected +message is dropped. `Ok(())` means accepted: between a stop request and the +death point (including during `stopped()`), sends still succeed and the +message is then discarded (a request is answered `ActorStopped` with the +final reason). Checked by: [`actor_scenarios`][actor_scenarios] ("accepted sends alone do not guarantee execution across termination"), [`mailbox_admission_loom`][mailbox_admission_loom] (threads send gate: publish before drain or fail). @@ -75,8 +77,8 @@ guarantee execution across termination"), [`mailbox_admission_loom`][mailbox_adm **D4. Stop discards the queue.** The run loop checks for stop before every receive. Once a stop is observed no further message is dispatched, including messages already queued. Queued messages are dropped one by one and every -queued reply sender is released (callers get `ActorStopped`) before the exit is -published. +queued request is answered (callers get `ActorStopped` with the final reason) +before the exit is published. Checked by: `src/tasks/actor.rs::stop_while_idle_does_not_dispatch_a_queued_request_on_resume`, `src/threads/actor.rs::repeated_stop_during_a_handler_discards_queued_requests`, `src/threads/actor.rs::stop_while_receiving_does_not_dispatch_a_later_request`, @@ -93,15 +95,17 @@ planned (rank 17). ([deferred replies](deferred-replies.md) §1) that is discharged exactly once. `request`, `request_with_timeout`, protocol calls and `spawned::tasks::request` end in exactly one of `Ok(reply)`, -`Err(ActorStopped)` (not accepted, the handler panicked, the request was still -queued at the owner's death, or a deferred token was open at death), +`Err(ActorStopped(reason))` (the handler panicked, the request was still +queued at the owner's death, or a deferred token was open at death; always +the owner's final exit reason, the one `wait_exit` and `Down` report), +`Err(NoProc)` (sent after the owner's death point), `Err(RequestTimeout)` (the deadline fixed at send passed), `Err(ReplyDropped)` (the handler dropped the reply obligation unanswered) or `Err(Overloaded)` (`defer()` at `MAX_OUTSTANDING`, or the handler answered -`overloaded()`). A request racing shutdown gets its reply or `ActorStopped`; -a slow teardown can still produce a timeout first. `ActorStopped` from a -deferred token is reported only at the owner's actual death, never while -`stopped()` still runs. +`overloaded()`). A request racing shutdown gets its reply, `ActorStopped` or +`NoProc`; a slow teardown can still produce a timeout first. `ActorStopped` is +reported only at the owner's actual death, never while `stopped()` still +runs (otp-exit-signals.md, "Exit errors wait for D"). Checked by: [`actor_scenarios`][actor_scenarios], `src/tasks/actor.rs::request_timeout`, [`timeout_outcomes`][ex_timeout_outcomes] example. @@ -130,22 +134,25 @@ Checked by: [`cancellation_safety::request_family_is_cancel_safe`][cancellation_ [`request_histories`][request_histories] (timed-out operations are "pending"). **R5. Handler panic.** A panic while handling a request drops its reply -obligation in the panicking turn: the drop is committed at the turn's end as -`ActorStopped` (a turn that returns without answering gives `ReplyDropped`), -and the actor exits with `Panic`. +obligation in the panicking turn: at the turn's end it becomes exit pending +(a turn that returns without answering gives `ReplyDropped`), the actor exits +and the request is answered at its death point with the final reason, +`ActorStopped(Panic(..))` (or `ActorStopped(Killed)` if a kill arrived +before death), or by the caller's deadline first. Checked by: [`deferred_replies`][deferred_replies] `::{tasks_mode::panicking_turn_answers_actor_stopped, threads_mode::panic_and_death_answer_actor_stopped}`, [`semantics_gaps`][semantics_gaps]`::{native,tasks}::panicking_request_handler_gives_the_caller_actor_stopped` -(exactly `ActorStopped`, every backend, a managed group, `Recipient` and raw -receivers), [`panic_safety`][panic_safety]. +(exactly `ActorStopped(Panic(_))`, every backend, a managed group, +`Recipient` and raw receivers), [`otp_exit_signals`][otp_exit_signals]`::*::caller_errors` +(answered only at death), [`panic_safety`][panic_safety]. **R6. Deferred replies.** A handler may answer later: `PendingReply::defer()` takes one of the instance's `MAX_OUTSTANDING` credits (or answers `Overloaded` at once) and returns a storable `ReplyTo`. The token is answered from a later turn, a job or `stopped()`; dropping it answers `ReplyDropped` -between turns, is committed at the turn's end inside a turn, and answers -`ActorStopped` while the owner exits. At the owner's death every open token -becomes `ActorStopped`, wherever it is held. The owner's deadline heap expires +between turns, is committed at the turn's end inside a turn, and while the +owner exits is held until its death point. At the owner's death every open +token becomes `ActorStopped` with the final reason, wherever it is held. The owner's deadline heap expires deferred tokens (returning their credits) even when no caller polls. Callers and protocols cannot tell a deferred reply from an immediate one. Checked by: `reply_slot_loom` (every concurrent pair of the slot word), @@ -222,8 +229,8 @@ destructors), [`panic_safety`][panic_safety], [`native_mailbox_drop`][native_mai **L7. Exit publication is last.** The exit reason is published after `stopped()`, after queued messages and reply senders are released, after links have been signalled and outgoing monitors cancelled. `join()`, `wait_exit()` -and `ChildHandle` waits return only then; afterwards `send`/`request` return -`ActorStopped`. +and `ChildHandle` waits return only then; `send`/`request` return `NoProc` +from the death point on, which comes before publication. Checked by: [`shutdown_publication`][shutdown_publication], `src/link_regression_tests.rs::never_polled_completion_waits_for_actor_payload_destruction`, [`actor_scenarios`][actor_scenarios] (post-exit requests). @@ -347,7 +354,8 @@ Checked by: `Supervision!AtMostOneGeneration`, `Cohorts!AtMostOneGeneration`, slot's generation. `SupervisedChild::current()` returns only the latest live, ready generation; `wait_for_generation(n)` waits for a ready generation newer than `n`. Old `ActorRef`s stay bound to the old instance and fail with -`ActorStopped`; queued messages are discarded, never replayed. +`NoProc` (queued requests got `ActorStopped` with its final reason); queued +messages are discarded, never replayed. Checked by: `Supervision!PublishedIsLive`, `Cohorts!PublishedIsLive`; [`supervisor_policies`][supervisor_policies] (request fences), [`supervision_recovery`][ex_supervision_recovery] example. @@ -596,7 +604,8 @@ ignored by non-trapping peers (X3), and the Permanent/Transient/Temporary table - v0.8, rank 5g: one exit reason for runtime-dropped actors in both modes (changes L4, X6). - Design for 5e, the 5f startup rule and 5g, accepted (not implemented): [otp-exit-signals.md](otp-exit-signals.md). Reason-bearing `ActorStopped` - and `NoProc` for requests wait for rank 11 (#239). + and `NoProc` for sends and requests are implemented on the exit-signals + branch (#240, on top of rank 11's #239). - Rank 11, deferred typed replies, forwarding and pipe-to-self: a reply may leave the handler's turn; adds exactly-one completion on panic and timeout. diff --git a/examples/capabilities/src/bin/deferred_replies.rs b/examples/capabilities/src/bin/deferred_replies.rs index 1c172360..7ec59a1c 100644 --- a/examples/capabilities/src/bin/deferred_replies.rs +++ b/examples/capabilities/src/bin/deferred_replies.rs @@ -402,9 +402,14 @@ async fn main() { let two = parker.request_raw_until(Park, Deadline::Never).unwrap(); parker.request(StopAnswering(1)).await.unwrap(); assert_eq!(one.await.unwrap(), "answered in stopped()"); - assert!(matches!(two.await, Err(ActorError::ActorStopped))); + assert!(matches!( + two.await, + Err(ActorError::ActorStopped( + spawned_concurrency::ExitReason::Normal + )) + )); parker.join().await; - println!("5. stopped() answered a stored token; death answered the rest ActorStopped"); + println!("5. stopped() answered a stored token; death answered the rest ActorStopped(normal)"); for actor in [ router.child_handle(), diff --git a/examples/capabilities/src/bin/streams_and_recipients.rs b/examples/capabilities/src/bin/streams_and_recipients.rs index 281bb3e1..8e79b280 100644 --- a/examples/capabilities/src/bin/streams_and_recipients.rs +++ b/examples/capabilities/src/bin/streams_and_recipients.rs @@ -69,7 +69,7 @@ mod asynchronous { .expect("async listener panicked"); assert!(matches!( recipients[0].send(Item(1)), - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) )); recipients[1].send(Item(1)).unwrap(); assert_eq!(second.request(Total).await.unwrap(), 4); @@ -162,7 +162,7 @@ mod synchronous { .expect("native counter did not stop"); assert!(matches!( recipients[0].send(Item(1)), - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) )); recipients[1].send(Item(1)).unwrap(); assert_eq!(second.request(Total).unwrap(), 4); diff --git a/testing/src/probe.rs b/testing/src/probe.rs index 3a05ffe1..cb7bbf21 100644 --- a/testing/src/probe.rs +++ b/testing/src/probe.rs @@ -48,7 +48,7 @@ impl ProbeCore { fn open(&self) -> Result, ActorError> { let closed = lock(&self.closed); if *closed { - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) } else { Ok(closed) } @@ -65,7 +65,7 @@ impl ProbeCore { } /// `Ok(None)` means the request was logged but there is no reply: the - /// caller drops the reply sender and observes `ActorStopped`. + /// caller observes `ActorStopped(NoProc)` (no actor stands behind it). /// /// A request is logged `InFlight` on admission. The returned [`Pending`] /// is finished by the caller *after* it has put the reply in the reply diff --git a/testing/src/shared.rs b/testing/src/shared.rs index 0f548a3d..0b684db9 100644 --- a/testing/src/shared.rs +++ b/testing/src/shared.rs @@ -75,7 +75,8 @@ macro_rules! mock_builder { } /// Fault: panic with `reason` on each `M`. The mock exits with - /// `ExitReason::Panic` and pending callers see `ActorStopped`. + /// `ExitReason::Panic` and pending callers see `ActorStopped` with that + /// `Panic` reason. pub fn crash_on(mut self, reason: impl Into) -> Self { self.rules.crash_on::(reason.into()); self @@ -181,7 +182,8 @@ macro_rules! probe_common { // Logged `Handled` only once the reply is in the slot. record.finish(); } - // An unexpected request is discharged as `ActorStopped`. + // An unexpected request is discharged as `ActorStopped` + // (`NoProc`: no actor stands behind a probe). None => reply.__discharge_stopped(), } Ok(rx) @@ -234,8 +236,8 @@ macro_rules! probe_common { self.core.inbox.take_all::() } - /// Fault: behave like a stopped actor; later sends and requests - /// return `ActorStopped`. + /// Fault: behave like an actor past its death point; later sends + /// and requests return `NoProc`. pub fn close(&self) { self.core.close(); } diff --git a/testing/tests/tasks_kit.rs b/testing/tests/tasks_kit.rs index 4703971e..ab04fc5f 100644 --- a/testing/tests/tasks_kit.rs +++ b/testing/tests/tasks_kit.rs @@ -112,7 +112,10 @@ async fn protocol_methods_run_directly_on_the_mock() { async fn unconfigured_request_fails_loudly() { let mock = Mock::new().allow_sends().start(); let error = mock.reserve("x".into()).await.unwrap_err(); - assert!(matches!(error, ActorError::ActorStopped)); + assert!(matches!( + error, + ActorError::ActorStopped(ExitReason::Panic(_)) + )); let ExitReason::Panic(reason) = join_within(&mock, WAIT).await else { panic!("mock should panic on an unconfigured request"); }; @@ -207,7 +210,11 @@ async fn replying_probe_answers_and_closed_probe_rejects() { let error = tasks::request(&*silent.recipient(), Quote(1), WAIT) .await .unwrap_err(); - assert!(matches!(error, ActorError::ActorStopped)); + // No actor stands behind a probe: its reason is NoProc. + assert!(matches!( + error, + ActorError::ActorStopped(ExitReason::NoProc) + )); assert_eq!(silent.received()[0].outcome, Outcome::Unexpected); probe.close(); diff --git a/verification/tla/DeferredReplies.tla b/verification/tla/DeferredReplies.tla index 7207f24c..33e493b1 100644 --- a/verification/tla/DeferredReplies.tla +++ b/verification/tla/DeferredReplies.tla @@ -16,7 +16,7 @@ (* Overloaded at BoundA, or nothing if the slot is already closed) into a *) (* local token, then stores it, pipes it into a linked job, or drops it; *) (* drops in a turn commit when the turn ends: Dropped if it returns, *) -(* Stopped if it panics. A linked job shares its token's credit while the *) +(* exit pending if it panics. A linked job shares its token's credit while the *) (* token is open and no other job of that request runs; otherwise it *) (* needs a fresh credit. A job's notification hands the token back to a *) (* turn (completion or job deadline; after a deadline the work keeps *) @@ -26,9 +26,16 @@ (* *) (* Exit: A enters "exiting" (stop between turns, or a panic in a turn). *) (* While exiting (stopped(), OTP terminate) stored tokens can still be *) -(* answered, drops answer Stopped, nothing is dispatched and jobs are *) -(* cancelled. Death closes the registry (every token A holds, wherever it *) -(* is) and discards the mailbox. If stopped() (or a destructor) is still *) +(* answered, nothing is dispatched and jobs are cancelled. A token *) +(* dropped while exiting, or unwound by the panicking turn, is "exit *) +(* pending" (pendA): the registry keeps it open and death answers it with *) +(* the final reason (docs/design/otp-exit-signals.md, "Required changes *) +(* to rank 11"); only the caller's deadline or abandonment comes first. *) +(* Death closes the registry (every token A holds, wherever it is, and *) +(* every exit-pending slot) and discards the mailbox. The model has no *) +(* reasons: the reason is fixed at death (Lifecycle.tla), so StoppedAtDeath*) +(* (Stopped only in the death step) is what makes callers see it. *) +(* If stopped() (or a destructor) is still *) (* running at the cleanup deadline, A becomes "stuck": nothing is closed *) (* early (ActorStopped means the actor is gone); stopped() may still *) (* reply; callers are bounded by their own deadlines; death may never come.*) @@ -49,19 +56,23 @@ (* LeakDownstream (DownstreamOwned) B's request outlives its job *) (* EarlyClose (NoEarlyClose) the cleanup deadline closes the *) (* registry before death *) +(* EarlyStopped (StoppedAtDeath) exit errors answered Stopped at *) +(* once (rank 11 before 5e) *) (***************************************************************************) EXTENDS Naturals, FiniteSets CONSTANTS N, BoundA, BoundB, MaxTime, MaxWait, Fault ASSUME Fault \in {"None", "NoAutoReply", "NoAtomicExpiry", "LateSuccess", "LoopDeadline", "Lengthen", "Unbounded", "NoDestAdmission", "ExitStateOnly", "UnwindDropped", "ShareRunning", - "AttachClosed", "LeakDownstream", "EarlyClose"} + "AttachClosed", "LeakDownstream", "EarlyClose", + "EarlyStopped"} Req == 1..N \* qA queued; turnA PendingReply in a turn; localA ReplyTo in a turn; \* dropA dropped in a turn (commits at turn end); stateA stored; -\* doneA captured by a linked job's done; gone no longer held. -Holders == {"none", "qA", "turnA", "localA", "dropA", "stateA", "doneA", "gone"} +\* doneA captured by a linked job's done; pendA exit pending (dropped while +\* exiting or unwound; the registry answers it at death); gone no longer held. +Holders == {"none", "qA", "turnA", "localA", "dropA", "stateA", "doneA", "pendA", "gone"} Terminal == {"ok", "timeout", "overloaded", "stopped", "dropped", "abandoned"} VARIABLES now, caller, sent, dl, slot, holder, discharges, okAt, @@ -87,6 +98,8 @@ Open(r) == slot[r] = "open" TokenHeld(r) == Open(r) /\ holder[r] \in {"localA", "dropA", "stateA", "doneA"} \* Credits: a request's token and its first job share one credit; every \* further running job, and every job whose token has closed, needs its own. +\* An exit-pending token keeps its credit until death in the implementation; +\* nothing takes a credit once exit has begun, so it is not counted here. CreditOf(r) == IF jobs[r] > 0 THEN jobs[r] ELSE IF TokenHeld(r) THEN 1 ELSE 0 Credits == Cardinality({<> \in Req \X {1, 2} : k <= CreditOf(r)}) HeldB == {r \in Req : down[r] = "stateB"} @@ -111,10 +124,14 @@ TryReply(r) == /\ UNCHANGED <> ELSE Keep -\* A drop outside a turn: Dropped while running, Stopped while exiting. +\* A drop outside a turn: Dropped while running; exit pending while exiting +\* (answered at death with the final reason). DropNow(r) == - IF ~Open(r) \/ Fault = "NoAutoReply" THEN Keep - ELSE Discharge(r, IF phase = "run" THEN "dropped" ELSE "stopped") + IF ~Open(r) \/ Fault = "NoAutoReply" THEN holder' = [holder EXCEPT ![r] = "gone"] /\ Keep + ELSE IF phase = "run" THEN holder' = [holder EXCEPT ![r] = "gone"] /\ Discharge(r, "dropped") + ELSE IF Fault = "EarlyStopped" + THEN holder' = [holder EXCEPT ![r] = "gone"] /\ Discharge(r, "stopped") + ELSE holder' = [holder EXCEPT ![r] = "pendA"] /\ Keep U(v) == UNCHANGED v @@ -201,18 +218,21 @@ EndTurn == /\ turnA' = 0 /\ U(<>) -\* Panic in a turn: the turn's tokens are dropped while unwinding and -\* answer Stopped; exit begins. +\* Panic in a turn: the turn's tokens are dropped while unwinding and are +\* exit pending (answered at death with the final reason); exit begins. Panic == /\ phase = "run" /\ turnA # 0 /\ LET r == turnA IN /\ panicked' = panicked \cup (IF holder[r] \in {"turnA", "localA", "dropA"} /\ Open(r) THEN {r} ELSE {}) - /\ IF holder[r] \in {"turnA", "localA", "dropA"} - THEN holder' = [holder EXCEPT ![r] = "gone"] - /\ IF Open(r) THEN Discharge(r, IF Fault = "UnwindDropped" THEN "dropped" - ELSE "stopped") - ELSE Keep + /\ IF holder[r] \in {"turnA", "localA", "dropA"} /\ Open(r) + THEN (CASE Fault = "UnwindDropped" -> + holder' = [holder EXCEPT ![r] = "gone"] /\ Discharge(r, "dropped") + [] Fault = "EarlyStopped" -> + holder' = [holder EXCEPT ![r] = "gone"] /\ Discharge(r, "stopped") + [] OTHER -> holder' = [holder EXCEPT ![r] = "pendA"] /\ Keep) + ELSE IF holder[r] \in {"turnA", "localA", "dropA"} + THEN holder' = [holder EXCEPT ![r] = "gone"] /\ Keep ELSE U(holder) /\ Keep /\ phase' = "exiting" /\ turnA' = 0 /\ U(<>) @@ -225,14 +245,15 @@ LaterReply(r) == LaterDrop(r) == /\ phase \in {"run", "exiting", "stuck"} /\ turnA = 0 /\ holder[r] = "stateA" - /\ holder' = [holder EXCEPT ![r] = "gone"] /\ DropNow(r) + /\ DropNow(r) /\ U(<>) ExitBegin == /\ phase = "run" /\ turnA = 0 /\ phase' = "exiting" /\ U(<>) -\* Death: the registry closes every token A holds; the mailbox is discarded. +\* Death: the registry closes every token A holds and every exit-pending +\* slot, with the final reason; the mailbox is discarded. \* Cleanup deadline while stopped() still runs: the instance is reported \* stuck; nothing is closed early, since ActorStopped means "gone". CleanupDeadline == @@ -246,8 +267,8 @@ CleanupDeadline == Death == /\ phase \in {"exiting", "stuck"} /\ phase' = "dead" /\ DischargeSet(OpenOf({r \in Req : holder[r] \in - (IF Fault = "ExitStateOnly" THEN {"qA", "stateA"} - ELSE {"qA", "stateA", "doneA"})}), "stopped") + (IF Fault = "ExitStateOnly" THEN {"qA", "stateA", "pendA"} + ELSE {"qA", "stateA", "doneA", "pendA"})}), "stopped") /\ U(<>) ------------------------------------------------------------------------------ @@ -279,7 +300,7 @@ JobEnds(r) == JobCancel(r) == /\ holder[r] = "doneA" /\ (~Open(r) \/ phase = "exiting") - /\ holder' = [holder EXCEPT ![r] = "gone"] /\ DropNow(r) + /\ DropNow(r) /\ U(<>) JobCallB(r) == @@ -350,13 +371,19 @@ AtMostOneDischarge == \A r \in Req : discharges[r] <= 1 NoOrphan == \A r \in Req : Open(r) => /\ phase # "dead" - /\ holder[r] \in {"qA", "turnA", "localA", "dropA", "stateA", "doneA"} + /\ holder[r] \in {"qA", "turnA", "localA", "dropA", "stateA", "doneA", "pendA"} NoSuccessAfterDeadline == \A r \in Req : slot[r] = "ok" => okAt[r] < dl[r] TimelyExpiry == \A r \in Req : Open(r) => now <= dl[r] DeadlineWithinCaller == \A r \in Req : slot[r] # "none" => dl[r] <= sent[r] BoundedA == Credits <= BoundA BoundedB == Cardinality(HeldB) <= BoundB -PanicStopped == \A r \in panicked : slot[r] = "stopped" +\* A request open in a panicking turn is never answered Dropped (or a value): +\* it waits for death (Stopped), unless its deadline or caller comes first. +PanicStopped == \A r \in panicked : slot[r] \in {"open", "stopped", "timeout", "abandoned"} +\* Exit errors wait for death: a Stopped discharge happens only in the death +\* step, where the reason is final (otp-exit-signals.md; the same property +\* Lifecycle.tla checks as StoppedAtDeath). +StoppedAtDeath == \A r \in Req : slot[r] = "stopped" => phase = "dead" \* The registry closes only at actual death: no token is answered Stopped \* by a close while the instance (its stopped()) is still alive. NoEarlyClose == earlyClosed = {} @@ -368,7 +395,8 @@ CallerConsistent == Safety == /\ TypeOK /\ AtMostOneDischarge /\ NoOrphan /\ NoSuccessAfterDeadline /\ TimelyExpiry /\ DeadlineWithinCaller /\ BoundedA /\ BoundedB - /\ PanicStopped /\ NoEarlyClose /\ DownstreamOwned /\ CallerConsistent + /\ PanicStopped /\ StoppedAtDeath /\ NoEarlyClose /\ DownstreamOwned + /\ CallerConsistent EveryAcceptedDischarged == \A r \in Req : (slot[r] = "open") ~> (slot[r] \in Terminal) CallerNeverHangs == \A r \in Req : (caller[r] = "waiting") ~> (caller[r] # "waiting") diff --git a/verification/tla/README.md b/verification/tla/README.md index ddd15193..b8f10805 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -13,7 +13,7 @@ this fits with Loom, Shuttle, Miri, ThreadSanitizer and mutation testing. | `Cohorts.tla`, `MCCohorts*.tla/.cfg` | OneForAll/RestForOne range selection, reverse cleanup, whole-cohort join barrier, ordered resume, stale notifications, superseding rounds and retired slots. Three-child mixed-mode and two-round configurations. | | `Coordinator.tla`, `MCCoordinator.tla/.cfg` | System shutdown coordinator: ordered stop of admitted roots, panic fallback that signals all remaining roots, records failure and retains runtime ownership until joins; ordered-shutdown invariants, cooperative cleanup liveness and four negative mutations. | | `NestedSupervision.tla`, `MCNested.tla/.cfg` | Dynamic subtree ownership and shutdown delivery; parent scope completion waits for descendant completion. Bounded three-node/two-generation model with fair cleanup. | -| `DeferredReplies.tla`, `MCDeferredReplies*.tla/.cfg` | Rank 11 design model (not implemented), part 1: reply slots, send-time deadlines with shortening, deferral credits shared by a token and its first linked job, the exiting phase with unwind-local drops, a stuck phase (cleanup deadline passed while `stopped()` runs: nothing closed early, death later or never), death, and downstream request cleanup. Thirteen named negative controls. | +| `DeferredReplies.tla`, `MCDeferredReplies*.tla/.cfg` | Rank 11 design model (not implemented), part 1: reply slots, send-time deadlines with shortening, deferral credits shared by a token and its first linked job, the exiting phase with exit-pending drops (unwound or dropped while exiting, answered at death with the final reason), a stuck phase (cleanup deadline passed while `stopped()` runs: nothing closed early, death later or never), death, and downstream request cleanup. Fourteen named negative controls. | | `DeferredJobs.tla`, `MCDeferredJobs*.tla/.cfg` | Rank 11 design model, part 2: pipe jobs with separate execution, cancellation and notification states; keyed replacement including notified jobs; optional job deadlines; a FIFO mailbox with emit credits and the terminal message last; `ActorId` fencing. `MCDeferredJobsEmits` is the fast configuration with emits. Nine named negative controls. | | `Lifecycle.tla`, `MCLifecycle{,Live,Fence,Deep,LiveDeep}.tla/.cfg` | Base actor lifecycle as proposed for v0.8 ([design](../../docs/design/otp-exit-signals.md)): links, monitors, `trap_exit`, exit signals (`Killed`, ignored `Normal`, `NoProc`, parent exits), administrative stops, requests around death, `Exit`/`Down` ordering. Seventeen negative-control faults (17 safety checks, 2 liveness checks): the v0.7 exit-signal behaviours (direct stops ignoring `trap_exit`, kills ending `Kill`, unfenced `unlink`/`demonitor`, real reasons for late registrations, `stopped()` after kills and untrapped signals, late signals ignored during exit), the PR #182/#185 bugs, early exit errors, kill and panic precedence, runtime-drop reasons, early notification, a split death point and a parent exit reaching `exit_received`. | | `TraceCohorts.tla`, `TraceNested.tla` | Policy and ownership projections of real cohort/nested Rust executions. | @@ -498,8 +498,11 @@ turns, a running-linked-job count per request, and downstream actor B. open and no other job of that request runs. - **Job notifications** hand the token back to a turn; after a job deadline the work keeps running. -- **Exit.** While exiting, stored tokens can still be answered and drops - answer `Stopped`. At the cleanup deadline, a still-running `stopped()` +- **Exit.** While exiting, stored tokens can still be answered. A token + dropped while exiting, or unwound by a panicking turn, is *exit pending* + (`pendA`): it stays open and death answers it `Stopped` with the final + reason (rank 5e, [otp-exit-signals.md](../../docs/design/otp-exit-signals.md#required-changes-to-rank-11-221-design-239-implementation)), + unless the caller's deadline or abandonment comes first. At the cleanup deadline, a still-running `stopped()` makes A stuck: nothing is closed early, `stopped()` may still reply, and callers are bounded by their own deadlines. Only death closes the registry and discards the mailbox. @@ -511,7 +514,10 @@ Invariants: - `AtMostOneDischarge`, `NoOrphan`, `NoSuccessAfterDeadline`, `TimelyExpiry`, `DeadlineWithinCaller`; - `BoundedA` (combined credits) and `BoundedB`; -- `PanicStopped`, `NoEarlyClose`, `DownstreamOwned`, `CallerConsistent`. +- `PanicStopped`, `StoppedAtDeath` (a `Stopped` discharge happens only in + the death step, where the reason is final; the model has no reasons, so + this is how it shows callers get the final one), `NoEarlyClose`, + `DownstreamOwned`, `CallerConsistent`. Liveness is `EveryAcceptedDischarged` and `CallerNeverHangs`, with fairness only on the clock and the deadline service. @@ -555,6 +561,7 @@ exactly the property it targets: | `DeferredReplies` | `AttachClosed` | `BoundedA` | | `DeferredReplies` | `LeakDownstream` | `DownstreamOwned` | | `DeferredReplies` | `EarlyClose` (closes the registry at the cleanup deadline) | `NoEarlyClose` | +| `DeferredReplies` | `EarlyStopped` (rank 11 before 5e: a token dropped while exiting or unwound answers `Stopped` at once) | `StoppedAtDeath` | | `DeferredJobs` | `NoInstanceCheck` | `NoLateResult` | | `DeferredJobs` | `NoCancelFence` | `NoStaleAfterCancel` | | `DeferredJobs` | `DoubleTerminal` | `AtMostOneTerminal` | @@ -565,12 +572,13 @@ exactly the property it targets: | `DeferredJobs` (`MCDeferredJobsEmits`) | `EmitAfterTerminal` | `TerminalLast` | | `DeferredJobs` (`MCDeferredJobsEmits`) | `NoEmitCredit` | `EmitCreditsBounded` | -Measured 2026-11-10 (TLC 2.19, fede): +Measured 2026-11-10 (TLC 2.19, fede; the two `MCDeferredReplies` rows on +2026-11-11, after the exit-pending change): | Configuration | Distinct states | Generated | Depth | Time | |---|---|---|---|---| -| `MCDeferredReplies` (fast: 2 requests, `BoundA` 2, `BoundB` 1, clock 2; safety and liveness) | 2,932,280 | 16,385,809 | 33 | 56 s, 10 workers | -| `MCDeferredRepliesDeep` (clock 3; safety) | 8,172,070 | 45,538,582 | 34 | 21 s, 10 workers | +| `MCDeferredReplies` (fast: 2 requests, `BoundA` 2, `BoundB` 1, clock 2; safety and liveness) | 4,035,345 | 21,287,403 | 33 | 2 min 19 s, 4 workers | +| `MCDeferredRepliesDeep` (clock 3; safety) | 11,016,668 | 58,096,711 | 34 | 55 s, 4 workers | | `MCDeferredJobs` (fast: 3 jobs, 2 keyed, 2 permits, 2 instances, no emits; safety and liveness) | 229,070 | 487,283 | 17 | 5 s, 8 workers | | `MCDeferredJobsEmits` (fast: 2 keyed jobs, 1 instance, 2 emits each, 1 credit; safety and liveness) | 20,422 | 37,220 | 18 | 1 s, 8 workers | | `MCDeferredJobsDeep` (3 jobs, 2 instances, 1 emit each; safety) | 5,876,658 | 12,792,039 | 23 | 25 s, 12 workers | @@ -615,13 +623,15 @@ together with the CAS in `Slot::cas`; a reply counts at `WRITING`), `Exit` death and mailbox disposal perform are that one model step and are not logged). `TraceReplies.tla` takes the turn-internal steps that leave no trace silently (a `Defer` that takes a credit, `Store`, `DropInTurn`, a turn end -that answers nothing, the clock); a `Discharge` must be the model action that +that answers nothing, a drop while exiting that makes the slot exit pending, +the clock); a `Discharge` must be the model action that discharges exactly that request with that kind. Linked jobs and the downstream actor are not traced (their notifications and job ends are not logged). `concurrency/tests/reply_trace.rs` records nine scenarios (reply in the turn, stored then answered or dropped from a later non-request turn, an unused obligation, `Overloaded` at the bound, deadline expiry by the service, the caller giving up, death with a stored token, a panicking turn); -`negative_checks.py` requires four edited traces to be rejected (`Overloaded` +`negative_checks.py` requires five edited traces to be rejected (`Overloaded` below the bound, a reply after death, a second discharge, a drop before the -turn). +turn, and a `Stopped` discharge of the panicking turn's token before its +`Exit`, the rank 11 behaviour before 5e). diff --git a/verification/tla/TraceReplies.tla b/verification/tla/TraceReplies.tla index 9ae1cd1c..898327b4 100644 --- a/verification/tla/TraceReplies.tla +++ b/verification/tla/TraceReplies.tla @@ -13,7 +13,8 @@ (* *) (* Turn-internal steps leave no trace and are taken silently: Defer that *) (* takes a credit (or finds the slot closed), Store, DropInTurn, a turn *) -(* end that answers nothing, and the clock. A Discharge is the model *) +(* end that answers nothing, a drop while exiting (the slot becomes exit *) +(* pending and is answered by Death), and the clock. A Discharge is the model *) (* action that discharges exactly that request with that kind. The model's *) (* turns are request turns: later replies run between them (from *) (* non-request turns, or stopped()). *) @@ -51,7 +52,7 @@ Step(e) == \* Steps that leave no trace: they answer no request. Silent == /\ \/ Tick \/ EndTurn - \/ \E r \in Req : Defer(r) \/ Store(r) \/ DropInTurn(r) + \/ \E r \in Req : Defer(r) \/ Store(r) \/ DropInTurn(r) \/ LaterDrop(r) /\ UNCHANGED slot TraceNext == diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 52caf952..16293aa7 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -239,6 +239,7 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCDeferredReplies", "AttachClosed", "BoundedA"), ("MCDeferredReplies", "LeakDownstream", "DownstreamOwned"), ("MCDeferredReplies", "EarlyClose", "NoEarlyClose"), + ("MCDeferredReplies", "EarlyStopped", "StoppedAtDeath"), ("MCDeferredJobs", "NoInstanceCheck", "NoLateResult"), ("MCDeferredJobs", "NoCancelFence", "NoStaleAfterCancel"), ("MCDeferredJobs", "DoubleTerminal", "AtMostOneTerminal"), @@ -280,6 +281,15 @@ def dropped_before_dispatch(events): require_rejected(by_name["replies_unused"], dropped_before_dispatch, "unused obligation dropped before its turn", tlc) + def stopped_before_death(events): + # Rank 11 before 5e: the token unwound by the panicking turn was + # answered Stopped at once, before the death point fixed the reason. + held = next(e for e in events if e["a"] == "Send") + exit_at = next(i for i, e in enumerate(events) if e["a"] == "Exit") + events.insert(exit_at, {"a": "Discharge", "r": held["r"], "x": "stopped"}) + require_rejected(by_name["replies_panic"], stopped_before_death, + "exit error answered Stopped before the death point", tlc) + def job_trace_controls(traces, tlc): """Edits of recorded pipe-to-self job traces that DeferredJobs.tla must From 54e5549e2c71ad76321bee2509d02e7371aa1474 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 02:40:51 -0300 Subject: [PATCH 087/208] test: let the system kill land before releasing started() in expired_shutdown test The E3 kill at grace raced the release of started(); 1 in ~12 runs ended Shutdown instead of Killed. Same fix as checkpoint 4's two kill races. --- concurrency/tests/execution_groups.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/concurrency/tests/execution_groups.rs b/concurrency/tests/execution_groups.rs index d6e568ff..a8e1d80d 100644 --- a/concurrency/tests/execution_groups.rs +++ b/concurrency/tests/execution_groups.rs @@ -152,6 +152,9 @@ async fn expired_shutdown_retains_closed_runtime_until_startup_cleanup_finishes( assert_eq!(report.unfinished_groups, ["slow-start"]); assert!(group.is_closed()); assert!(Worker.start_on(&group).is_err()); + // The system kills what remains at grace (E3); let the kill land before + // started() returns, so the outcome does not race it. + spawned_rt::tasks::sleep(Duration::from_millis(100)).await; release.send(()).unwrap(); assert!(report.wait(LIMIT).await.complete); assert!(actor.ready().await.is_err()); From ada6ec41169153cbf9676759c8d79f96dbc21f81 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 02:40:51 -0300 Subject: [PATCH 088/208] feat(trace): lifecycle-trace mapping of real runs to Lifecycle.tla (rank 5e step 8a) The actor runtime records, with lifecycle-trace, one LifeEvent per Lifecycle.tla action for the actors spawned while record_lifecycle() is on (both modes): Trap, Send, Dispatch (item kind), Panic, Stop, Signal, Link/Unlink/Monitor/Demonitor, Deliver, Hook (stopped() run/skip), Die, Drop (runtime drop) and Publish, each inside the critical section of the state change it describes. TraceLifecycle.tla replays them (schema lifecycle-v1 in trace_check.py); tests/lifecycle_trace.rs records 14 scenarios, all accepted; eleven edited traces must be rejected (negative_checks.py). Zero cost without the feature. --- .github/workflows/verification.yml | 2 +- CHANGELOG.md | 11 +- concurrency/src/actor_cleanup.rs | 27 +- concurrency/src/child_handle.rs | 8 +- concurrency/src/lifecycle_trace.rs | 194 +++++++- concurrency/src/link.rs | 94 +++- concurrency/src/monitor.rs | 4 + concurrency/src/system.rs | 2 +- concurrency/src/tasks/actor.rs | 236 +++++++-- concurrency/src/threads/actor.rs | 231 +++++++-- concurrency/tests/lifecycle_trace.rs | 700 +++++++++++++++++++++++++++ docs/TESTING.md | 8 +- verification/tla/README.md | 42 +- verification/tla/TraceLifecycle.tla | 149 ++++++ verification/tla/negative_checks.py | 83 ++++ verification/tla/trace_check.py | 130 ++++- verification/tla/trace_coverage.py | 4 +- 17 files changed, 1796 insertions(+), 129 deletions(-) create mode 100644 concurrency/tests/lifecycle_trace.rs create mode 100644 verification/tla/TraceLifecycle.tla diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml index 670c7313..df77e335 100644 --- a/.github/workflows/verification.yml +++ b/.github/workflows/verification.yml @@ -30,7 +30,7 @@ jobs: - name: Record supervisor lifecycle traces env: LIFECYCLE_TRACE_DIR: ${{ github.workspace }}/lifecycle-traces - run: cargo test --locked -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace + run: cargo test --locked -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace --test lifecycle_trace - name: Model-check (fast configuration) and validate traces run: nix run .#check-tla -- --traces lifecycle-traces --negative-controls # Which model actions, projected states and transitions the recorded diff --git a/CHANGELOG.md b/CHANGELOG.md index 32664ac3..c98f1bc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,8 +59,15 @@ `DeferredReplies.tla` models it (`pendA`, invariant `StoppedAtDeath`, control `EarlyStopped`; one more reply trace control); metrics `requests_noproc_total`. - Remaining: lifecycle-trace mapping to `Lifecycle.tla`, Loom and simulator - coverage, GUIDE and examples. + Trace validation against `Lifecycle.tla`: with `lifecycle-trace`, the + runtime records exit signals, stops, links, monitors, sends, dispatch, + panics, `stopped()` run or skip, the death point, runtime drop, + publication and notice delivery (schema `lifecycle-v1`, + `TraceLifecycle.tla`); `tests/lifecycle_trace.rs` records 14 scenarios + (both modes), all accepted, and eleven edited traces (stopped() after a + kill, `Kill` for `Killed`, publication before D, real reason instead of + `NoProc`, notices after unlink/demonitor, ...) are rejected. + Remaining: Loom and simulator coverage, GUIDE and examples. - **Breaking (0.8), in progress:** deferred replies and pipe-to-self (ROADMAP rank 11), implementing diff --git a/concurrency/src/actor_cleanup.rs b/concurrency/src/actor_cleanup.rs index 39a8986c..f1fb9c39 100644 --- a/concurrency/src/actor_cleanup.rs +++ b/concurrency/src/actor_cleanup.rs @@ -1,4 +1,5 @@ //! Shared terminal publication for actor futures and native threads. +use crate::lifecycle_trace as trace; use crate::{ child_handle::ActorId, error::ExitReason, @@ -77,8 +78,14 @@ impl ActorCleanup { if let Some(reason) = &self.reason { return reason.clone(); } - let reason = crate::link::lock_request(&self.cause).die(); - self.close(reason) + let id = self.id.raw(); + trace::life_section(id, || { + let reason = crate::link::lock_request(&self.cause).die(); + trace::life_emit(id, || { + trace::LifeEvent::new("Die", id).x(trace::life_reason(&reason)) + }); + self.close(reason) + }) } fn close(&mut self, reason: ExitReason) -> ExitReason { let replies = self.replies.as_deref(); @@ -109,8 +116,14 @@ impl Drop for ActorCleanup { let reason = match self.reason.clone() { Some(reason) => reason, None => { - let reason = crate::link::lock_request(&self.cause).runtime_drop(); - self.close(reason) + let id = self.id.raw(); + trace::life_section(id, || { + let reason = crate::link::lock_request(&self.cause).runtime_drop(); + trace::life_emit(id, || { + trace::LifeEvent::new("Drop", id).x(trace::life_reason(&reason)) + }); + self.close(reason) + }) } }; if let Some(message) = self.failures.as_ref().and_then(|f| f.first()) { @@ -127,6 +140,12 @@ impl Drop for ActorCleanup { // wait_exit, join and monitors observe the final reason. Delivering // first keeps v0.7's guarantee that once `wait_exit` returns, every // linked peer's `Exit` is already queued (or its exit requested). + // The trace's `Publish` is this point: destruction is over and the + // notices are delivered after it. + let id = self.id.raw(); + trace::life_section(id, || { + trace::life_emit(id, || trace::LifeEvent::new("Publish", id)); + }); if let Some(notices) = self.notices.take() { notices.deliver(); } diff --git a/concurrency/src/child_handle.rs b/concurrency/src/child_handle.rs index 0a46e3c1..97308b1c 100644 --- a/concurrency/src/child_handle.rs +++ b/concurrency/src/child_handle.rs @@ -88,7 +88,9 @@ impl ChildHandle { pub(crate) fn new(target: Arc, completion: Completion, links: LinkTable) -> Self { let stop = target.clone(); Self { - cancel: Arc::new(move || crate::link::normal_stop(&stop.request, || (stop.wake)())), + cancel: Arc::new(move || { + crate::link::normal_stop(stop.id, &stop.request, || (stop.wake)()) + }), target, completion, links, @@ -156,7 +158,9 @@ impl ChildHandle { /// runs. `stop_with_reason(Kill)` ends `Kill`; use [`kill`](Self::kill) /// for the untrappable kill signal that ends `Killed`. pub fn stop_with_reason(&self, reason: ExitReason) { - crate::link::stop_with_reason(&self.target.request, reason, || (self.target.wake)()); + crate::link::stop_with_reason(self.target.id, &self.target.request, reason, || { + (self.target.wake)() + }); } /// Administrative stop with reason `Shutdown`, as [`stop`](Self::stop). diff --git a/concurrency/src/lifecycle_trace.rs b/concurrency/src/lifecycle_trace.rs index 0e237995..ee8b4f5b 100644 --- a/concurrency/src/lifecycle_trace.rs +++ b/concurrency/src/lifecycle_trace.rs @@ -1,9 +1,13 @@ -//! Supervisor lifecycle trace recording for TLA+ trace validation. +//! Lifecycle trace recording for TLA+ trace validation. //! //! With the `lifecycle-trace` feature, `run_supervisor`/`run_child` record one //! event per action of `verification/tla/Supervision.tla`, tagged with the -//! supervisor's trace id. `verification/tla/trace_check.py` then checks that -//! each recorded trace is a behavior of the model. +//! supervisor's trace id; the reply and job cores record `DeferredReplies.tla` +//! and `DeferredJobs.tla` actions; and the actor runtime records +//! `Lifecycle.tla` actions (`LifeEvent`, see `TraceLifecycle.tla`) for the +//! actors spawned while `record_lifecycle` is on. +//! `verification/tla/trace_check.py` then checks that each recorded trace is +//! a behavior of its model. //! //! The model's actions are atomic; the code's are not. Every event is //! recorded inside a process-wide *section*, and code that reads the @@ -164,6 +168,85 @@ impl ReplyEvent { } } +/// One base-lifecycle action of `verification/tla/Lifecycle.tla`, recorded +/// for the actors spawned while `record_lifecycle` is on. `t` is the actor +/// the action belongs to (the target of a signal or message, the caller of +/// link/monitor, the receiver of a notice), `s` the other actor or the +/// sender (`ActorId` numbers, normalized by the checker; 0 for outside code), +/// `x` the reason, flag, kind or outcome, `m` the monitor, `o` the notice +/// origin (`death`, `late`) or `sup` for a supervisor's signal. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LifeEvent { + pub action: &'static str, + pub t: u64, + pub s: u64, + pub x: &'static str, + pub m: u64, + pub k: &'static str, + pub o: &'static str, +} + +impl LifeEvent { + #[allow(dead_code)] // recording is compiled out without the feature + pub(crate) fn new(action: &'static str, t: u64) -> Self { + Self { + action, + t, + s: 0, + x: "", + m: 0, + k: "", + o: "", + } + } + #[allow(dead_code)] + pub(crate) fn s(mut self, s: u64) -> Self { + self.s = s; + self + } + #[allow(dead_code)] + pub(crate) fn x(mut self, x: &'static str) -> Self { + self.x = x; + self + } + #[allow(dead_code)] + pub(crate) fn m(mut self, m: u64) -> Self { + self.m = m; + self + } + #[allow(dead_code)] + pub(crate) fn k(mut self, k: &'static str) -> Self { + self.k = k; + self + } + #[allow(dead_code)] + pub(crate) fn o(mut self, o: &'static str) -> Self { + self.o = o; + self + } + /// JSON object understood by `verification/tla/trace_check.py`. + pub fn to_json(&self) -> String { + format!( + r#"{{"a":"{}","t":{},"s":{},"x":"{}","m":{},"k":"{}","o":"{}"}}"#, + self.action, self.t, self.s, self.x, self.m, self.k, self.o + ) + } +} + +/// Model name of an exit reason (`Lifecycle.tla` `Reasons`). +#[allow(dead_code)] // recording is compiled out without the feature +pub(crate) fn life_reason(reason: &crate::ExitReason) -> &'static str { + use crate::ExitReason; + match reason { + ExitReason::Normal => "Normal", + ExitReason::Shutdown => "Shutdown", + ExitReason::Panic(_) => "Panic", + ExitReason::Kill => "Kill", + ExitReason::Killed => "Killed", + ExitReason::NoProc => "NoProc", + } +} + pub(crate) fn flag(value: bool) -> &'static str { if value { "true" @@ -197,7 +280,7 @@ mod imp { cell::Cell, collections::BTreeMap, sync::{ - atomic::{AtomicU64, Ordering}, + atomic::{AtomicBool, AtomicU64, Ordering}, Mutex, MutexGuard, }, }; @@ -284,6 +367,90 @@ mod imp { lock(&WATCHED).remove(&owner); lock(&JOBS).remove(&owner).unwrap_or_default() } + + // Base lifecycle (`Lifecycle.tla`): one process-wide recording at a time. + static LIFE_ON: AtomicBool = AtomicBool::new(false); + static MEMBERS: Mutex> = + Mutex::new(std::collections::BTreeSet::new()); + static LIFE: Mutex> = Mutex::new(Vec::new()); + + /// Start recording the base-lifecycle events of every actor spawned from + /// now on until [`take_lifecycle`]. Tests that record must not run + /// concurrently with each other. + pub fn record_lifecycle() { + atomic(|| { + lock(&MEMBERS).clear(); + lock(&LIFE).clear(); + LIFE_ON.store(true, Ordering::Release); + }); + } + + /// Stop recording and return the events in their recorded order (a + /// linearization: each event is recorded in the critical section of the + /// state change it describes). + pub fn take_lifecycle() -> Vec { + atomic(|| { + LIFE_ON.store(false, Ordering::Release); + lock(&MEMBERS).clear(); + std::mem::take(&mut *lock(&LIFE)) + }) + } + + /// Whether actor `id` is recorded. + pub(crate) fn life_traced(id: u64) -> bool { + LIFE_ON.load(Ordering::Acquire) && lock(&MEMBERS).contains(&id) + } + + /// Run `f` as one atomic step of the recorded linearization if actor + /// `id` is recorded (and as is otherwise). Every lifecycle section is + /// keyed by the actor whose locks it takes first, and nested sections use + /// the same key, so the section is always the outermost lock. + pub(crate) fn life_section(id: u64, f: impl FnOnce() -> R) -> R { + if life_traced(id) { + atomic(f) + } else { + f() + } + } + + /// Record an event of actor `t` (inside its section). + pub(crate) fn life_emit(t: u64, event: impl FnOnce() -> super::LifeEvent) { + if life_traced(t) { + lock(&LIFE).push(event()); + } + } + + /// A new actor instance (after its `start_linked` link, before + /// `started()`): recorded from now on. + pub(crate) fn life_spawn(id: u64, parent: u64) { + if !LIFE_ON.load(Ordering::Acquire) { + return; + } + atomic(|| { + lock(&MEMBERS).insert(id); + lock(&LIFE).push(super::LifeEvent::new("Spawn", id).s(parent)); + }); + } + + /// The model's mailbox item kind of a message of type `M`. + pub(crate) fn life_kind() -> &'static str { + if std::any::TypeId::of::() == std::any::TypeId::of::() { + "down" + } else { + "req" + } + } + + /// A send or request to `owner` was admitted or refused (`NoProc`). A + /// `Down` is recorded by its monitor as a notice delivery instead. + pub(crate) fn life_send(owner: u64, admitted: bool) { + if life_kind::() == "down" { + return; + } + life_emit(owner, || { + super::LifeEvent::new("Send", owner).x(if admitted { "queued" } else { "noproc" }) + }); + } thread_local! { static DEPTH: Cell = const { Cell::new(0) }; static SUPPRESS_CLOSE: Cell = const { Cell::new(false) }; @@ -572,7 +739,24 @@ mod imp { #[allow(dead_code)] // recording is compiled out without the feature #[inline(always)] pub(crate) fn job(_: u64, _: impl FnOnce() -> super::JobEvent) {} + #[allow(dead_code)] + #[inline(always)] + pub(crate) fn life_section(_: u64, f: impl FnOnce() -> R) -> R { + f() + } + #[allow(dead_code)] + #[inline(always)] + pub(crate) fn life_emit(_: u64, _: impl FnOnce() -> super::LifeEvent) {} + #[allow(dead_code)] + #[inline(always)] + pub(crate) fn life_spawn(_: u64, _: u64) {} + #[allow(dead_code)] + #[inline(always)] + pub(crate) fn life_send(_: u64, _: bool) {} } #[cfg(feature = "lifecycle-trace")] -pub use imp::{recorded, take, take_jobs, take_replies, take_tree, watch_deferred}; +pub use imp::{ + record_lifecycle, recorded, take, take_jobs, take_lifecycle, take_replies, take_tree, + watch_deferred, +}; diff --git a/concurrency/src/link.rs b/concurrency/src/link.rs index cbe07cc6..aa9e8d9b 100644 --- a/concurrency/src/link.rs +++ b/concurrency/src/link.rs @@ -1,5 +1,6 @@ use crate::child_handle::ActorId; use crate::error::{ActorError, ExitReason}; +use crate::lifecycle_trace as trace; use std::panic::{catch_unwind, AssertUnwindSafe}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex, Weak}; @@ -209,6 +210,26 @@ pub(crate) fn deliver_signal( reason: ExitReason, origin: Origin, from_parent: bool, +) { + trace::life_section(target.id.raw(), || { + if origin == Origin::Direct { + trace::life_emit(target.id.raw(), || { + trace::LifeEvent::new("Signal", target.id.raw()) + .s(from.raw()) + .x(trace::life_reason(&reason)) + .o(if from_parent { "sup" } else { "" }) + }); + } + apply_signal(target, from, reason, origin, from_parent) + }) +} + +fn apply_signal( + target: &SignalTarget, + from: ActorId, + reason: ExitReason, + origin: Origin, + from_parent: bool, ) { if origin == Origin::Direct && reason == ExitReason::Kill { if lock_request(&target.request).kill() { @@ -223,6 +244,10 @@ pub(crate) fn deliver_signal( } let exit = Exit { from, reason }; if request.starting { + // Phase 0 is not in the lifecycle trace mapping (TraceLifecycle.tla). + trace::life_emit(target.id.raw(), || { + trace::LifeEvent::new("Held", target.id.raw()).s(from.raw()) + }); // Held in arrival order, and as an envelope in its mailbox place. request.held.push(exit.reason.clone()); let _ = (target.send_held)(exit, from_parent); @@ -335,6 +360,18 @@ pub(crate) fn register_link( if own.id == peer.id || Arc::ptr_eq(own_links, peer_links) { return LinkOutcome::Unchanged; } + trace::life_section(own.id.raw(), || { + link_in_section(own, own_links, peer, peer_links, try_only) + }) +} + +fn link_in_section( + own: &Arc, + own_links: &LinkTable, + peer: &Arc, + peer_links: &LinkTable, + try_only: bool, +) -> LinkOutcome { // Sender lifetime (design Terms, semantics X2): `link` works only while // the caller runs. Once its exit began (or past its D, checked again // under the table lock below) the call is inert: no record, no notice, @@ -376,6 +413,17 @@ pub(crate) fn register_link( }); LinkOutcome::Linked }); + if outcome != LinkOutcome::Unchanged { + trace::life_emit(own.id.raw(), || { + trace::LifeEvent::new("Link", own.id.raw()) + .s(peer.id.raw()) + .x(match (&outcome, try_only) { + (LinkOutcome::Linked, _) => "linked", + (_, false) => "noproc", + (_, true) => "try", + }) + }); + } if outcome == LinkOutcome::NoProc && !try_only { deliver_notice(own_links, own, peer.id, gen, ExitReason::NoProc); } @@ -401,7 +449,15 @@ pub(crate) fn unregister_link( if own_id == peer_id || Arc::ptr_eq(own, peer) { return; } - let removed = with_pair(own, peer, |a, b| (remove(a, peer_id), remove(b, own_id))); + let removed = trace::life_section(own_id.raw(), || { + let removed = with_pair(own, peer, |a, b| (remove(a, peer_id), remove(b, own_id))); + if removed.0.is_some() || removed.1.is_some() { + trace::life_emit(own_id.raw(), || { + trace::LifeEvent::new("Unlink", own_id.raw()).s(peer_id.raw()) + }); + } + removed + }); drop(removed); } @@ -414,6 +470,18 @@ fn deliver_notice( from: ActorId, gen: u64, reason: ExitReason, +) { + trace::life_section(receiver.id.raw(), || { + deliver_notice_in_section(receiver_links, receiver, from, gen, reason) + }) +} + +fn deliver_notice_in_section( + receiver_links: &LinkTable, + receiver: &SignalTarget, + from: ActorId, + gen: u64, + reason: ExitReason, ) { let mut table = receiver_links.lock().unwrap_or_else(|p| p.into_inner()); let Some(i) = table @@ -424,6 +492,13 @@ fn deliver_notice( return; }; let record = table.entries.swap_remove(i); + trace::life_emit(receiver.id.raw(), || { + trace::LifeEvent::new("Deliver", receiver.id.raw()) + .s(from.raw()) + .x(trace::life_reason(&reason)) + .k("link") + .o(if record.late { "late" } else { "death" }) + }); // One failing internal callback must not prevent other peers from being // notified, or panic again during actor-future destruction. if let Err(payload) = catch_unwind(AssertUnwindSafe(|| { @@ -503,15 +578,28 @@ pub(crate) fn propagate_exit(own_id: ActorId, own: &LinkTable, reason: &ExitReas /// A self-exit or administrative stop commits its cause before waking. The /// first requested reason wins over later self-exits and stops; a later /// untrapped signal or kill can still replace it until D. -pub(crate) fn normal_stop(reason: &LinkedExitReason, cancel: impl FnOnce()) { - stop_with_reason(reason, ExitReason::Normal, cancel); +pub(crate) fn normal_stop(id: ActorId, reason: &LinkedExitReason, cancel: impl FnOnce()) { + stop_with_reason(id, reason, ExitReason::Normal, cancel); } +/// A self-exit or administrative stop of actor `id` (recorded as `Stop`). pub(crate) fn stop_with_reason( + id: ActorId, reason: &LinkedExitReason, cause: ExitReason, cancel: impl FnOnce(), ) { + trace::life_section(id.raw(), || { + trace::life_emit(id.raw(), || { + trace::LifeEvent::new("Stop", id.raw()).x(trace::life_reason(&cause)) + }); + request_stop(reason, cause, cancel); + }); +} + +/// As [`stop_with_reason`], unrecorded: a parent exit being dispatched (part +/// of the model's `Dispatch`) or a managed scope's stop. +pub(crate) fn request_stop(reason: &LinkedExitReason, cause: ExitReason, cancel: impl FnOnce()) { if lock_request(reason).request(cause) { cancel(); } diff --git a/concurrency/src/monitor.rs b/concurrency/src/monitor.rs index 578f835c..fdb04db7 100644 --- a/concurrency/src/monitor.rs +++ b/concurrency/src/monitor.rs @@ -23,6 +23,10 @@ impl MonitorRef { pub(crate) fn next() -> Self { Self(NEXT_MONITOR_ID.fetch_add(1, Ordering::Relaxed)) } + #[allow(dead_code)] // used by lifecycle tracing only + pub(crate) fn raw(self) -> u64 { + self.0 + } } impl std::fmt::Display for MonitorRef { diff --git a/concurrency/src/system.rs b/concurrency/src/system.rs index ebb36c94..1980df78 100644 --- a/concurrency/src/system.rs +++ b/concurrency/src/system.rs @@ -989,7 +989,7 @@ impl Scope { state.ended_at = Some(Instant::now()); } } - link::stop_with_reason(&self.cause, reason, || self.token.cancel()); + link::request_stop(&self.cause, reason, || self.token.cancel()); } pub(crate) fn close_admission(&self) { lock(&self.state).open = false; diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index b700d7ca..78d02737 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -140,6 +140,11 @@ pub trait Handler: Actor { // --------------------------------------------------------------------------- trait Envelope: Send { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "other" + } #[cfg(feature = "diagnostics")] fn message_type(&self) -> &'static str { std::any::type_name::() @@ -170,6 +175,11 @@ where A: Actor + Handler, M: Message, { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + crate::lifecycle_trace::life_kind::() + } #[cfg(feature = "diagnostics")] fn message_type(&self) -> &'static str { std::any::type_name::() @@ -216,25 +226,32 @@ where #[cfg(feature = "metrics")] observation: Some(_mailbox.enqueued()), }); - if replies.refuses_admission() { - #[cfg(feature = "metrics")] - envelope.reject(); - return Err(ActorError::NoProc); - } - match sender.send(envelope) { - Ok(()) => { + // Admission and its trace event are one step of the lifecycle trace. + let owner = replies.owner().raw(); + crate::lifecycle_trace::life_section(owner, || { + if replies.refuses_admission() { #[cfg(feature = "metrics")] - crate::instrumentation::accepted(); - Ok(()) + envelope.reject(); + crate::lifecycle_trace::life_send::(owner, false); + return Err(ActorError::NoProc); } - Err(_error) => { - #[cfg(feature = "metrics")] - { - _error.0.reject(); + match sender.send(envelope) { + Ok(()) => { + #[cfg(feature = "metrics")] + crate::instrumentation::accepted(); + crate::lifecycle_trace::life_send::(owner, true); + Ok(()) + } + Err(_error) => { + #[cfg(feature = "metrics")] + { + _error.0.reject(); + } + crate::lifecycle_trace::life_send::(owner, false); + Err(ActorError::NoProc) } - Err(ActorError::NoProc) } - } + }) } // Private lifecycle signal, excluded from application message/handler metrics. @@ -266,6 +283,11 @@ struct ExitEnvelope { } impl Envelope for ExitEnvelope { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "exit" + } #[cfg(feature = "diagnostics")] fn message_type(&self) -> &'static str { "Exit" @@ -289,6 +311,11 @@ struct ParentExitEnvelope { } impl Envelope for ParentExitEnvelope { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "pexit" + } #[cfg(feature = "diagnostics")] fn message_type(&self) -> &'static str { "ParentExit" @@ -299,7 +326,7 @@ impl Envelope for ParentExitEnvelope { _actor: &'a mut A, ctx: &'a Context, ) -> Pin + Send + 'a>> { - Box::pin(async move { ctx.stop_with_reason(self.exit.reason) }) + Box::pin(async move { ctx.parent_exit(self.exit.reason) }) } } @@ -312,6 +339,11 @@ struct HeldEnvelope { } impl Envelope for HeldEnvelope { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "held" + } #[cfg(feature = "diagnostics")] fn message_type(&self) -> &'static str { "Exit" @@ -327,7 +359,7 @@ impl Envelope for HeldEnvelope { return; } if self.parent { - ctx.stop_with_reason(self.exit.reason); + ctx.parent_exit(self.exit.reason); } else { actor.exit_received(self.exit, ctx).await; } @@ -467,7 +499,15 @@ impl Context { /// a panic (in the handler or `stopped()`) with `Panic`. `Kill` stays /// `Kill` and propagates over links as `Kill` (trappable). pub fn stop_with_reason(&self, reason: ExitReason) { - link::stop_with_reason(&self.shared.linked_reason, reason, || { + link::stop_with_reason(self.shared.id, &self.shared.linked_reason, reason, || { + self.shared.cancellation_token.cancel() + }); + } + + /// A parent exit being dispatched: normal termination with its reason + /// (part of the model's `Dispatch`, so not recorded as a stop). + fn parent_exit(&self, reason: ExitReason) { + link::request_stop(&self.shared.linked_reason, reason, || { self.shared.cancellation_token.cancel() }); } @@ -613,7 +653,8 @@ impl Context { let monitor_ref = MonitorRef::next(); let active = spawned_rt::tasks::CancellationToken::new(); - { + let observer = self.shared.id.raw(); + let registered = crate::lifecycle_trace::life_section(observer, || { let mut monitors = self .shared .monitors @@ -623,21 +664,41 @@ impl Context { // The closed table below serializes registration with exit cleanup, // even on paths that unwind before cancelling the owner token. if self.shared.cancellation_token.is_cancelled() { - return monitor_ref; + return false; } let Some(monitors) = monitors.as_mut() else { - return monitor_ref; + return false; + }; + let event = |x| { + crate::lifecycle_trace::LifeEvent::new("Monitor", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x(x) }; // Linearised against the target's death point (E4): past it the // only notice is NoProc, delivered under the observer's lock. if target.is_past_death() { + crate::lifecycle_trace::life_emit(observer, || event("late")); + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Deliver", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x("NoProc") + .k("down") + .o("late") + }); let _ = self.send(Down { monitor_ref, reason: ExitReason::NoProc, }); - return monitor_ref; + return false; } monitors.insert(monitor_ref, active.clone()); + crate::lifecycle_trace::life_emit(observer, || event("active")); + true + }); + if !registered { + return monitor_ref; } let target = target.clone(); @@ -659,18 +720,28 @@ impl Context { // Receiver-side fence (E6): the entry is removed and the Down // enqueued in one critical section, so once `demonitor` returns // no Down of this monitor is enqueued. - let mut entries = monitors.lock().unwrap_or_else(|p| p.into_inner()); - if entries - .as_mut() - .and_then(|entries| entries.remove(&monitor_ref)) - .is_some() - { - let _ = actor_ref.send(Down { - monitor_ref, - reason, - }); - } - drop(entries); + crate::lifecycle_trace::life_section(observer, || { + let mut entries = monitors.lock().unwrap_or_else(|p| p.into_inner()); + if entries + .as_mut() + .and_then(|entries| entries.remove(&monitor_ref)) + .is_some() + { + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Deliver", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x(crate::lifecycle_trace::life_reason(&reason)) + .k("down") + .o("death") + }); + let _ = actor_ref.send(Down { + monitor_ref, + reason, + }); + } + drop(entries); + }); }); monitor_ref @@ -681,13 +752,23 @@ impl Context { /// A fence (OTP `demonitor/1`): once it returns, no `Down` of this /// monitor is enqueued; one already queued stays. pub fn demonitor(&self, monitor_ref: MonitorRef) { - let active = self - .shared - .monitors - .lock() - .unwrap_or_else(|p| p.into_inner()) - .as_mut() - .and_then(|entries| entries.remove(&monitor_ref)); + let observer = self.shared.id.raw(); + let active = crate::lifecycle_trace::life_section(observer, || { + let active = self + .shared + .monitors + .lock() + .unwrap_or_else(|p| p.into_inner()) + .as_mut() + .and_then(|entries| entries.remove(&monitor_ref)); + if active.is_some() { + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Demonitor", observer) + .m(monitor_ref.raw()) + }); + } + active + }); // Waking the waiter must happen after releasing the table lock. if let Some(active) = active { active.cancel(); @@ -762,7 +843,16 @@ impl Context { /// `started()`: signals that arrive while `started()` runs are resolved /// when it returns. Handlers are never preempted. pub fn trap_exit(&self, enabled: bool) { - self.shared.trap_exit.store(enabled, Ordering::Release); + let id = self.shared.id.raw(); + crate::lifecycle_trace::life_section(id, || { + // Recorded only when it changes (the model's SetTrap). + if self.shared.trap_exit.swap(enabled, Ordering::AcqRel) != enabled { + crate::lifecycle_trace::life_emit(id, || { + crate::lifecycle_trace::LifeEvent::new("Trap", id) + .x(crate::lifecycle_trace::flag(enabled)) + }); + } + }); } /// Type-erased handle to this actor, without an intermediate `ActorRef`. @@ -1237,9 +1327,12 @@ impl ActorRef { scope.set_jobs(Arc::downgrade(&actor_ref.shared.jobs)); } + let parent_id = parent.as_ref().map_or(0, |p| p.id().raw()); if let Some(parent) = parent { ctx.link(&parent); } + // Recorded from here: the start_linked link is the model's initial one. + crate::lifecycle_trace::life_spawn(id.raw(), parent_id); #[cfg(feature = "backend-experiments")] let scheduler = if managed.is_some() { @@ -1543,6 +1636,7 @@ async fn run_actor_lifecycle( ) { let _monitor_cleanup = MonitorCleanup(ctx.shared.monitors.clone()); let request = ctx.shared.linked_reason.clone(); + let trace_id = ctx.shared.id.raw(); if let Some(scope) = &ctx.shared.managed { if !scope.can_start() { scope.stop(ExitReason::Shutdown); @@ -1561,7 +1655,13 @@ async fn run_actor_lifecycle( tracing::error!("Panic in started() callback: {msg}"); }); // No stopped() after a failed start (L5). - link::lock_request(&request).panic(ExitReason::Panic(format!("panic in started(): {msg}"))); + crate::lifecycle_trace::life_section(trace_id, || { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("started") + }); + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in started(): {msg}"))); + }); cancellation_token.cancel(); return; } @@ -1584,10 +1684,22 @@ async fn run_actor_lifecycle( // Stop and closed-mailbox both take the normal lifecycle exit, // just like the cancellation arm below. Check before consuming // work, including when receive resumes from a pending poll. - if cancellation_token.is_cancelled() { - return std::task::Poll::Ready(None); - } - let result = rx.poll_recv(cx); + // The check, the receive and its trace event are one step + // of the lifecycle trace: a recorded stop is seen here. + let result = crate::lifecycle_trace::life_section(trace_id, || { + if cancellation_token.is_cancelled() { + return std::task::Poll::Ready(None); + } + let result = rx.poll_recv(cx); + #[cfg(feature = "lifecycle-trace")] + if let std::task::Poll::Ready(Some(envelope)) = &result { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Dispatch", trace_id) + .k(envelope.trace_kind()) + }); + } + result + }); #[cfg(feature = "backend-experiments")] if let Some(turn) = &mut turn { if result.is_pending() { @@ -1637,8 +1749,13 @@ async fn run_actor_lifecycle( }); // A panic begins exit, or replaces the requested reason // unless a kill was received. - link::lock_request(&request) - .panic(ExitReason::Panic(format!("panic in handler: {msg}"))); + crate::lifecycle_trace::life_section(trace_id, || { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("turn") + }); + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in handler: {msg}"))); + }); break; } #[cfg(feature = "backend-experiments")] @@ -1663,7 +1780,18 @@ async fn run_actor_lifecycle( ctx.shared.jobs.cancel_all(); // stopped() is skipped when a kill, an untrapped exit signal or link // notice began the exit (decision 2); one that already began finishes. - if !link::lock_request(&request).begin_hook() { + let run_hook = crate::lifecycle_trace::life_section(trace_id, || { + let run = link::lock_request(&request).begin_hook(); + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Hook", trace_id).x(if run { + "run" + } else { + "skip" + }) + }); + run + }); + if !run_hook { return; } let stop_result = AssertUnwindSafe(async { @@ -1677,7 +1805,13 @@ async fn run_actor_lifecycle( crate::actor_cleanup::log_safely(|| { tracing::error!("Panic in stopped() callback: {msg}"); }); - link::lock_request(&request).panic(ExitReason::Panic(format!("panic in stopped(): {msg}"))); + crate::lifecycle_trace::life_section(trace_id, || { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("stopped") + }); + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in stopped(): {msg}"))); + }); } } diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index cc54eb7b..13689acb 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -79,6 +79,11 @@ pub trait Handler: Actor { // --------------------------------------------------------------------------- trait Envelope: Send { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "other" + } #[cfg(feature = "metrics")] fn observation(&mut self) -> Option { None @@ -101,6 +106,11 @@ where A: Actor + Handler, M: Message, { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + crate::lifecycle_trace::life_kind::() + } #[cfg(feature = "metrics")] fn observation(&mut self) -> Option { self.observation.take() @@ -139,27 +149,34 @@ where #[cfg(feature = "metrics")] observation: Some(_mailbox.enqueued()), }); - if replies.refuses_admission() { - #[cfg(feature = "metrics")] - envelope.reject(); - return Err(ActorError::NoProc); - } - match sender.send(MailboxItem::Message(envelope)) { - Ok(()) => { + // Admission and its trace event are one step of the lifecycle trace. + let owner = replies.owner().raw(); + crate::lifecycle_trace::life_section(owner, || { + if replies.refuses_admission() { #[cfg(feature = "metrics")] - crate::instrumentation::accepted(); - Ok(()) + envelope.reject(); + crate::lifecycle_trace::life_send::(owner, false); + return Err(ActorError::NoProc); } - Err(_error) => { - #[cfg(feature = "metrics")] - { - if let MailboxItem::Message(envelope) = _error.0 { - envelope.reject(); + match sender.send(MailboxItem::Message(envelope)) { + Ok(()) => { + #[cfg(feature = "metrics")] + crate::instrumentation::accepted(); + crate::lifecycle_trace::life_send::(owner, true); + Ok(()) + } + Err(_error) => { + #[cfg(feature = "metrics")] + { + if let MailboxItem::Message(envelope) = _error.0 { + envelope.reject(); + } } + crate::lifecycle_trace::life_send::(owner, false); + Err(ActorError::NoProc) } - Err(ActorError::NoProc) } - } + }) } /// System envelope that dispatches an `Exit` notification to the actor's @@ -169,6 +186,11 @@ struct ExitEnvelope { } impl Envelope for ExitEnvelope { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "exit" + } fn handle(self: Box, actor: &mut A, ctx: &Context) { actor.exit_received(self.exit, ctx); } @@ -181,8 +203,13 @@ struct ParentExitEnvelope { } impl Envelope for ParentExitEnvelope { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "pexit" + } fn handle(self: Box, _actor: &mut A, ctx: &Context) { - ctx.stop_with_reason(self.exit.reason); + ctx.parent_exit(self.exit.reason); } } @@ -195,12 +222,17 @@ struct HeldEnvelope { } impl Envelope for HeldEnvelope { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "held" + } fn handle(self: Box, actor: &mut A, ctx: &Context) { if !link::lock_request(&ctx.linked_reason).held_trapped() { return; } if self.parent { - ctx.stop_with_reason(self.exit.reason); + ctx.parent_exit(self.exit.reason); } else { actor.exit_received(self.exit, ctx); } @@ -367,7 +399,15 @@ impl Context { /// a panic (in the handler or `stopped()`) with `Panic`. `Kill` stays /// `Kill` and propagates over links as `Kill` (trappable). pub fn stop_with_reason(&self, reason: ExitReason) { - link::stop_with_reason(&self.linked_reason, reason, || { + link::stop_with_reason(self.id, &self.linked_reason, reason, || { + self.cancellation_token.cancel() + }); + } + + /// A parent exit being dispatched: normal termination with its reason + /// (part of the model's `Dispatch`, so not recorded as a stop). + fn parent_exit(&self, reason: ExitReason) { + link::request_stop(&self.linked_reason, reason, || { self.cancellation_token.cancel() }); } @@ -512,27 +552,48 @@ impl Context { let monitor_ref = MonitorRef::next(); let active = spawned_rt::sync::CancellationToken::new(); - { + let observer = self.id.raw(); + let registered = crate::lifecycle_trace::life_section(observer, || { let mut monitors = self.monitors.lock().unwrap_or_else(|p| p.into_inner()); // The token check makes registration inert once shutdown starts. // The closed table below serializes registration with exit cleanup, // even on paths that unwind before cancelling the owner token. if self.cancellation_token.is_cancelled() { - return monitor_ref; + return false; } let Some(monitors) = monitors.as_mut() else { - return monitor_ref; + return false; + }; + let event = |x| { + crate::lifecycle_trace::LifeEvent::new("Monitor", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x(x) }; // Linearised against the target's death point (E4): past it the // only notice is NoProc, delivered under the observer's lock. if target.is_past_death() { + crate::lifecycle_trace::life_emit(observer, || event("late")); + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Deliver", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x("NoProc") + .k("down") + .o("late") + }); let _ = self.send(Down { monitor_ref, reason: ExitReason::NoProc, }); - return monitor_ref; + return false; } monitors.insert(monitor_ref, active.clone()); + crate::lifecycle_trace::life_emit(observer, || event("active")); + true + }); + if !registered { + return monitor_ref; } let target = target.clone(); @@ -557,18 +618,28 @@ impl Context { // Receiver-side fence (E6): the entry is removed and the Down // enqueued in one critical section, so once `demonitor` returns // no Down of this monitor is enqueued. - let mut entries = monitors.lock().unwrap_or_else(|p| p.into_inner()); - if entries - .as_mut() - .and_then(|entries| entries.remove(&monitor_ref)) - .is_some() - { - let _ = actor_ref.send(Down { - monitor_ref, - reason, - }); - } - drop(entries); + crate::lifecycle_trace::life_section(observer, || { + let mut entries = monitors.lock().unwrap_or_else(|p| p.into_inner()); + if entries + .as_mut() + .and_then(|entries| entries.remove(&monitor_ref)) + .is_some() + { + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Deliver", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x(crate::lifecycle_trace::life_reason(&reason)) + .k("down") + .o("death") + }); + let _ = actor_ref.send(Down { + monitor_ref, + reason, + }); + } + drop(entries); + }); }); monitor_ref @@ -579,12 +650,22 @@ impl Context { /// A fence (OTP `demonitor/1`): once it returns, no `Down` of this /// monitor is enqueued; one already queued stays. pub fn demonitor(&self, monitor_ref: MonitorRef) { - let active = self - .monitors - .lock() - .unwrap_or_else(|p| p.into_inner()) - .as_mut() - .and_then(|entries| entries.remove(&monitor_ref)); + let observer = self.id.raw(); + let active = crate::lifecycle_trace::life_section(observer, || { + let active = self + .monitors + .lock() + .unwrap_or_else(|p| p.into_inner()) + .as_mut() + .and_then(|entries| entries.remove(&monitor_ref)); + if active.is_some() { + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Demonitor", observer) + .m(monitor_ref.raw()) + }); + } + active + }); // Waking the waiter must happen after releasing the table lock. if let Some(active) = active { active.cancel(); @@ -650,7 +731,16 @@ impl Context { /// trapping actor normally, through `stopped()`. Signals that arrive while /// `started()` runs are resolved when it returns. pub fn trap_exit(&self, enabled: bool) { - self.trap_exit.store(enabled, Ordering::Release); + let id = self.id.raw(); + crate::lifecycle_trace::life_section(id, || { + // Recorded only when it changes (the model's SetTrap). + if self.trap_exit.swap(enabled, Ordering::AcqRel) != enabled { + crate::lifecycle_trace::life_emit(id, || { + crate::lifecycle_trace::LifeEvent::new("Trap", id) + .x(crate::lifecycle_trace::flag(enabled)) + }); + } + }); } pub(crate) fn cancellation_token(&self) -> CancellationToken { @@ -993,9 +1083,12 @@ impl ActorRef { cleaned: actor_ref.cleaned.clone(), }; + let parent_id = parent.as_ref().map_or(0, |p| p.id().raw()); if let Some(parent) = parent { actor_ref.context().link(&parent); } + // Recorded from here: the start_linked link is the model's initial one. + crate::lifecycle_trace::life_spawn(id.raw(), parent_id); let cleanup_token = cancellation_token.clone(); let cleanup = crate::actor_cleanup::ActorCleanup::new( @@ -1079,6 +1172,7 @@ fn run_actor_receiving( ) { let mut discarded = None; let request = ctx.linked_reason.clone(); + let trace_id = ctx.id.raw(); let _monitor_cleanup = MonitorCleanup(ctx.monitors.clone()); let start_result = catch_unwind(AssertUnwindSafe(|| { fail_point!("actor.before_started"); @@ -1090,7 +1184,13 @@ fn run_actor_receiving( tracing::error!("Panic in started() callback: {msg}"); }); // No stopped() after a failed start (L5). - link::lock_request(&request).panic(ExitReason::Panic(format!("panic in started(): {msg}"))); + crate::lifecycle_trace::life_section(trace_id, || { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("started") + }); + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in started(): {msg}"))); + }); cancellation_token.cancel(); return; } @@ -1114,7 +1214,19 @@ fn run_actor_receiving( // A stop observed after receive wins over dispatch, whether // the channel waited or was buffered. Defer the message drop. - if cancellation_token.is_cancelled() { + // The check and the trace event are one lifecycle-trace step. + let stopped = crate::lifecycle_trace::life_section(trace_id, || { + if cancellation_token.is_cancelled() { + return true; + } + #[cfg(feature = "lifecycle-trace")] + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Dispatch", trace_id) + .k(envelope.trace_kind()) + }); + false + }); + if stopped { discarded = Some(envelope); break; } @@ -1138,8 +1250,13 @@ fn run_actor_receiving( }); // A panic begins exit, or replaces the requested reason // unless a kill was received. - link::lock_request(&request) - .panic(ExitReason::Panic(format!("panic in handler: {msg}"))); + crate::lifecycle_trace::life_section(trace_id, || { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("turn") + }); + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in handler: {msg}"))); + }); break; } } @@ -1152,7 +1269,18 @@ fn run_actor_receiving( ctx.jobs.cancel_all(); // stopped() is skipped when a kill, an untrapped exit signal or link // notice began the exit (decision 2); one that already began finishes. - if link::lock_request(&request).begin_hook() { + let run_hook = crate::lifecycle_trace::life_section(trace_id, || { + let run = link::lock_request(&request).begin_hook(); + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Hook", trace_id).x(if run { + "run" + } else { + "skip" + }) + }); + run + }); + if run_hook { let stop_result = catch_unwind(AssertUnwindSafe(|| { fail_point!("actor.before_stopped"); actor.stopped(&ctx); @@ -1162,8 +1290,13 @@ fn run_actor_receiving( crate::actor_cleanup::log_safely(|| { tracing::error!("Panic in stopped() callback: {msg}"); }); - link::lock_request(&request) - .panic(ExitReason::Panic(format!("panic in stopped(): {msg}"))); + crate::lifecycle_trace::life_section(trace_id, || { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("stopped") + }); + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in stopped(): {msg}"))); + }); } } diff --git a/concurrency/tests/lifecycle_trace.rs b/concurrency/tests/lifecycle_trace.rs new file mode 100644 index 00000000..2e665b7e --- /dev/null +++ b/concurrency/tests/lifecycle_trace.rs @@ -0,0 +1,700 @@ +//! Records base-lifecycle traces (exit signals, links, monitors, requests, +//! the death point and publication) from real actors for TLA+ trace +//! validation against `verification/tla/Lifecycle.tla` (mapping in +//! `TraceLifecycle.tla`). Each scenario writes `lifecycle_.json` to +//! `$LIFECYCLE_TRACE_DIR` (default `target/lifecycle-traces`); +//! `verification/tla/trace_check.py` checks them (schema `lifecycle-v1`). +//! +//! Recording is process-wide, so the scenarios run one at a time. +#![cfg(feature = "lifecycle-trace")] + +use spawned_concurrency::{ + lifecycle_trace, + message::Message, + tasks::{Actor, ActorRef, ActorStart, Backend, Context, Handler}, + ActorError, ChildHandle, Down, Exit, ExitReason, MonitorRef, PendingReply, +}; +use std::{ + path::PathBuf, + sync::{Arc, Mutex}, + time::Duration, +}; + +const LIMIT: Duration = Duration::from_secs(10); + +/// Held for a whole scenario (also across awaits, so an async mutex). +static SERIAL: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +fn trace_dir() -> PathBuf { + std::env::var_os("LIFECYCLE_TRACE_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../target/lifecycle-traces") + }) +} + +/// Stop recording and write the trace; returns the events as +/// `action[:x]` of the actor numbered by spawn order (`a1`, `a2`, ...). +fn write(name: &str) -> Vec { + let events = lifecycle_trace::take_lifecycle(); + let json = format!( + "{{\"name\":\"lifecycle_{name}\",\"schema\":\"lifecycle-v1\",\"events\":[\n{}\n]}}\n", + events + .iter() + .map(|e| e.to_json()) + .collect::>() + .join(",\n") + ); + let dir = trace_dir(); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join(format!("lifecycle_{name}.json")), json).unwrap(); + let actors: Vec = events + .iter() + .filter(|e| e.action == "Spawn") + .map(|e| e.t) + .collect(); + let index = |id: u64| actors.iter().position(|a| *a == id).map_or(0, |i| i + 1); + events + .iter() + .filter(|e| e.action != "Spawn") + .map(|e| { + let x = if e.x.is_empty() { + String::new() + } else { + format!(":{}", e.x) + }; + format!("a{}.{}{x}", index(e.t), e.action) + }) + .collect() +} + +fn has(events: &[String], wanted: &[&str]) { + // `wanted` must appear in this order (other events may interleave). + let mut at = 0; + for w in wanted { + match events[at..].iter().position(|e| e == w) { + Some(i) => at += i + 1, + None => panic!("{w} missing (in order) from {events:#?}"), + } + } +} + +type Log = Arc>>; + +fn logged(log: &Log) -> Vec { + log.lock().unwrap().clone() +} + +async fn wait_for(log: &Log, entry: &str) { + spawned_rt::tasks::timeout(LIMIT, async { + while !log.lock().unwrap().iter().any(|e| e == entry) { + spawned_rt::tasks::sleep(Duration::from_millis(2)).await; + } + }) + .await + .unwrap_or_else(|_| panic!("{entry} never logged: {:?}", logged(log))); +} + +struct Node { + trap: bool, + log: Log, + /// `stopped()` waits for it. + hold_stopped: Option>, + /// The state destructor waits for it (a native thread: Backend::Thread). + hold_drop: Option>, +} + +impl Node { + fn new(log: &Log) -> Self { + Self { + trap: false, + log: log.clone(), + hold_stopped: None, + hold_drop: None, + } + } + fn trapping(log: &Log) -> Self { + let mut node = Self::new(log); + node.trap = true; + node + } + fn holding_stopped(log: &Log, gate: tokio::sync::oneshot::Receiver<()>) -> Self { + let mut node = Self::new(log); + node.hold_stopped = Some(gate); + node + } + fn holding_drop(gate: std::sync::mpsc::Receiver<()>) -> Self { + let mut node = Self::new(&Log::default()); + node.hold_drop = Some(gate); + node + } + fn note(&self, entry: impl Into) { + self.log.lock().unwrap().push(entry.into()); + } +} + +impl Drop for Node { + fn drop(&mut self) { + if let Some(gate) = self.hold_drop.take() { + let _ = gate.recv_timeout(LIMIT); + } + } +} + +impl Actor for Node { + async fn started(&mut self, ctx: &Context) { + if self.trap { + ctx.trap_exit(true); + } + } + async fn stopped(&mut self, _: &Context) { + self.note("stopped"); + if let Some(gate) = self.hold_stopped.take() { + let _ = gate.await; + } + } + async fn exit_received(&mut self, exit: Exit, _: &Context) { + self.note(format!("exit:{:?}", exit.reason)); + } +} + +impl Handler for Node { + async fn handle(&mut self, down: Down, _: PendingReply<'_, ()>, _: &Context) { + self.note(format!("down:{:?}", down.reason)); + } +} + +macro_rules! request { + ($name:ident ( $($field:ty),* ) -> $result:ty, |$this:ident, $msg:ident, $ctx:ident| $body:expr) => { + struct $name($(pub $field),*); + impl Message for $name { + type Result = $result; + } + impl Handler<$name> for Node { + async fn handle( + &mut self, + $msg: $name, + reply: PendingReply<'_, $result>, + $ctx: &Context, + ) { + let $this = self; + let result = $body; + reply.reply(result); + } + } + }; +} + +request!(Sync() -> (), |_this, _msg, _ctx| ()); +request!(Link(ChildHandle) -> (), |_this, msg, ctx| ctx.link(&msg.0)); +request!(TryLink(ChildHandle) -> bool, |_this, msg, ctx| ctx.try_link(&msg.0).is_ok()); +request!(Unlink(ChildHandle) -> (), |_this, msg, ctx| ctx.unlink(&msg.0)); +request!(Monitor(ChildHandle) -> MonitorRef, |_this, msg, ctx| ctx.monitor(&msg.0)); +request!(Demonitor(MonitorRef) -> (), |_this, msg, ctx| ctx.demonitor(msg.0)); + +/// A request whose handler panics. +struct Boom(); +impl Message for Boom { + type Result = (); +} +impl Handler for Node { + async fn handle(&mut self, _: Boom, _: PendingReply<'_, ()>, _: &Context) { + panic!("boom"); + } +} + +/// A request whose handler waits for a gate. +struct Block(tokio::sync::oneshot::Receiver<()>); +impl Message for Block { + type Result = (); +} +impl Handler for Node { + async fn handle(&mut self, msg: Block, reply: PendingReply<'_, ()>, _: &Context) { + self.note("blocked"); + let _ = msg.0.await; + reply.reply(()); + } +} + +async fn exit_of(actor: &ActorRef) -> ExitReason { + spawned_rt::tasks::timeout(LIMIT, actor.wait_exit()) + .await + .expect("actor did not exit") +} + +/// A kill ends a running actor `Killed` without `stopped()`; the request +/// queued behind the current handler is answered at D with `Killed`, and a +/// request after D gets `NoProc`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn kill_answers_queued_request_at_death() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::new(&log).start(); + let (release, gate) = tokio::sync::oneshot::channel(); + let blocked = a1.request_raw(Block(gate)).unwrap(); + wait_for(&log, "blocked").await; + let queued = a1.request_raw(Sync()).unwrap(); + a1.child_handle().kill(); + release.send(()).unwrap(); + assert!(blocked.await.is_ok()); + assert!(matches!( + queued.await, + Err(ActorError::ActorStopped(ExitReason::Killed)) + )); + assert_eq!(exit_of(&a1).await, ExitReason::Killed); + assert!(matches!(a1.request(Sync()).await, Err(ActorError::NoProc))); + assert!(!logged(&log).contains(&"stopped".to_string())); + let events = write("kill_queued_request"); + has( + &events, + &[ + "a1.Dispatch", + "a1.Send:queued", + "a1.Signal:Kill", + "a1.Hook:skip", + "a1.Die:Killed", + "a1.Publish", + "a1.Send:noproc", + ], + ); +} + +/// A kill during a running `stopped()` replaces the requested reason; the +/// `stopped()` that began finishes. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn kill_during_stopped() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let (release, gate) = tokio::sync::oneshot::channel(); + let a1 = Node::holding_stopped(&log, gate).start(); + a1.request(Sync()).await.unwrap(); + a1.child_handle().stop(); + wait_for(&log, "stopped").await; + a1.child_handle().kill(); + release.send(()).unwrap(); + assert_eq!(exit_of(&a1).await, ExitReason::Killed); + let events = write("kill_during_stopped"); + has( + &events, + &[ + "a1.Stop:Normal", + "a1.Hook:run", + "a1.Signal:Kill", + "a1.Die:Killed", + "a1.Publish", + ], + ); +} + +/// An untrapped non-Normal exit signal ends the actor with its reason and +/// skips `stopped()`; a Normal one is ignored. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn untrapped_signal_skips_stopped() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::new(&log).start(); + a1.request(Sync()).await.unwrap(); + a1.child_handle().exit(ExitReason::Normal); + a1.request(Sync()).await.unwrap(); + a1.child_handle().exit(ExitReason::Shutdown); + assert_eq!(exit_of(&a1).await, ExitReason::Shutdown); + assert!(!logged(&log).contains(&"stopped".to_string())); + let events = write("untrapped_signal"); + has( + &events, + &[ + "a1.Signal:Normal", + "a1.Dispatch", + "a1.Signal:Shutdown", + "a1.Hook:skip", + "a1.Die:Shutdown", + "a1.Publish", + ], + ); +} + +/// A trapping actor gets exit signals as `Exit` messages, from outside and +/// from another actor, then stops normally through `stopped()`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn trapped_exit_reaches_exit_received() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::trapping(&log).start(); + let a2 = Node::new(&log).start(); + a1.request(Sync()).await.unwrap(); + a2.request(Sync()).await.unwrap(); + a1.child_handle().exit(ExitReason::Shutdown); + wait_for(&log, "exit:Shutdown").await; + a2.context().exit(&a1.child_handle(), ExitReason::Kill); + // The second signal was a kill: untrappable, a1 ends Killed. + assert_eq!(exit_of(&a1).await, ExitReason::Killed); + a2.child_handle().stop(); + assert_eq!(exit_of(&a2).await, ExitReason::Normal); + let events = write("trapped_exit"); + has( + &events, + &[ + "a1.Trap:true", + "a1.Signal:Shutdown", + "a1.Dispatch", + "a1.Signal:Kill", + "a1.Die:Killed", + "a2.Stop:Normal", + "a2.Hook:run", + "a2.Die:Normal", + ], + ); +} + +/// `ctx.exit` to a trapping actor queues an `Exit` that `exit_received` +/// handles; the actor keeps running until stopped. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn trapped_exit_then_stop() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::trapping(&log).start(); + a1.request(Sync()).await.unwrap(); + a1.child_handle().exit(ExitReason::Shutdown); + wait_for(&log, "exit:Shutdown").await; + a1.child_handle().stop(); + assert_eq!(exit_of(&a1).await, ExitReason::Normal); + assert!(logged(&log).contains(&"stopped".to_string())); + let events = write("trapped_exit_then_stop"); + has( + &events, + &[ + "a1.Trap:true", + "a1.Signal:Shutdown", + "a1.Dispatch", + "a1.Stop:Normal", + "a1.Hook:run", + "a1.Die:Normal", + "a1.Publish", + ], + ); +} + +/// Link, try_link and monitor against an actor past D: a trapping caller +/// gets `Exit{NoProc}`, `try_link` returns `Err` and keeps nothing, the +/// monitor delivers `Down{NoProc}`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn late_registrations_get_noproc() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::trapping(&log).start(); + let a2 = Node::new(&Log::default()).start(); + a2.child_handle().stop(); + assert_eq!(exit_of(&a2).await, ExitReason::Normal); + a1.request(Link(a2.child_handle())).await.unwrap(); + assert!(!a1.request(TryLink(a2.child_handle())).await.unwrap()); + a1.request(Monitor(a2.child_handle())).await.unwrap(); + wait_for(&log, "exit:NoProc").await; + wait_for(&log, "down:NoProc").await; + a1.child_handle().stop(); + assert_eq!(exit_of(&a1).await, ExitReason::Normal); + let events = write("late_registrations"); + has( + &events, + &[ + "a2.Die:Normal", + "a2.Publish", + "a1.Link:noproc", + "a1.Deliver:NoProc", + "a1.Link:try", + "a1.Monitor:late", + "a1.Deliver:NoProc", + "a1.Die:Normal", + ], + ); +} + +/// A non-trapping caller of link to an actor past D exits `NoProc` +/// without `stopped()`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn late_link_ends_a_non_trapping_caller() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::new(&log).start(); + let a2 = Node::new(&Log::default()).start(); + a2.child_handle().stop(); + exit_of(&a2).await; + a1.request(Link(a2.child_handle())).await.unwrap(); + assert_eq!(exit_of(&a1).await, ExitReason::NoProc); + assert!(!logged(&log).contains(&"stopped".to_string())); + let events = write("late_link_untrapped"); + has( + &events, + &[ + "a1.Link:noproc", + "a1.Deliver:NoProc", + "a1.Hook:skip", + "a1.Die:NoProc", + ], + ); +} + +/// Unlink between the peer's D and its publication cancels the notice taken +/// at D (E6): no `Exit` reaches the trapping caller. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn unlink_fences_a_notice_taken_at_death() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::trapping(&log).start(); + let (release, gate) = std::sync::mpsc::channel(); + let a2 = Node::holding_drop(gate).start_with_backend(Backend::Thread); + a1.request(Link(a2.child_handle())).await.unwrap(); + a2.request(Sync()).await.unwrap(); + a2.context().stop_with_reason(ExitReason::Shutdown); + // Past D (requests get NoProc), destruction held: not yet published. + past_death(&a2).await; + a1.request(Unlink(a2.child_handle())).await.unwrap(); + release.send(()).unwrap(); + assert_eq!(exit_of(&a2).await, ExitReason::Shutdown); + a1.request(Sync()).await.unwrap(); + assert!( + !logged(&log).iter().any(|e| e.starts_with("exit:")), + "{:?}", + logged(&log) + ); + a1.child_handle().stop(); + exit_of(&a1).await; + let events = write("unlink_fence"); + has( + &events, + &[ + "a1.Link:linked", + "a2.Die:Shutdown", + "a1.Unlink", + "a2.Publish", + "a1.Die:Normal", + ], + ); + assert!(!events.iter().any(|e| e.starts_with("a1.Deliver"))); +} + +/// Demonitor between the target's D and its publication: no `Down`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn demonitor_fences_a_down() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::new(&log).start(); + let (release, gate) = std::sync::mpsc::channel(); + let a2 = Node::holding_drop(gate).start_with_backend(Backend::Thread); + let monitor = a1.request(Monitor(a2.child_handle())).await.unwrap(); + a2.child_handle().stop(); + past_death(&a2).await; + a1.request(Demonitor(monitor)).await.unwrap(); + release.send(()).unwrap(); + exit_of(&a2).await; + spawned_rt::tasks::sleep(Duration::from_millis(20)).await; + a1.request(Sync()).await.unwrap(); + assert!( + !logged(&log).iter().any(|e| e.starts_with("down:")), + "{:?}", + logged(&log) + ); + a1.child_handle().stop(); + exit_of(&a1).await; + let events = write("demonitor_fence"); + has( + &events, + &[ + "a1.Monitor:active", + "a2.Die:Normal", + "a1.Demonitor", + "a2.Publish", + "a1.Die:Normal", + ], + ); + assert!(!events.iter().any(|e| e.starts_with("a1.Deliver"))); +} + +async fn past_death(actor: &ActorRef) { + spawned_rt::tasks::timeout(LIMIT, async { + while !matches!(actor.request(Sync()).await, Err(ActorError::NoProc)) { + spawned_rt::tasks::sleep(Duration::from_millis(2)).await; + } + }) + .await + .expect("never reached its death point"); +} + +/// A monitor of a running actor delivers `Down` with the final reason after +/// publication. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn monitor_delivers_the_final_reason() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::new(&log).start(); + let a2 = Node::new(&Log::default()).start(); + a1.request(Monitor(a2.child_handle())).await.unwrap(); + a2.child_handle().stop_with_reason(ExitReason::Shutdown); + wait_for(&log, "down:Shutdown").await; + a1.child_handle().stop(); + exit_of(&a1).await; + let events = write("monitor_down"); + has( + &events, + &[ + "a1.Monitor:active", + "a2.Stop:Shutdown", + "a2.Die:Shutdown", + "a2.Publish", + "a1.Deliver:Shutdown", + "a1.Dispatch", + "a1.Die:Normal", + ], + ); +} + +/// The `start_linked` parent's exit ends a trapping child normally: a +/// parent exit, `stopped()` runs, and the child ends with the parent's +/// reason (E3). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn parent_exit_ends_a_trapping_child_through_stopped() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::new(&Log::default()).start(); + a1.request(Sync()).await.unwrap(); + let a2 = Node::trapping(&log).start_linked(&a1.context()); + a2.request(Sync()).await.unwrap(); + a1.child_handle().stop_with_reason(ExitReason::Shutdown); + assert_eq!(exit_of(&a2).await, ExitReason::Shutdown); + assert!(logged(&log).contains(&"stopped".to_string())); + assert!(!logged(&log).iter().any(|e| e.starts_with("exit:"))); + let events = write("parent_exit"); + has( + &events, + &[ + "a2.Trap:true", + "a1.Stop:Shutdown", + "a1.Die:Shutdown", + "a1.Publish", + "a2.Deliver:Shutdown", + "a2.Dispatch", + "a2.Hook:run", + "a2.Die:Shutdown", + ], + ); +} + +/// A linked actor's panic ends a non-trapping peer with `Panic`, without +/// `stopped()`; the panicking request is answered at D with `Panic`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn link_notice_ends_a_non_trapping_peer() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::new(&log).start(); + let a2 = Node::new(&Log::default()).start(); + a1.request(Link(a2.child_handle())).await.unwrap(); + assert!(matches!( + a2.request(Boom()).await, + Err(ActorError::ActorStopped(ExitReason::Panic(_))) + )); + assert!(matches!(exit_of(&a1).await, ExitReason::Panic(_))); + assert!(!logged(&log).contains(&"stopped".to_string())); + let events = write("linked_panic"); + has( + &events, + &[ + "a1.Link:linked", + "a2.Dispatch", + "a2.Panic:turn", + "a2.Hook:run", + "a2.Die:Panic", + "a2.Publish", + "a1.Deliver:Panic", + "a1.Hook:skip", + "a1.Die:Panic", + ], + ); +} + +/// The runtime drops a running actor: it ends `Killed`, no `stopped()` (G1). +#[test] +fn runtime_drop_ends_killed() { + let _serial = SERIAL.blocking_lock(); + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let a1 = runtime.block_on(async { + let a1 = Node::new(&log).start(); + a1.request(Sync()).await.unwrap(); + a1 + }); + drop(runtime); + assert_eq!(a1.child_handle().wait_exit_blocking(), ExitReason::Killed); + assert!(!logged(&log).contains(&"stopped".to_string())); + let events = write("runtime_drop"); + has(&events, &["a1.Dispatch", "a1.Drop:Killed", "a1.Publish"]); +} + +/// Threads mode: a trapped exit, then a kill skipping `stopped()`. +#[test] +fn threads_trapped_exit_then_kill() { + use spawned_concurrency::threads; + let _serial = SERIAL.blocking_lock(); + lifecycle_trace::record_lifecycle(); + struct Native(Log); + impl threads::Actor for Native { + fn started(&mut self, ctx: &threads::Context) { + ctx.trap_exit(true); + } + fn stopped(&mut self, _: &threads::Context) { + self.0.lock().unwrap().push("stopped".into()); + } + fn exit_received(&mut self, exit: Exit, _: &threads::Context) { + self.0 + .lock() + .unwrap() + .push(format!("exit:{:?}", exit.reason)); + } + } + struct Ping; + impl Message for Ping { + type Result = (); + } + impl threads::Handler for Native { + fn handle(&mut self, _: Ping, reply: PendingReply<'_, ()>, _: &threads::Context) { + reply.reply(()); + } + } + use threads::ActorStart as _; + let log = Log::default(); + let a1 = Native(log.clone()).start(); + a1.request(Ping).unwrap(); + a1.child_handle().exit(ExitReason::Shutdown); + a1.request(Ping).unwrap(); + assert_eq!(logged(&log), ["exit:Shutdown"]); + a1.child_handle().kill(); + assert_eq!(a1.child_handle().wait_exit_blocking(), ExitReason::Killed); + assert!(!logged(&log).contains(&"stopped".to_string())); + let events = write("threads_trapped_exit_kill"); + has( + &events, + &[ + "a1.Trap:true", + "a1.Signal:Shutdown", + "a1.Dispatch", + "a1.Signal:Kill", + "a1.Hook:skip", + "a1.Die:Killed", + "a1.Publish", + ], + ); +} diff --git a/docs/TESTING.md b/docs/TESTING.md index 10b6f2bd..28b6e10f 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -109,7 +109,7 @@ Status of each layer. "Planned" items are tracked in the | Design | TLA+ and TLC | Holes in lifecycle, restart and shutdown rules | `Supervision`, `Cohorts`, `NestedSupervision` and `Coordinator` models in `verification/tla`, run in CI | | Design, unbounded depth | Apalache inductive invariants | An invariant that a state TLC never reached would break | Safety invariants of all four models proven inductive at fixed sizes, with negative controls; `tla-apalache` CI job | | Design, any size | TLAPS or Veil | An invariant that fails only for larger configurations | Planned ([verification tools](research/verification-tools.md)) | -| Spec and code | Trace validation | The implementation doing something the model forbids | Real OneForOne, cohort and nested traces plus invalid-trace and model-mutation controls | +| Spec and code | Trace validation | The implementation doing something the model forbids | Real OneForOne, cohort, nested, deferred-reply, job and base-lifecycle (exit signals, links, monitors, D, publication) traces plus invalid-trace and model-mutation controls | | Spec and code coverage | Trace coverage (`verification/tla/trace_coverage.py`) | Model actions, states and transitions no test ever reaches | Report on every PR (actions covered by the dedicated trace tests may not regress); whole test suite weekly. See [trace coverage](#trace-coverage) | | Primitives | Loom | Atomic and memory-ordering bugs in cancellation, ready gate and wakeups | In CI; expand to reply slots and mailbox publication versus shutdown | | Whole runtime, deterministic | Simulation backend for `spawned-rt::tasks` (random, PCT, bounded DFS) | Async races, replayed from a seed with virtual time | In review (PRs #212, #214, #218) | @@ -801,6 +801,10 @@ seeded random runs, and TLC checks every recorded trace is a behavior of the model. Hand-edited traces are rejected at the edited event. Trace validation found one model gap (a running child may exit with `Shutdown` while the supervisor is open, via `ChildHandle::shutdown`); no implementation deviation. +`lifecycle_trace` does the same for the base actor lifecycle against +`Lifecycle.tla` (exit signals, stops, links, monitors, requests, `stopped()` +run or skip, the death point, runtime drop and publication, in both modes; +eleven edited traces must be rejected). Apalache checks that the safety invariants of `Supervision`, `Cohorts`, `NestedSupervision` and `Coordinator` are inductive (`Init => IndInv`, @@ -812,7 +816,7 @@ controls must break induction. See [the model README](../verification/tla/README ```bash nix run .#check-tla # model check (TLC via the flake) nix run .#check-apalache -- --negative-controls # inductive invariants (Apalache) -cargo test -p spawned-concurrency --features lifecycle-trace --test supervision_trace +cargo test -p spawned-concurrency --features lifecycle-trace --test supervision_trace --test lifecycle_trace nix run .#check-tla -- --traces target/lifecycle-traces ``` diff --git a/verification/tla/README.md b/verification/tla/README.md index b8f10805..36653e61 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -41,7 +41,7 @@ runs real actors through the scenarios below and writes each trace as JSON; TLC then checks that each trace is a behavior of `Supervision.tla`. ```bash -cargo test -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace +cargo test -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace --test lifecycle_trace nix run .#check-tla -- --traces-only target/lifecycle-traces --negative-controls # or --traces DIR to also model-check nix develop -c python3 verification/tla/trace_check.py target/lifecycle-traces ``` @@ -108,7 +108,7 @@ the recorded runs never exercise? `trace_coverage.py` reports it per model with trace validation (`Supervision`, `Cohorts`, `NestedSupervision`). ```bash -cargo test -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace +cargo test -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace --test lifecycle_trace nix develop -c python3 verification/tla/trace_coverage.py --traces dedicated=target/lifecycle-traces \ --model-keys verification/tla/coverage-model.json.gz --baseline verification/tla/coverage-baseline.json # every test binary: each finished supervisor dumps its traces @@ -460,9 +460,41 @@ message is dispatched after stop; each monitor's `Down` is delivered at most once and, unless its observer died, exactly once; linked non-trapping peers of an abnormal exit stop; and the exit-signal rules of rank 5e. Reintroducing the PR #182 bug (`Fault = "Retain182"`) and the PR #185 bug (`"Dispatch185"`) each -produce a counterexample (`negative_checks.py`). Still to do with the -implementation: lifecycle-trace events for exit signals, links, monitors and -requests so real runs replay against it, as supervision traces do. +produce a counterexample (`negative_checks.py`). + +**Lifecycle trace validation.** With `lifecycle-trace`, the runtime records +one `LifeEvent` per `Lifecycle.tla` action for the actors spawned while +`lifecycle_trace::record_lifecycle()` is on (both `tasks` and `threads`): +`Trap`, `Send` (a request from `Ext`, admitted or `NoProc`), `Dispatch` (with +the mailbox item kind), `Panic`, `Stop`, `Signal` (also a supervisor's +Shutdown or kill), `Link`/`Unlink`/`Monitor`/`Demonitor`, `Deliver` (a link +notice or `Down` applied by its receiver), `Hook` (`stopped()` runs or is +skipped), `Die` (D, final reason), `Drop` (runtime drop) and `Publish`. Each +is recorded in the critical section of the change it describes: the +dispatch with the run loop's stop check, the stop with its cancel, the +admission check with the push, D with the exit-table close. A +recorded actor's sections take the process-wide trace section first, so the +log is a linearization. `TraceLifecycle.tla` replays it with no unlogged +steps; its header lists the mapping (actors `a1`, `a2` in spawn order, +`start_linked` parents as `LinkParent`, every send a request from `Ext`, +folded request panics, the late `NoProc` notice of a non-trapping caller, +and `Hook` "skip" accepted after a kill that arrived before `stopped()` +began, which the model's `Kill` leaves out). Phase 0 (signals held during +`started()`), managed scopes' own stops and actor-to-actor sends are not +mapped: they reject. `concurrency/tests/lifecycle_trace.rs` records 14 +scenarios (`lifecycle_*.json`, schema `lifecycle-v1`): kill with a queued +request answered `Killed` and a later `NoProc`, kill during `stopped()`, +untrapped and `Normal` signals, trapped exits (also from an actor, then a +kill), late link/try_link/monitor (`NoProc`), a late link ending a +non-trapping caller, the unlink and demonitor fences between D and +publication, a monitor's `Down`, a `start_linked` parent exit to a trapping +child, a link notice ending a non-trapping peer, runtime drop (`Killed`), and +one threads-mode run. Controls (`lifecycle_trace_controls`): `stopped()` +after a kill or after an untrapped signal, `Kill` instead of `Killed`, +publication before D, a `Normal` signal stopping an actor, a late `Down` or +link notice with the real reason, a notice after unlink, a `Down` after +demonitor, a parent exit dispatched as an `Exit`, and a runtime drop ending +`Normal`: all eleven are rejected. `trace_coverage.py` skips this schema. Source: [verification strategy](../../legacy/docs/research/verification-strategy.md#acceptance). `Coordinator.tla` checks coordinator failure ownership independently of the actor diff --git a/verification/tla/TraceLifecycle.tla b/verification/tla/TraceLifecycle.tla new file mode 100644 index 00000000..801957da --- /dev/null +++ b/verification/tla/TraceLifecycle.tla @@ -0,0 +1,149 @@ +--------------------------- MODULE TraceLifecycle --------------------------- +(***************************************************************************) +(* Trace validation for Lifecycle.tla (rank 5e exit signals, 5g runtime *) +(* drop; design docs/design/otp-exit-signals.md). *) +(* *) +(* With the `lifecycle-trace` feature, spawned-concurrency records, for *) +(* the actors spawned while recording is on, one event per model action *) +(* (concurrency/src/lifecycle_trace.rs, LifeEvent). Each event is recorded *) +(* inside the critical section of the state change it describes, under *) +(* one process-wide trace section, so the recorded order is a *) +(* linearization: *) +(* Trap ctx.trap_exit, when the flag changes (SetTrap) *) +(* Request, a send or request admitted ("queued") or refused ("noproc") *) +(* Enqueue (one recorded Send; trace_check.py splits it in two) *) +(* Dispatch the run loop takes the mailbox head of kind k, in the same *) +(* step as its stop check (Dispatch); p: the turn of a request *) +(* panicked (the model's panic branch) *) +(* Panic a panic in a turn, started() or stopped() (SelfExit or *) +(* PanicWhileExiting); p: already taken by its Dispatch *) +(* Stop ctx.stop*, ChildHandle::stop* (SelfExit or AdminStop) *) +(* Signal ctx.exit, ChildHandle::exit/kill (ExitSignal), or a *) +(* supervisor's Shutdown or kill (SupShutdown, SupKill) *) +(* Link link/try_link that installed a record or found the peer *) +(* past D (Link, TryLink); Unlink, Monitor, Demonitor *) +(* Deliver a link notice or Down applied by its receiver (Deliver) *) +(* Hook stopped() begins ("run") or is skipped ("skip") *) +(* Die the death point D with the final reason (Die) *) +(* Drop the runtime dropped the actor before D (RuntimeDrop) *) +(* Publish destruction finished; link notices are delivered after it *) +(* and wait_exit returns after them (Publish) *) +(* *) +(* Identities: the recorded actors are a1, a2, ... in spawn order (their *) +(* start_linked parents give LinkParent; the model starts them all, idle, *) +(* and a Spawn event is not replayed). Outside code is Ext; a supervisor's *) +(* signal is Sup and its target is in Children. Every send or request is a *) +(* request from Ext: the model's actor-to-actor Send is not recorded (the *) +(* recorder does not know the sending actor), so MaxSends = 0. *) +(* *) +(* Mapping notes: *) +(* - Handlers are atomic in the model. Dispatch is recorded when the *) +(* message is taken, and a handler's own actions (Stop, Link, Monitor, *) +(* ...) follow it as separate steps of a running actor. A request turn *) +(* that panics is the model's panic branch of Dispatch only when the *) +(* actor did nothing else in between; otherwise its Dispatch is the *) +(* reply branch and the Panic a SelfExit. *) +(* - A parent exit is the model's Dispatch (which begins exit); the stop *) +(* it requests in code is not recorded. *) +(* - A non-trapping caller of link to an actor past D exits NoProc in the *) +(* model's Link step; the code applies the late notice right after, in *) +(* the same section, so that Deliver is a stutter step here. *) +(* - Hook "skip" is also accepted when the model's hook ghost is TRUE but *) +(* a kill arrived before stopped() began: the code then skips stopped() *) +(* (E1), while Lifecycle.tla's Kill during "stopping" leaves hook as it *) +(* was. Hook "run" requires the ghost (SkipHook). *) +(* - A panic after a kill, or a second panic, changes nothing in the *) +(* model's reason classes and is a stutter step. *) +(* - Not in the mapping (an event the model cannot take rejects the *) +(* trace): phase 0 (signals held while started() runs, recorded as *) +(* Held), managed scopes' own stops, actor-to-actor sends, time. *) +(* *) +(* There are no unlogged steps: every model step is a logged event, so *) +(* TLC replays a trace almost deterministically. The trace is accepted iff *) +(* some behavior consumes the whole log (TraceIncomplete violated). *) +(* verification/tla/trace_check.py (schema lifecycle-v1) generates the *) +(* constants and runs TLC. *) +(***************************************************************************) +EXTENDS Lifecycle, TLC + +CONSTANT Trace + +VARIABLE i + +traceVars == <> + +TraceInit == Init /\ i = 0 + +DispatchAs(a, k, p) == + /\ mbox[a] # <<>> /\ Head(mbox[a]).k = k + /\ Dispatch(a) + /\ CASE k = "req" -> req'[Head(mbox[a]).q].s = IF p THEN "exiting" ELSE "ok" + \* exit_received's own stop, if any, is a later Stop event + [] k = "exit" -> UNCHANGED life + [] OTHER -> TRUE + +PanicAs(a, folded) == + IF folded THEN st[a] = "stopping" /\ why[a] = "Panic" /\ UNCHANGED vars + ELSE IF st[a] = "run" THEN SelfExit(a, "Panic") + ELSE IF st[a] = "stopping" /\ (killed[a] \/ why[a] = "Panic") THEN UNCHANGED vars + ELSE PanicWhileExiting(a) + +SignalAs(e) == + IF e.s = Sup + THEN IF e.x = "Kill" THEN SupKill(e.t) ELSE e.x = "Shutdown" /\ SupShutdown(e.t) + ELSE ExitSignal(e.s, e.t, e.x) + +LinkAs(e) == + CASE e.x = "try" -> TryLink(e.t, e.s) + [] e.x = "linked" -> ~IsGone(e.s) /\ Link(e.t, e.s) + [] e.x = "noproc" -> IsGone(e.s) /\ Link(e.t, e.s) + [] OTHER -> FALSE + +Matches(n, e) == + /\ n.to = e.t /\ n.from = e.s /\ n.k = e.k /\ n.r = e.x /\ n.o = e.o + /\ (e.k = "down" => n.m = e.m) + +DeliverAs(e) == + \/ \E n \in inflight : Matches(n, e) /\ Deliver(n) + \* The late NoProc notice of a non-trapping caller: applied by Link. + \/ /\ e.k = "link" /\ e.o = "late" /\ e.x = "NoProc" + /\ ~\E n \in inflight : Matches(n, e) + /\ ~trap[e.t] /\ st[e.t] = "stopping" /\ cause[e.t] = "noproc" + /\ UNCHANGED vars + +HookAs(a, x) == + /\ st[a] = "stopping" + /\ IF x = "run" THEN hook[a] ELSE ~hook[a] \/ killed[a] + /\ UNCHANGED vars + +\* The model action named by logged event e, constrained by its parameters. +Step(e) == + CASE e.a = "Trap" -> SetTrap(e.t, e.x = "true") + [] e.a = "Request" -> Request(e.q, e.t) + [] e.a = "Enqueue" -> Enqueue(e.q) /\ req'[e.q].s = e.x + [] e.a = "Dispatch" -> DispatchAs(e.t, e.k, e.p) + [] e.a = "Panic" -> PanicAs(e.t, e.p) + [] e.a = "Stop" -> SelfExit(e.t, e.x) \/ AdminStop(e.t, e.x) + [] e.a = "Signal" -> SignalAs(e) + [] e.a = "Link" -> LinkAs(e) + [] e.a = "Unlink" -> Unlink(e.t, e.s) + [] e.a = "Monitor" -> Monitor(e.t, e.s, e.m) /\ mstate'[e.m] = e.x + [] e.a = "Demonitor" -> Demonitor(e.t, e.m) + [] e.a = "Deliver" -> DeliverAs(e) + [] e.a = "Hook" -> HookAs(e.t, e.x) + [] e.a = "Die" -> why[e.t] = e.x /\ Die(e.t) + [] e.a = "Drop" -> RuntimeDrop(e.t) /\ why'[e.t] = e.x + [] e.a = "Publish" -> Publish(e.t) + [] OTHER -> FALSE + +TraceNext == + /\ i < Len(Trace) + /\ Step(Trace[i + 1]) + /\ i' = i + 1 + +TraceSpec == TraceInit /\ [][TraceNext]_traceVars + +\* Violated exactly when the whole log has been matched: TLC reporting this +\* invariant as violated means the trace is ACCEPTED. +TraceIncomplete == i < Len(Trace) +============================================================================= diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 16293aa7..6fce2b1d 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -291,6 +291,88 @@ def stopped_before_death(events): "exit error answered Stopped before the death point", tlc) +def lifecycle_trace_controls(traces, tlc): + """Edits of recorded base-lifecycle traces that Lifecycle.tla must + reject: each reintroduces a v0.7 behaviour or breaks a rank 5e/5g rule.""" + by_name = {t["name"]: t for t in traces if t.get("schema") == "lifecycle-v1"} + + def spawned(events, n): + return [e["t"] for e in events if e["a"] == "Spawn"][n - 1] + + def index(events, action, x=None, after=0): + return next(i for i, e in enumerate(events) + if i >= after and e["a"] == action and (x is None or e["x"] == x)) + + def hook_after_kill(events): + events[index(events, "Hook", "skip")]["x"] = "run" + require_rejected(by_name["lifecycle_kill_queued_request"], hook_after_kill, + "stopped() runs after a kill", tlc) + + def killed_as_kill(events): + events[index(events, "Die", "Killed")]["x"] = "Kill" + require_rejected(by_name["lifecycle_kill_queued_request"], killed_as_kill, + "a kill ends Kill instead of Killed (v0.7)", tlc) + + def publish_before_death(events): + events.insert(index(events, "Die"), events.pop(index(events, "Publish"))) + require_rejected(by_name["lifecycle_kill_queued_request"], publish_before_death, + "publication before the death point", tlc) + + def untrapped_runs_hook(events): + events[index(events, "Hook", "skip")]["x"] = "run" + require_rejected(by_name["lifecycle_untrapped_signal"], untrapped_runs_hook, + "stopped() runs after an untrapped exit signal (v0.7)", tlc) + + def normal_signal_stops(events): + # v0.7: a Normal signal stopped a non-trapping actor. + at = index(events, "Signal", "Normal") + events[at + 1:at + 1] = [ + {"a": "Hook", "t": events[at]["t"], "s": 0, "x": "skip", "m": 0, "k": "", "o": ""}] + require_rejected(by_name["lifecycle_untrapped_signal"], normal_signal_stops, + "a Normal exit signal stops a non-trapping actor (v0.7)", tlc) + + def down_with_real_reason(events): + down = next(e for e in events if e["a"] == "Deliver" and e["k"] == "down") + down["x"] = "Normal" + require_rejected(by_name["lifecycle_late_registrations"], down_with_real_reason, + "late monitor's Down carries the real reason, not NoProc (v0.7)", tlc) + + def late_link_with_real_reason(events): + notice = next(e for e in events if e["a"] == "Deliver" and e["k"] == "link") + notice["x"] = "Normal" + require_rejected(by_name["lifecycle_late_registrations"], late_link_with_real_reason, + "late link's notice carries the real reason, not NoProc (v0.7)", tlc) + + def notice_after_unlink(events): + a1, a2 = spawned(events, 1), spawned(events, 2) + at = next(i for i, e in enumerate(events) if e["a"] == "Publish" and e["t"] == a2) + events.insert(at + 1, {"a": "Deliver", "t": a1, "s": a2, "x": "Shutdown", + "m": 0, "k": "link", "o": "death"}) + require_rejected(by_name["lifecycle_unlink_fence"], notice_after_unlink, + "death notice delivered after unlink (v0.7 fence)", tlc) + + def down_after_demonitor(events): + a1, a2 = spawned(events, 1), spawned(events, 2) + m = next(e["m"] for e in events if e["a"] == "Monitor") + at = next(i for i, e in enumerate(events) if e["a"] == "Publish" and e["t"] == a2) + events.insert(at + 1, {"a": "Deliver", "t": a1, "s": a2, "x": "Normal", + "m": m, "k": "down", "o": "death"}) + require_rejected(by_name["lifecycle_demonitor_fence"], down_after_demonitor, + "Down delivered after demonitor (v0.7 fence)", tlc) + + def parent_exit_as_exit(events): + a2 = spawned(events, 2) + next(e for e in events if e["a"] == "Dispatch" and e["t"] == a2 + and e["k"] == "pexit")["k"] = "exit" + require_rejected(by_name["lifecycle_parent_exit"], parent_exit_as_exit, + "parent Shutdown reaches exit_received", tlc) + + def drop_keeps_normal(events): + events[index(events, "Drop")]["x"] = "Normal" + require_rejected(by_name["lifecycle_runtime_drop"], drop_keeps_normal, + "runtime drop ends Normal instead of Killed (v0.7)", tlc) + + def job_trace_controls(traces, tlc): """Edits of recorded pipe-to-self job traces that DeferredJobs.tla must reject: each erases or reorders what the job core enforces.""" @@ -409,6 +491,7 @@ def decide_at_once(events): named_fault(module, fault, invariant, args.tlc) job_trace_controls(traces, args.tlc) reply_trace_controls(traces, args.tlc) + lifecycle_trace_controls(traces, args.tlc) for module, fault, invariant in LIFECYCLE_CONTROLS: named_fault(module, fault, invariant, args.tlc) for fault, prop in LIFECYCLE_TEMPORAL_CONTROLS: diff --git a/verification/tla/trace_check.py b/verification/tla/trace_check.py index b3e3bb0c..5c36e389 100644 --- a/verification/tla/trace_check.py +++ b/verification/tla/trace_check.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Check recorded lifecycle traces against Supervision.tla (and, by schema, -Cohorts.tla, NestedSupervision.tla, DeferredJobs.tla and DeferredReplies.tla). +Cohorts.tla, NestedSupervision.tla, DeferredJobs.tla, DeferredReplies.tla and +Lifecycle.tla). Each JSON trace (written by concurrency/tests/supervision_trace.rs with the `lifecycle-trace` feature) looks like: @@ -48,7 +49,8 @@ def tla_event(event, cohort=False): def generate(trace, workdir): schema = trace.get("schema", "one-for-one-v1") - if schema not in {"one-for-one-v1", "cohort-v1", "tree-v1", "jobs-v1", "replies-v1"}: + if schema not in {"one-for-one-v1", "cohort-v1", "tree-v1", "jobs-v1", "replies-v1", + "lifecycle-v1"}: raise ValueError(f"unsupported trace schema: {schema}") if schema == "tree-v1": return generate_tree(trace, workdir) @@ -56,6 +58,8 @@ def generate(trace, workdir): return generate_jobs(trace, workdir) if schema == "replies-v1": return generate_replies(trace, workdir) + if schema == "lifecycle-v1": + return generate_lifecycle(trace, workdir) cohort = schema == "cohort-v1" modules = ("Cohorts.tla", "TraceCohorts.tla") if cohort else MODELS for model in modules: @@ -142,6 +146,128 @@ def generate_replies(trace, workdir): "INVARIANT Safety\nINVARIANT TraceIncomplete\n") +OWN_ACTIONS = {"Trap", "Stop", "Link", "Unlink", "Monitor", "Demonitor"} + + +def lifecycle_records(trace): + """Normalize a lifecycle-v1 trace: actor and monitor numbers, Send split + into Request and Enqueue, and request turns that panicked folded into + their Dispatch (TraceLifecycle.tla). Returns (records, constants).""" + events = trace["events"] + actors, parents = {}, {} + for e in events: + if e["a"] == "Spawn": + actors[int(e["t"])] = f"a{len(actors) + 1}" + if int(e["s"]): + parents[int(e["t"])] = int(e["s"]) + if not actors: + raise ValueError("a lifecycle trace needs at least one spawned actor") + for child, parent in parents.items(): + if parent not in actors: + raise ValueError(f"start_linked parent {parent} is not recorded") + + def actor(raw): + if int(raw) not in actors: + raise ValueError(f"event for an unrecorded actor {raw}") + return actors[int(raw)] + + # Request turns that panicked with nothing of the actor's own in between. + folded = set() + turn = {} + for i, e in enumerate(events): + t = int(e["t"]) + if e["a"] == "Dispatch": + turn[t] = i if e["k"] == "req" else None + elif e["a"] in OWN_ACTIONS: + turn[t] = None + elif e["a"] == "Panic" and e["x"] == "turn" and turn.get(t) is not None: + folded |= {turn[t], i} + turn[t] = None + + monitors, requests, children = {}, 0, set() + signals = late = 0 + records = [] + + def record(a, t="none", s="none", x="", m=0, q=0, k="", o="", p=False): + records.append("[a |-> {}, t |-> {}, s |-> {}, x |-> {}, m |-> {}, q |-> {}, k |-> {}, o |-> {}, p |-> {}]".format( + tla_string(a), tla_string(t), tla_string(s), tla_string(x), m, q, + tla_string(k), tla_string(o), "TRUE" if p else "FALSE")) + + for i, e in enumerate(events): + a = e["a"] + if a == "Spawn": + continue + t = actor(e["t"]) + if a == "Send": + requests += 1 + record("Request", t=t, q=requests) + record("Enqueue", t=t, q=requests, x=e["x"]) + continue + m = 0 + if int(e.get("m", 0)): + m = monitors.setdefault(int(e["m"]), len(monitors) + 1) + if a == "Signal": + if e["o"] == "sup": + source = "sup" + children.add(t) + else: + source = actors.get(int(e["s"]), "ext") + signals += 1 + record(a, t=t, s=source, x=e["x"]) + continue + if a == "Stop": + signals += 1 + if a in {"Link", "Monitor"} and e["x"] in {"noproc", "try", "late"}: + late += 1 + source = actor(e["s"]) if a in {"Link", "Unlink", "Monitor", "Deliver"} else "none" + record(a, t=t, s=source, x=e["x"], m=m, k=e["k"], o=e["o"], p=i in folded) + constants = { + "actors": [actors[k] for k in actors], + "parents": {actors[c]: actors[p] for c, p in parents.items()}, + "children": sorted(children), + "requests": requests, + "monitors": len(monitors), + "signals": signals, + "late": late, + } + return records, constants + + +def generate_lifecycle(trace, workdir): + """A group of actors' exit signals, links, monitors and requests against + Lifecycle.tla.""" + for model in ("Lifecycle.tla", "TraceLifecycle.tla"): + shutil.copy(os.path.join(HERE, model), workdir) + records, c = lifecycle_records(trace) + + def tla_set(values): + return "{" + ", ".join(tla_string(v) for v in values) + "}" + if c["parents"]: + parent = "CASE " + " [] ".join(f"a = {tla_string(ch)} -> {tla_string(p)}" + for ch, p in c["parents"].items()) + ' [] OTHER -> "none"' + else: + parent = '"none"' + with open(os.path.join(workdir, "TraceRun.tla"), "w") as out: + out.write("---- MODULE TraceRun ----\nEXTENDS TraceLifecycle\n" + f"TraceActors == {tla_set(c['actors'])}\n" + f"TraceChildren == {tla_set(c['children'])}\n" + f"TraceLinkParent == [a \\in TraceActors |-> {parent}]\n" + f"TraceReqs == 1..{c['requests']}\n" + f"TraceMonIds == 1..{c['monitors']}\n" + "TraceLog == <<\n " + ",\n ".join(records) + "\n>>\n====\n") + with open(os.path.join(workdir, "TraceRun.cfg"), "w") as out: + out.write("INIT TraceInit\nNEXT TraceNext\nCONSTANTS\n" + " Actors <- TraceActors\n Children <- TraceChildren\n" + " LinkParent <- TraceLinkParent\n Reqs <- TraceReqs\n" + " MonIds <- TraceMonIds\n MaxSends = 0\n" + f" MaxSignals = {c['signals']}\n MaxLate = {c['late']}\n" + " UnlinkEnabled = TRUE\n TrapCanClear = TRUE\n" + " SelfReasons <- Reasons\n SignalReasons <- Reasons\n" + " AdminReasons <- Reasons\n DropEnabled = TRUE\n" + " Fault = \"None\"\n Trace <- TraceLog\n" + "INVARIANT Safety\nINVARIANT TraceIncomplete\n") + + def generate_jobs(trace, workdir): """One owner instance's pipe-to-self jobs against DeferredJobs.tla.""" for model in ("DeferredJobs.tla", "TraceJobs.tla"): diff --git a/verification/tla/trace_coverage.py b/verification/tla/trace_coverage.py index 20e00698..5b2b814d 100644 --- a/verification/tla/trace_coverage.py +++ b/verification/tla/trace_coverage.py @@ -261,8 +261,8 @@ def trace_keys(path, tlc): # Trace schemas validated by trace_check.py but without a coverage model -# (the deferred-replies models): skipped here. -UNCOVERED_SCHEMAS = {"jobs-v1", "replies-v1"} +# (the deferred-replies models and Lifecycle.tla): skipped here. +UNCOVERED_SCHEMAS = {"jobs-v1", "replies-v1", "lifecycle-v1"} def covered(path): From 0f844b776024aa85c66c54ce99a57eed1bbd584a Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 02:50:13 -0300 Subject: [PATCH 089/208] feat(reply): client-port API refinements for rank 11 From porting ethlambda and ethrex to the 0.8 stack: - #[actor(max_outstanding = N, max_native_jobs = N)], and per-ChildSpec max_outstanding/max_native_jobs overrides (the design's per-spec bound) - tasks-mode blocking jobs on Tokio's blocking pool (spawn_blocking), still under MAX_NATIVE_JOBS; threads mode keeps owned threads - reserve-then-spawn permits (ctx.reserve_job / reserve_blocking_job): a refusal keeps what would move into work/done; spawning cannot fail - JobCtx::emit_at / emit_at_blocking: delayed, fenced, credit-accounted emits; the terminal message is held until they fire (Loom model) - JobOptions::deadline_after; spawned_concurrency::Instant re-export - #[send_handler] on a manual impl Handler: no reply parameter, answers () when handle returns - spawned_test::{tasks,threads}::in_actor - JobOutcome, JobError, PipeErrorKind are #[non_exhaustive] --- concurrency/src/lib.rs | 8 +- concurrency/src/reply/job_core.rs | 123 ++++- concurrency/src/reply/jobs.rs | 200 +++++++- concurrency/src/supervision.rs | 15 + concurrency/src/system.rs | 10 + concurrency/src/tasks/actor.rs | 62 ++- concurrency/src/tasks/jobs.rs | 475 +++++++++++++----- concurrency/src/tasks/mod.rs | 3 +- concurrency/src/threads/actor.rs | 37 +- concurrency/src/threads/jobs.rs | 311 ++++++++---- concurrency/src/threads/mod.rs | 3 +- concurrency/tests/deferred_api.rs | 380 ++++++++++++++ concurrency/tests/deferred_jobs.rs | 17 +- concurrency/tests/job_core_loom.rs | 79 +++ docs/GUIDE.md | 134 ++++- docs/TESTING.md | 4 +- .../capabilities/src/bin/deferred_replies.rs | 10 +- macros/src/lib.rs | 132 ++++- testing/src/tasks.rs | 33 ++ testing/src/threads.rs | 23 + testing/tests/tasks_kit.rs | 28 ++ testing/tests/threads_kit.rs | 27 + 22 files changed, 1852 insertions(+), 262 deletions(-) create mode 100644 concurrency/tests/deferred_api.rs diff --git a/concurrency/src/lib.rs b/concurrency/src/lib.rs index 4f6a0752..f020d533 100644 --- a/concurrency/src/lib.rs +++ b/concurrency/src/lib.rs @@ -115,7 +115,13 @@ pub use reply::{ ReplyTo, }; pub use response::Response; -pub use spawned_macros::{actor, protocol}; +pub use spawned_macros::{actor, protocol, send_handler}; +/// The instant type of reply and job deadlines (`Deadline::At`, +/// `JobOptions::deadline`, `ReplyTo::deadline`, `JobCtx::emit_at`): +/// `std::time::Instant`, or the simulated clock's instant under +/// `--cfg spawned_sim`. Re-exported so callers need not depend on +/// `spawned-rt`. +pub use spawned_rt::time::Instant; #[cfg(all(kani, feature = "async"))] mod kani_proofs; diff --git a/concurrency/src/reply/job_core.rs b/concurrency/src/reply/job_core.rs index eb6dbe3c..6ca487c8 100644 --- a/concurrency/src/reply/job_core.rs +++ b/concurrency/src/reply/job_core.rs @@ -30,17 +30,43 @@ pub(crate) enum Notification { Suppressed, } +/// A terminal notification held back until the job's scheduled emits have +/// fired (`JobCtx::emit_at`); run outside the lock. +pub(crate) type HeldTerminal = Box; + struct Inner { cancelled: bool, notification: Notification, running: bool, emit_credits: u16, + /// Delayed emits (`emit_at`) holding a credit, not yet fired. + scheduled: u16, + /// The work's result, waiting for `scheduled` to reach 0 so the terminal + /// message stays last. + held: Option, } impl Inner { + /// The terminal notification can still be taken. fn open(&self) -> bool { !self.cancelled && self.notification == Notification::Pending } + /// New emits are accepted: open, and the work has not handed in its + /// result (a held terminal). + fn accepting(&self) -> bool { + self.open() && self.held.is_none() + } +} + +/// What [`JobCore::finish`] did with the work's result. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Finish { + /// Notified now (the enqueue ran). + Taken, + /// Held until the scheduled emits fire. + Held, + /// Cancelled or already notified (deadline): nothing to deliver. + Refused, } fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { @@ -62,6 +88,8 @@ impl JobCore { notification: Notification::Pending, running: true, emit_credits, + scheduled: 0, + held: None, }), emitters: Condvar::new(), cancelled_fast: AtomicBool::new(false), @@ -76,7 +104,7 @@ impl JobCore { /// wakes blocked emitters. Never undoes effects. Returns whether this /// call set the flag. The caller wakes async emitters afterwards. pub(crate) fn cancel(&self) -> bool { - let newly = { + let (newly, held) = { let mut inner = lock(&self.inner); let newly = !inner.cancelled; inner.cancelled = true; @@ -86,8 +114,10 @@ impl JobCore { // Set under the lock, so a dispatch that sees an enqueued // message's job as not cancelled ran before this cancel. self.cancelled_fast.store(true, Ordering::Release); - newly + (newly, inner.held.take()) }; + // A held result is suppressed; it drops outside the lock. + drop(held); self.emitters.notify_all(); newly } @@ -99,18 +129,47 @@ impl JobCore { /// caller after the lock is released. When taken, the caller wakes async /// emitters. pub(crate) fn notify(&self, enqueue: impl FnOnce() -> Option) -> (bool, Option) { - let unsent = { + let (unsent, held) = { let mut inner = lock(&self.inner); if !inner.open() { return (false, None); } inner.notification = Notification::Notified; - enqueue() + // A deadline overtaking a held result: the result is dropped. + (enqueue(), inner.held.take()) }; + drop(held); self.emitters.notify_all(); (true, unsent) } + /// The work's result: take the notification as [`notify`](Self::notify) + /// does, unless scheduled emits are still pending; then store `later` + /// (which calls `notify` again) for the last of them to run, so the + /// terminal message stays last. + pub(crate) fn finish( + &self, + enqueue: impl FnOnce() -> Option, + later: impl FnOnce() -> HeldTerminal, + ) -> (Finish, Option) { + { + let mut inner = lock(&self.inner); + if !inner.open() { + return (Finish::Refused, None); + } + if inner.scheduled > 0 { + inner.held = Some(later()); + return (Finish::Held, None); + } + } + // Nothing scheduled, and only the work schedules: `notify` takes it + // unless a cancel or the deadline came in between. + match self.notify(enqueue) { + (true, unsent) => (Finish::Taken, unsent), + (false, _) => (Finish::Refused, None), + } + } + /// Whether the terminal notification can still be taken (the job /// deadline is still due). pub(crate) fn is_pending(&self) -> bool { @@ -141,7 +200,7 @@ impl JobCore { pub(crate) fn acquire_blocking(&self) -> Result<(), JobGone> { let mut inner = lock(&self.inner); loop { - if !inner.open() { + if !inner.accepting() { return Err(JobGone); } if inner.emit_credits > 0 { @@ -159,7 +218,7 @@ impl JobCore { #[cfg_attr(not(feature = "async"), allow(dead_code))] pub(crate) fn try_acquire(&self, register: impl FnOnce()) -> Option> { let mut inner = lock(&self.inner); - if !inner.open() { + if !inner.accepting() { return Some(Err(JobGone)); } if inner.emit_credits > 0 { @@ -179,7 +238,7 @@ impl JobCore { enqueue: impl FnOnce() -> Result<(), X>, ) -> Result<(), (JobGone, Option)> { let mut inner = lock(&self.inner); - if !inner.open() { + if !inner.accepting() { inner.emit_credits += 1; drop(inner); self.emitters.notify_all(); @@ -188,6 +247,49 @@ impl JobCore { enqueue().map_err(|unsent| (JobGone, Some(unsent))) } + /// Delayed emit, step 2 at the call: with a credit taken, register a + /// scheduled emit unless the job is cancelled, notified or finished. + /// The caller's credit guard returns the credit on failure. + pub(crate) fn schedule(&self) -> Result<(), JobGone> { + let mut inner = lock(&self.inner); + if !inner.accepting() { + return Err(JobGone); + } + inner.scheduled += 1; + Ok(()) + } + + /// Delayed emit, step 2 at its deadline, or when dropped unfired + /// (`enqueue` is `None`): under the job's lock, enqueue if the + /// notification is still pending. A message not enqueued returns its + /// credit through its guard, after the lock. The last scheduled emit + /// hands back a held terminal, to run outside the lock. + pub(crate) fn fire( + &self, + enqueue: Option Result<(), X>>, + ) -> (Result<(), (JobGone, Option)>, Option) { + let mut inner = lock(&self.inner); + debug_assert!(inner.scheduled > 0, "scheduled emit underflow"); + inner.scheduled = inner.scheduled.saturating_sub(1); + let result = match enqueue { + Some(enqueue) if inner.open() => enqueue().map_err(|unsent| (JobGone, Some(unsent))), + _ => Err((JobGone, None)), + }; + let held = if inner.scheduled == 0 { + inner.held.take() + } else { + None + }; + drop(inner); + self.emitters.notify_all(); + (result, held) + } + + #[allow(dead_code)] // used by the Loom model + pub(crate) fn scheduled(&self) -> u16 { + lock(&self.inner).scheduled + } + /// Emit step 3: a credit comes back when its message is dispatched, /// fenced or dropped with the mailbox. The caller wakes async emitters. pub(crate) fn return_emit_credit(&self) { @@ -201,6 +303,11 @@ pub(crate) trait Job { fn cancel(&self); fn is_running(&self) -> bool; fn finished(&self); + /// Kept in the job list at compaction (so `cancel_all` reaches it): + /// running, or with work left such as scheduled emits. + fn is_live(&self) -> bool { + self.is_running() + } } /// Compact the weak job list when it reaches this length (and every power of @@ -263,7 +370,7 @@ impl JobSetCore { if jobs.len() >= COMPACT_AT && jobs.len().is_power_of_two() { jobs.retain(|job| match job.upgrade() { Some(job) => { - let keep = job.is_running(); + let keep = job.is_live(); upgraded.push(job); keep } diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 5385a8a9..39063eff 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -34,6 +34,28 @@ pub fn default_max_native_jobs() -> usize { .min(4) } +/// Per-instance overrides of `Actor::MAX_OUTSTANDING` and +/// `Actor::MAX_NATIVE_JOBS` (set per `ChildSpec`); `None` keeps the actor's +/// constant. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct Bounds { + pub(crate) max_outstanding: Option, + pub(crate) max_native_jobs: Option, +} + +impl Bounds { + /// The effective `(max_outstanding, max_native_jobs)` for an actor whose + /// constants are `outstanding` and `native`. + pub(crate) fn resolve(self, outstanding: usize, native: Option) -> (usize, usize) { + ( + self.max_outstanding.unwrap_or(outstanding), + self.max_native_jobs + .or(native) + .unwrap_or_else(default_max_native_jobs), + ) + } +} + /// A key for cancel-previous jobs (`pipe_keyed`, `JobOptions::key`). #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct JobKey(std::borrow::Cow<'static, str>); @@ -69,6 +91,22 @@ pub struct JobOptions { pub emit_credits: u16, } +impl JobOptions { + /// Set [`deadline`](Self::deadline) to `timeout` from now, on the + /// runtime's clock (the simulated one under `spawned_sim`): + /// `JobOptions::default().deadline_after(Duration::from_secs(2))`. + /// + /// The deadline is hard: when it passes, `done` receives + /// `JobError::DeadlineExceeded`, the job's later emits fail with + /// [`JobGone`], and its eventual result is dropped. For a soft deadline + /// (stop starting new work, finish what is in hand), check the time + /// inside the work instead. + pub fn deadline_after(mut self, timeout: std::time::Duration) -> Self { + self.deadline = Some(spawned_rt::time::Instant::now() + timeout); + self + } +} + impl Default for JobOptions { fn default() -> Self { Self { @@ -82,6 +120,7 @@ impl Default for JobOptions { /// How a job ended, delivered to `done` in the owner's turn. #[derive(Debug)] +#[non_exhaustive] pub enum JobOutcome { Ok(T), Err(JobError), @@ -98,17 +137,21 @@ impl JobOutcome { /// Why a job produced no value. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] pub enum JobError { /// The work panicked; caught at the job boundary. #[error("job panicked: {0}")] Panicked(String), - /// `JobOptions::deadline` passed first. The work may still be running. + /// `JobOptions::deadline` passed first. The deadline is hard: the job's + /// later emits fail and its eventual result is dropped. The work may + /// still be running (cooperative cancellation). #[error("job deadline exceeded")] DeadlineExceeded, } /// Why a job was not spawned. #[derive(Debug, thiserror::Error)] +#[non_exhaustive] pub enum PipeErrorKind { /// The owner's `MAX_OUTSTANDING` credits are in use. #[error("deferred credits at the bound")] @@ -154,7 +197,7 @@ impl PipeReplyError { } pub use super::job_core::JobGone; -use super::job_core::{Job, JobCore, JobSetCore}; +use super::job_core::{Finish, HeldTerminal, Job, JobCore, JobSetCore}; use crate::lifecycle_trace as trace; /// One job's shared state: the core protocol plus the async wake-ups. @@ -170,6 +213,9 @@ pub(crate) struct JobState { /// Wakers of async emitters waiting for a credit. #[cfg(feature = "async")] emit_wakers: Mutex>, + /// Delayed emits (`JobCtx::emit_at`) not yet fired: their strong + /// references (the deadline heap holds `Weak`s). Cleared at cancellation. + scheduled: Mutex>>, } fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { @@ -187,6 +233,7 @@ impl JobState { cancel: spawned_rt::tasks::CancellationToken::new(), #[cfg(feature = "async")] emit_wakers: Mutex::new(Vec::new()), + scheduled: Mutex::new(Vec::new()), }) } @@ -220,6 +267,12 @@ impl JobState { self.wake_cancelled(); } + /// A permit used after the owner began exiting: the job never runs. + pub(crate) fn never_started(&self) { + self.core.cancel(); + self.core.finished(); + } + fn cancel_quietly(&self) { if self.core.cancel() { self.count_cancel(); @@ -235,6 +288,12 @@ impl JobState { } fn wake_cancelled(&self) { + // Scheduled emits will never be delivered: drop them now (their + // messages and credits), outside every lock. + let scheduled = std::mem::take(&mut *lock(&self.scheduled)); + for emit in scheduled { + emit.discard(self); + } #[cfg(feature = "async")] { self.cancel.cancel(); @@ -281,6 +340,79 @@ impl JobState { (taken, unsent) } + /// The work's result (see [`JobCore::finish`]): taken now, or held + /// until the job's scheduled emits have fired (`later` then runs + /// `notify`). Returns the refused envelope, to drop outside locks. + pub(crate) fn finish( + &self, + end: &'static str, + enqueue: impl FnOnce() -> Option, + later: impl FnOnce() -> HeldTerminal, + ) -> Option { + let mut enqueue = Some(enqueue); + let (finish, unsent) = trace::atomic(|| { + let (finish, unsent) = self + .core + .finish(|| enqueue.take().and_then(|enqueue| enqueue()), later); + match finish { + Finish::Taken => { + crate::instrumentation::hooks::job_notified(end); + self.record(|job| trace::JobEvent { + x: "notified", + ..trace::JobEvent::new("Finish", job) + }); + } + Finish::Refused => self.record(|job| trace::JobEvent { + x: "quiet", + ..trace::JobEvent::new("Finish", job) + }), + Finish::Held => {} + } + (finish, unsent) + }); + #[cfg(feature = "async")] + if finish == Finish::Taken { + self.wake_emitters(); + } + #[cfg(not(feature = "async"))] + let _ = finish; + unsent + } + + /// Register a delayed emit at `at` (ticks): `send` enqueues its envelope + /// when the deadline fires, fenced like any emit, and the job's terminal + /// message waits for it. `track` arms the owner's deadline heap. On + /// `Err` the caller's credit guard (inside `send`) returns the credit. + pub(crate) fn schedule_emit( + self: &Arc, + at: u64, + send: SendScheduled, + track: impl FnOnce(u64, std::sync::Weak), + ) -> Result<(), JobGone> { + self.core.schedule()?; + let emit = Arc::new(ScheduledEmit { + at, + job: Arc::downgrade(self), + send: Mutex::new(Some(send)), + }); + lock(&self.scheduled).push(emit.clone()); + if self.is_cancelled() { + // Cancelled between the check and the registration: the cancel + // may have missed it. + for emit in std::mem::take(&mut *lock(&self.scheduled)) { + emit.discard(self); + } + return Ok(()); + } + if at <= super::now_ticks() { + emit.expire(at); + } else { + let weak: std::sync::Weak = Arc::downgrade(&emit) as _; + track(at, weak); + } + Ok(()) + } + pub(crate) fn is_running(&self) -> bool { self.core.is_running() } @@ -367,6 +499,10 @@ impl Job for JobState { fn is_running(&self) -> bool { self.core.is_running() } + fn is_live(&self) -> bool { + // Scheduled emits keep a finished job reachable for `cancel_all`. + self.core.is_running() || self.core.scheduled() > 0 + } fn finished(&self) { self.core.finished(); } @@ -584,6 +720,66 @@ impl Expirable for JobDeadline { } } +/// Enqueues a delayed emit's envelope; a refused envelope comes back erased, +/// to be dropped outside locks. +pub(crate) type SendScheduled = Box Result<(), Box> + Send>; + +/// One delayed emit (`JobCtx::emit_at`), serviced by the owner's deadline +/// heap. Holds its message and emit credit until it fires or is discarded. +pub(crate) struct ScheduledEmit { + at: u64, + job: std::sync::Weak, + send: Mutex>, +} + +impl ScheduledEmit { + /// Fire (`deliver`) or discard: step 2 of the emit protocol under the + /// job's lock, then remove it from the job, then run a held terminal. + fn settle(&self, job: &JobState, deliver: bool) { + let Some(send) = lock(&self.send).take() else { + return; + }; + // A discarded message (and its credit guard) drops at the end, + // outside the trace section and every lock. + let (deliver, _discarded) = if deliver { + (Some(send), None) + } else { + (None, Some(send)) + }; + let (result, held) = trace::atomic(|| { + let (result, held) = job.core.fire(deliver); + if result.is_ok() { + crate::instrumentation::hooks::job_emitted(); + job.record(|id| trace::JobEvent::new("Emit", id)); + } + (result, held) + }); + // A refused envelope (and its credit guard) drops here, unlocked. + drop(result); + lock(&job.scheduled).retain(|emit| !std::ptr::eq(Arc::as_ptr(emit), self)); + if let Some(held) = held { + held(); + } + #[cfg(feature = "async")] + job.wake_emitters(); + } + + fn discard(&self, job: &JobState) { + self.settle(job, false); + } +} + +impl Expirable for ScheduledEmit { + fn pending_deadline(&self) -> Option { + lock(&self.send).is_some().then_some(self.at) + } + fn expire(&self, _now: u64) { + if let Some(job) = self.job.upgrade() { + self.settle(&job, true); + } + } +} + /// Threads-mode cleanup completion (death plus the bounded job join), for /// `join()`. Set when the guard drops, so a panicking cleanup also releases /// joiners. diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs index 618316e7..2fd0b208 100644 --- a/concurrency/src/supervision.rs +++ b/concurrency/src/supervision.rs @@ -234,6 +234,7 @@ pub struct ChildSpec { restart: RestartMode, startup_timeout: Duration, shutdown_timeout: Duration, + bounds: crate::reply::jobs::Bounds, } impl ChildSpec { /// Create a permanent child with five-second startup and shutdown limits. @@ -255,8 +256,21 @@ impl ChildSpec { restart: RestartMode::Permanent, startup_timeout: Duration::from_secs(5), shutdown_timeout: Duration::from_secs(5), + bounds: Default::default(), } } + /// Override `Actor::MAX_OUTSTANDING` (deferred credits per instance, + /// capped at 65535) for every generation of this child. + pub fn max_outstanding(mut self, max: usize) -> Self { + self.bounds.max_outstanding = Some(max); + self + } + /// Override `Actor::MAX_NATIVE_JOBS` (blocking jobs running at once) for + /// every generation of this child. + pub fn max_native_jobs(mut self, max: usize) -> Self { + self.bounds.max_native_jobs = Some(max); + self + } pub fn restart(mut self, restart: RestartMode) -> Self { self.restart = restart; self @@ -1907,6 +1921,7 @@ async fn run_child(spec: ChildSpec, view: Arc>, core: Arc>>, + /// Per-`ChildSpec` overrides of `Actor::MAX_OUTSTANDING` and + /// `Actor::MAX_NATIVE_JOBS`, read when the actor instance is created. + bounds: Mutex, } impl Scope { + pub(crate) fn set_bounds(&self, bounds: crate::reply::jobs::Bounds) { + *lock(&self.bounds) = bounds; + } + pub(crate) fn bounds(&self) -> crate::reply::jobs::Bounds { + *lock(&self.bounds) + } /// Shorten the cleanup deadline that bounds the actor's job join. pub(crate) fn set_cleanup_timeout(&self, timeout: Duration) { let nanos = u64::try_from(timeout.as_nanos()).unwrap_or(u64::MAX); diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index eed8d055..56b4621f 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -34,7 +34,7 @@ pub const DEFAULT_STOP_DEADLINE: Duration = Duration::from_secs(5); #[path = "jobs.rs"] mod jobs; -pub use jobs::JobCtx; +pub use jobs::{BlockingJobPermit, JobCtx, JobPermit}; // --------------------------------------------------------------------------- // Backend @@ -225,6 +225,33 @@ where } } +/// A closure run in one of the actor's turns (`spawned_test::tasks::in_actor`). +type InTurn = Box) -> R + Send>; +struct InTurnEnvelope { + f: InTurn, + slot: SlotHandle, +} + +impl Envelope for InTurnEnvelope { + #[cfg(feature = "diagnostics")] + fn message_type(&self) -> &'static str { + "InTurn" + } + fn handle<'a>( + self: Box, + actor: &'a mut A, + ctx: &'a Context, + ) -> Pin + Send + 'a>> { + let InTurnEnvelope { f, mut slot } = *self; + // Inside the future, so a panicking closure is a handler panic. + Box::pin(async move { + let reply = PendingReply::new(slot.take(), &ctx.shared.replies); + let value = f(actor, ctx); + let _ = reply.reply(value); + }) + } +} + // Private lifecycle signal, excluded from application message/handler metrics. // Nested supervisor adapters have no public mailbox traffic to bury this signal. struct FailureEnvelope(String); @@ -883,6 +910,26 @@ impl ActorRef { .await } + /// Run `f` in one of the actor's turns, with its state and `Context`, + /// and receive its result. For tests: use `spawned_test::tasks::in_actor`. + #[doc(hidden)] + pub fn __in_turn(&self, f: F) -> Result, ActorError> + where + R: Send + 'static, + F: FnOnce(&mut A, &Context) -> R + Send + 'static, + { + let (slot, rx) = reply_slot(Deadline::Never, Some(&self.shared.replies)); + let envelope: Box + Send> = Box::new(InTurnEnvelope { + f: Box::new(f), + slot, + }); + self.shared + .sender + .send(envelope) + .map_err(|_| ActorError::ActorStopped)?; + Ok(rx) + } + /// Send a request and wait for the reply with a custom timeout. pub async fn request_with_timeout( &self, @@ -1083,14 +1130,17 @@ impl ActorRef { let id = ActorId::next(); let (cleaned_tx, cleaned_rx) = watch::channel(false); + // A supervised child's `ChildSpec` may override the actor's bounds. + let (max_outstanding, max_native) = managed + .as_ref() + .map(|scope| scope.bounds()) + .unwrap_or_default() + .resolve(A::MAX_OUTSTANDING, A::MAX_NATIVE_JOBS); let actor_ref = ActorRef { shared: Arc::new(ActorShared { id, - replies: Registry::new(id, A::MAX_OUTSTANDING), - jobs: Arc::new(crate::reply::jobs::JobSet::new( - id, - A::MAX_NATIVE_JOBS.unwrap_or_else(crate::reply::jobs::default_max_native_jobs), - )), + replies: Registry::new(id, max_outstanding), + jobs: Arc::new(crate::reply::jobs::JobSet::new(id, max_native)), cleaned: cleaned_rx, sender: tx, cancellation_token: cancellation_token.clone(), diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index 95d19941..09993d8b 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -9,7 +9,7 @@ use crate::message::Message; use crate::reply::deadline::Expirable; use crate::reply::jobs::{ JobCredit, JobDeadline, JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, JobState, - PipeError, PipeErrorKind, PipeReplyError, + PipeError, PipeErrorKind, PipeReplyError, SendScheduled, }; use crate::reply::slot::{Refused, Slot}; use crate::reply::{now_ticks, PendingReply, ReplyTo}; @@ -123,6 +123,14 @@ struct Run { done: Mutex>>, } +fn end_of(outcome: &JobOutcome) -> &'static str { + match outcome { + JobOutcome::Ok(_) => "ok", + JobOutcome::Err(JobError::Panicked(_)) => "panicked", + JobOutcome::Err(JobError::DeadlineExceeded) => "deadline", + } +} + impl Run where A: Actor + Handler, @@ -131,20 +139,41 @@ where { /// Take the terminal notification and enqueue `(done, outcome)`. fn notify(&self, outcome: JobOutcome) { - let end = match &outcome { - JobOutcome::Ok(_) => "ok", - JobOutcome::Err(JobError::Panicked(_)) => "panicked", - JobOutcome::Err(JobError::DeadlineExceeded) => "deadline", - }; - let (_, unsent) = self.state.notify(end, || { - let done = self.done.lock().unwrap_or_else(|p| p.into_inner()).take()?; - let envelope: Box + Send> = Box::new(TerminalEnvelope { - job: self.state.clone(), - done, - outcome, - }); - self.ctx.shared.sender.send(envelope).err() + let (_, unsent) = self + .state + .notify(end_of(&outcome), || self.enqueue_terminal(outcome)); + drop(unsent); + } + + fn enqueue_terminal(&self, outcome: JobOutcome) -> Option + Send>> { + let done = self.done.lock().unwrap_or_else(|p| p.into_inner()).take()?; + let envelope: Box + Send> = Box::new(TerminalEnvelope { + job: self.state.clone(), + done, + outcome, }); + self.ctx.shared.sender.send(envelope).err().map(|e| e.0) + } + + /// The work's result: notify now, or, while delayed emits are still + /// scheduled, hold it for the last of them so the terminal message + /// stays last. + fn finish(self: &Arc, outcome: JobOutcome) { + let end = end_of(&outcome); + let outcome = std::cell::Cell::new(Some(outcome)); + let unsent = self.state.finish( + end, + || self.enqueue_terminal(outcome.take()?), + || { + let run = self.clone(); + let outcome = outcome.take(); + Box::new(move || { + if let Some(outcome) = outcome { + run.notify(outcome); + } + }) + }, + ); drop(unsent); } } @@ -229,6 +258,56 @@ impl JobCtx { self.enqueue(msg) } + /// [`emit`](Self::emit) delivered at `at` instead of now (OTP's + /// `erlang:send_after` to the owner, fenced with the job). Takes an emit + /// credit now (waiting while all are in flight) and keeps it until the + /// message is dispatched. At `at` the message is enqueued like an + /// `emit`: dropped if the job was cancelled or replaced, or its deadline + /// passed. The job's terminal message waits for its scheduled emits, so + /// it stays last even when the work finishes first. Fails with + /// [`JobGone`] at the call once the job is cancelled or its notification + /// was taken. + pub async fn emit_at(&self, at: crate::Instant, msg: M) -> Result<(), JobGone> + where + A: Handler, + M: Message, + { + std::future::poll_fn(|cx| self.state.poll_acquire(cx)).await?; + self.schedule(at, msg) + } + + /// Blocking [`emit_at`](Self::emit_at), for `pipe_blocking*` jobs. + pub fn emit_at_blocking(&self, at: crate::Instant, msg: M) -> Result<(), JobGone> + where + A: Handler, + M: Message, + { + self.state.acquire_blocking()?; + self.schedule(at, msg) + } + + fn schedule(&self, at: crate::Instant, msg: M) -> Result<(), JobGone> + where + A: Handler, + M: Message, + { + let credit = EmitCredit(self.state.clone()); + let ctx = self.ctx.clone(); + let send: SendScheduled = Box::new(move || { + let envelope: Box + Send> = Box::new(EmitEnvelope { msg, credit }); + ctx.shared + .sender + .send(envelope) + .map_err(|refused| Box::new(refused.0) as Box) + }); + JobState::schedule_emit( + &self.state, + spawned_rt::time::to_ticks(at), + send, + |at, item| self.ctx.shared.replies.track(at, item), + ) + } + /// Blocking [`emit`](Self::emit), for `pipe_blocking*` jobs. pub fn emit_blocking(&self, msg: M) -> Result<(), JobGone> where @@ -261,12 +340,199 @@ impl JobCtx { // Context API // --------------------------------------------------------------------------- +/// A job admission reserved ahead of building the work: `ctx.reserve_job()` +/// takes the credit (and checks the owner is not stopping) without consuming +/// anything, so a caller that is refused still holds what it would have +/// moved into `work` and `done`. Spawning from a permit cannot fail. +/// Dropping an unused permit returns its credit. +#[must_use = "an unused permit only holds a credit; spawn the job with it or drop it"] +pub struct JobPermit { + ctx: Context, + credit: JobCredit, +} + +/// [`JobPermit`] for a blocking job: it also holds a native slot +/// (`MAX_NATIVE_JOBS`). From `ctx.reserve_blocking_job()`. +#[must_use = "an unused permit only holds a credit and a native slot; spawn the job with it or drop it"] +pub struct BlockingJobPermit { + ctx: Context, + credit: JobCredit, + native: NativeSlot, +} + +impl std::fmt::Debug for JobPermit { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("JobPermit").finish_non_exhaustive() + } +} + +impl std::fmt::Debug for BlockingJobPermit { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BlockingJobPermit").finish_non_exhaustive() + } +} + +impl JobPermit { + /// [`Context::pipe`] with this permit's credit. + pub fn pipe(self, work: W, done: D) -> JobHandle + where + A: Handler, + M: Message, + T: Send + 'static, + W: Future + Send + 'static, + D: FnOnce(JobOutcome) -> M + Send + 'static, + { + self.pipe_with(JobOptions::default(), work, done) + } + + /// [`Context::pipe_with`] with this permit's credit. If the owner began + /// exiting since the reservation, nothing runs: `work` and `done` are + /// dropped and the handle reports the job cancelled (as owner exit + /// cancels every job). + pub fn pipe_with(self, opts: JobOptions, work: W, done: D) -> JobHandle + where + A: Handler, + M: Message, + T: Send + 'static, + W: Future + Send + 'static, + D: FnOnce(JobOutcome) -> M + Send + 'static, + { + let JobPermit { ctx, credit } = self; + let Some(state) = ctx.start_job(&opts) else { + return ctx.refused_job(&opts, (work, done)); + }; + ctx.launch_async::( + opts, + state, + Hold::Credit(credit), + work, + Box::new(done), + None, + ) + } + + /// [`Context::pipe_with_ctx`] with this permit's credit. + pub fn pipe_with_ctx(self, opts: JobOptions, work: F, done: D) -> JobHandle + where + A: Handler, + M: Message, + T: Send + 'static, + F: FnOnce(JobCtx) -> W, + W: Future + Send + 'static, + D: FnOnce(JobOutcome) -> M + Send + 'static, + { + let JobPermit { ctx, credit } = self; + let Some(state) = ctx.start_job(&opts) else { + return ctx.refused_job(&opts, (work, done)); + }; + let job = JobCtx { + state: state.clone(), + ctx: ctx.clone(), + }; + let work = work(job); + ctx.launch_async::( + opts, + state, + Hold::Credit(credit), + work, + Box::new(done), + None, + ) + } +} + +impl BlockingJobPermit { + /// [`Context::pipe_blocking_with`] with this permit's credit and native + /// slot; cannot fail. If the owner began exiting since the reservation, + /// nothing runs (see [`JobPermit::pipe_with`]). + pub fn pipe_blocking_with(self, opts: JobOptions, work: W, done: D) -> JobHandle + where + A: Handler, + M: Message, + T: Send + 'static, + W: FnOnce(&JobCtx) -> T + Send + 'static, + D: FnOnce(JobOutcome) -> M + Send + 'static, + { + let BlockingJobPermit { + ctx, + credit, + native, + } = self; + let Some(state) = ctx.start_job(&opts) else { + return ctx.refused_job(&opts, (work, done, native)); + }; + ctx.launch_blocking::( + opts, + state, + Hold::Credit(credit), + native, + work, + Box::new(done), + ) + } +} + impl Context { - fn admit(&self, opts: &JobOptions) -> Result, PipeErrorKind> { + /// The job's state, or `None` once the owner is exiting. + fn start_job(&self, opts: &JobOptions) -> Option> { + (!self.shared.replies.is_exiting()).then(|| JobState::new(self.id(), opts.emit_credits)) + } + + /// A permit used after the owner began exiting: nothing runs, the parts + /// drop here (outside locks), and the handle reports cancelled. + fn refused_job

(&self, opts: &JobOptions, parts: P) -> JobHandle { + drop(parts); + let state = JobState::new(self.id(), opts.emit_credits); + state.never_started(); + JobHandle::new(state) + } + + fn reserve_credit(&self) -> Result { if self.shared.replies.is_exiting() { - return Err(PipeErrorKind::OwnerStopping); + return Err(PipeError { + kind: PipeErrorKind::OwnerStopping, + }); } - Ok(JobState::new(self.id(), opts.emit_credits)) + JobCredit::reserve(&self.shared.replies).ok_or(PipeError { + kind: PipeErrorKind::Overloaded, + }) + } + + fn reserve_native(&self) -> Result, PipeErrorKind> { + if !self.shared.jobs.reserve_native() { + return Err(PipeErrorKind::NativeBudget); + } + Ok(NativeSlot(self.clone())) + } + + /// Reserve an unlinked job's admission (the owner is running, one + /// `MAX_OUTSTANDING` credit) before building `work` and `done`: a + /// refusal leaves the caller everything it would have moved into them, + /// and spawning from the permit cannot fail. + /// + /// ```ignore + /// let Ok(permit) = ctx.reserve_job() else { + /// return self.report(id, Verdict::Overloaded); // `id` was not moved + /// }; + /// permit.pipe(fetch(object), move |res| Fetched { id, res }); + /// ``` + pub fn reserve_job(&self) -> Result, PipeError> { + Ok(JobPermit { + ctx: self.clone(), + credit: self.reserve_credit()?, + }) + } + + /// [`reserve_job`](Self::reserve_job) for a blocking job: also takes a + /// native slot (`MAX_NATIVE_JOBS`, `NativeBudget` at the bound). + pub fn reserve_blocking_job(&self) -> Result, PipeError> { + let credit = self.reserve_credit()?; + let native = self.reserve_native().map_err(|kind| PipeError { kind })?; + Ok(BlockingJobPermit { + ctx: self.clone(), + credit, + native, + }) } fn arm_deadline( @@ -347,7 +613,7 @@ impl Context { // count) before the terminal message can be dispatched, // so `done`'s turn can start the next job at the bound. drop(_hold); - run.notify(caught(result)) + run.finish(caught(result)) } // Cancelled (or the opt-in caller give-up): the work future is // dropped here, and with it any downstream receiver. @@ -360,14 +626,40 @@ impl Context { handle } + /// Run a blocking job's body on Tokio's blocking pool (the managed + /// actor's group runtime, or the current one). Outside a Tokio runtime, + /// and under the simulator, on an owned thread. + fn spawn_native(&self, task: impl FnOnce() + Send + 'static) { + #[cfg(not(spawned_sim))] + { + let handle = match &self.shared.managed { + Some(scope) => Some(scope.handle.clone()), + None => spawned_rt::tasks::Handle::try_current().ok(), + }; + if let Some(handle) = handle { + drop(handle.spawn_blocking(task)); + return; + } + } + if let Err(error) = std::thread::Builder::new() + .name("spawned-job".into()) + .spawn(task) + { + // The task (its native slot, credit and `done`) dropped with the + // failed spawn: the job ends without a terminal message. + tracing::error!(%error, actor = %self.id(), "could not spawn a native job thread"); + } + } + fn launch_blocking( &self, opts: JobOptions, state: Arc, hold: Hold, + native: NativeSlot, work: W, done: Done, - ) -> Result + ) -> JobHandle where A: Handler, M: Message, @@ -375,60 +667,37 @@ impl Context { R: Send + 'static, W: FnOnce(&JobCtx) -> T + Send + 'static, { - if !self.shared.jobs.reserve_native() { - return Err(PipeErrorKind::NativeBudget); - } - let native = NativeSlot(self.clone()); + self.shared + .jobs + .admit(&state, opts.key.clone(), opts.deadline.is_some()); let run = Arc::new(Run { state: state.clone(), ctx: self.clone(), done: Mutex::new(Some(done)), }); - let thread_run = run.clone(); + let deadline = self.arm_deadline(&opts, &run); + let ended = Ended(self.clone(), state.clone()); let job = JobCtx { state: state.clone(), ctx: self.clone(), }; - // Admission happens only once the thread exists; the thread waits - // for it, so a spawn failure leaves no trace in the job set. - type Start = (Ended, Option>); - let (start, started) = std::sync::mpsc::channel::>(); - let spawned = std::thread::Builder::new() - .name("spawned-job".into()) - .spawn(move || { - let _native = native; - let _hold = hold; - let Ok((ended, deadline)) = started.recv() else { - return; - }; - let _ended = ended; - let _deadline = deadline; - let result = std::panic::catch_unwind(AssertUnwindSafe(|| work(&job))); - // Execution ended: release the native slot and credit first. - drop((_native, _hold)); - thread_run.notify(caught(result)); - }); - match spawned { - Ok(_thread) => { - self.shared - .jobs - .admit(&state, opts.key.clone(), opts.deadline.is_some()); - let deadline = self.arm_deadline(&opts, &run); - let ended = Ended(self.clone(), state.clone()); - let _ = start.send((ended, deadline)); - Ok(JobHandle::new(state)) - } - // The closure (native slot, hold, `done`) dropped with the failed - // spawn, releasing what was reserved. - Err(error) => Err(PipeErrorKind::SpawnFailed(error)), - } + self.spawn_native(move || { + let _ended = ended; + let _deadline = deadline; + let result = std::panic::catch_unwind(AssertUnwindSafe(|| work(&job))); + // Execution ended: release the native slot and credit first. + drop((native, hold)); + run.finish(caught(result)); + }); + JobHandle::new(state) } /// Run `work` as a job owned by this actor instance and deliver /// `done(outcome)` as one message in a later turn (`handle_info`). A /// panic arrives as `JobOutcome::Err(JobError::Panicked)`. Holds one /// `MAX_OUTSTANDING` credit while it executes. Rejects at once at a - /// bound; never waits. + /// bound; never waits. On `Err`, `work` and `done` are dropped: use + /// [`reserve_job`](Self::reserve_job) to keep them. pub fn pipe(&self, work: W, done: D) -> Result where A: Handler, @@ -476,18 +745,7 @@ impl Context { W: Future + Send + 'static, D: FnOnce(JobOutcome) -> M + Send + 'static, { - let state = self.admit(&opts).map_err(|kind| PipeError { kind })?; - let credit = JobCredit::reserve(&self.shared.replies).ok_or(PipeError { - kind: PipeErrorKind::Overloaded, - })?; - Ok(self.launch_async::( - opts, - state, - Hold::Credit(credit), - work, - Box::new(done), - None, - )) + Ok(self.reserve_job()?.pipe_with(opts, work, done)) } /// [`pipe_with`](Self::pipe_with) for async work that needs its @@ -508,28 +766,15 @@ impl Context { W: Future + Send + 'static, D: FnOnce(JobOutcome) -> M + Send + 'static, { - let state = self.admit(&opts).map_err(|kind| PipeError { kind })?; - let credit = JobCredit::reserve(&self.shared.replies).ok_or(PipeError { - kind: PipeErrorKind::Overloaded, - })?; - let job = JobCtx { - state: state.clone(), - ctx: self.clone(), - }; - let work = work(job); - Ok(self.launch_async::( - opts, - state, - Hold::Credit(credit), - work, - Box::new(done), - None, - )) + Ok(self.reserve_job()?.pipe_with_ctx(opts, work, done)) } - /// Run blocking `work` on an owned native thread, under the actor's - /// `MAX_NATIVE_JOBS` budget. `work` gets a [`JobCtx`] to check - /// cancellation and `emit_blocking` fenced messages. + /// Run blocking `work` on Tokio's blocking pool, under the actor's + /// `MAX_NATIVE_JOBS` budget (cancellation stays cooperative: the job + /// checks its [`JobCtx`]). `work` gets a [`JobCtx`] to check + /// cancellation and `emit_blocking` fenced messages; a panic arrives as + /// `JobError::Panicked`. On `Err`, `work` and `done` are dropped: use + /// [`reserve_blocking_job`](Self::reserve_blocking_job) to keep them. pub fn pipe_blocking_with( &self, opts: JobOptions, @@ -543,12 +788,9 @@ impl Context { W: FnOnce(&JobCtx) -> T + Send + 'static, D: FnOnce(JobOutcome) -> M + Send + 'static, { - let state = self.admit(&opts).map_err(|kind| PipeError { kind })?; - let credit = JobCredit::reserve(&self.shared.replies).ok_or(PipeError { - kind: PipeErrorKind::Overloaded, - })?; - self.launch_blocking::(opts, state, Hold::Credit(credit), work, Box::new(done)) - .map_err(|kind| PipeError { kind }) + Ok(self + .reserve_blocking_job()? + .pipe_blocking_with(opts, work, done)) } /// Delegate a deferred reply: run `work` as a job linked to `token`'s @@ -596,7 +838,8 @@ impl Context { Ok(self.launch_async(opts, state, hold, work, linked_done(token, done), watch)) } - /// Blocking [`pipe_reply`](Self::pipe_reply) on an owned native thread. + /// Blocking [`pipe_reply`](Self::pipe_reply) on Tokio's blocking pool, + /// under the `MAX_NATIVE_JOBS` budget. pub fn pipe_reply_blocking_with( &self, opts: JobOptions, @@ -613,29 +856,19 @@ impl Context { D: FnOnce(JobOutcome, ReplyTo) -> M + Send + 'static, { let (state, slot, token) = self.link_reply(&opts, token)?; + let hold = Hold::Linked(LinkedJob(slot.clone())); + let native = match self.reserve_native() { + Ok(native) => native, + // Dropping the hold detaches the linked job from the slot. + Err(kind) => { + drop(hold); + return Err(PipeReplyError::Failed { token, kind }); + } + }; if opts.cancel_on_caller_gone { crate::reply::jobs::cancel_when_closed(&slot, &state); } - let hold = Hold::Linked(LinkedJob(slot)); - let cell = Arc::new(Mutex::new(Some(token))); - let shared = cell.clone(); - let done: Done = Box::new(move |outcome| { - let token = shared - .lock() - .unwrap_or_else(|p| p.into_inner()) - .take() - .expect("the token is taken once, by done or by a spawn failure"); - done(outcome, token) - }); - self.launch_blocking(opts, state, hold, work, done) - .map_err(|kind| { - let token = cell - .lock() - .unwrap_or_else(|p| p.into_inner()) - .take() - .expect("done never ran"); - PipeReplyError::Failed { token, kind } - }) + Ok(self.launch_blocking(opts, state, hold, native, work, linked_done(token, done))) } /// Checks shared by the `pipe_reply*` calls: owner, owner exit, and the @@ -650,9 +883,11 @@ impl Context { if token.owner() != self.id() { return Err(PipeReplyError::WrongOwner(token)); } - let state = match self.admit(opts) { - Ok(state) => state, - Err(kind) => return Err(PipeReplyError::Failed { token, kind }), + let Some(state) = self.start_job(opts) else { + return Err(PipeReplyError::Failed { + token, + kind: PipeErrorKind::OwnerStopping, + }); }; let slot = token.slot_arc().clone(); match slot.attach(&now_ticks) { diff --git a/concurrency/src/tasks/mod.rs b/concurrency/src/tasks/mod.rs index c1880fdf..f52f1157 100644 --- a/concurrency/src/tasks/mod.rs +++ b/concurrency/src/tasks/mod.rs @@ -16,10 +16,11 @@ pub use crate::reply::{ ReplyReceiver, ReplyTo, DEFAULT_MAX_OUTSTANDING, }; pub use crate::response::Response; +pub use crate::Instant; pub use actor::{ request, send_message_on, Actor, ActorRef, ActorStart, Backend, Context, Handler, Receiver, Recipient, DEFAULT_REQUEST_TIMEOUT, }; -pub use actor::{JobCtx, DEFAULT_STOP_DEADLINE}; +pub use actor::{BlockingJobPermit, JobCtx, JobPermit, DEFAULT_STOP_DEADLINE}; pub use stream::spawn_listener; pub use time::{send_after, send_interval, TimerHandle}; diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 801b5bae..4479207c 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -28,7 +28,7 @@ pub const DEFAULT_STOP_DEADLINE: Duration = Duration::from_secs(5); #[path = "jobs.rs"] mod jobs; -pub use jobs::JobCtx; +pub use jobs::{BlockingJobPermit, JobCtx}; // --------------------------------------------------------------------------- // Actor trait @@ -152,6 +152,22 @@ where } } +/// A closure run in one of the actor's turns (`spawned_test::threads::in_actor`). +type InTurn = Box) -> R + Send>; +struct InTurnEnvelope { + f: InTurn, + slot: SlotHandle, +} + +impl Envelope for InTurnEnvelope { + fn handle(self: Box, actor: &mut A, ctx: &Context) { + let InTurnEnvelope { f, mut slot } = *self; + let reply = PendingReply::new(slot.take(), &ctx.replies); + let value = f(actor, ctx); + let _ = reply.reply(value); + } +} + /// System envelope that dispatches an `Exit` notification to the actor's /// `exit_received` callback. Does not require `Handler` on the actor. struct ExitEnvelope { @@ -714,6 +730,25 @@ impl ActorRef { self.request_with_timeout(msg, DEFAULT_REQUEST_TIMEOUT) } + /// Run `f` in one of the actor's turns, with its state and `Context`, + /// and receive its result. For tests: use `spawned_test::threads::in_actor`. + #[doc(hidden)] + pub fn __in_turn(&self, f: F) -> Result, ActorError> + where + R: Send + 'static, + F: FnOnce(&mut A, &Context) -> R + Send + 'static, + { + let (slot, rx) = reply_slot(Deadline::Never, None); + let envelope: Box + Send> = Box::new(InTurnEnvelope { + f: Box::new(f), + slot, + }); + self.sender + .send(MailboxItem::Message(envelope)) + .map_err(|_| ActorError::ActorStopped)?; + Ok(rx) + } + /// Send a request and block until the reply arrives, with a custom timeout. pub fn request_with_timeout( &self, diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index 08aab7dd..78f2421f 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -11,7 +11,7 @@ use crate::message::Message; use crate::reply::deadline::Expirable; use crate::reply::jobs::{ JobCredit, JobDeadline, JobError, JobGone, JobHandle, JobOptions, JobOutcome, JobState, - PipeError, PipeErrorKind, PipeReplyError, + PipeError, PipeErrorKind, PipeReplyError, SendScheduled, }; use crate::reply::slot::{Refused, Slot}; use crate::reply::{now_ticks, PendingReply, ReplyTo}; @@ -101,6 +101,14 @@ struct Run { done: Mutex>>, } +fn end_of(outcome: &JobOutcome) -> &'static str { + match outcome { + JobOutcome::Ok(_) => "ok", + JobOutcome::Err(JobError::Panicked(_)) => "panicked", + JobOutcome::Err(JobError::DeadlineExceeded) => "deadline", + } +} + impl Run where A: Actor + Handler, @@ -109,20 +117,45 @@ where { /// Take the terminal notification and enqueue `(done, outcome)`. fn notify(&self, outcome: JobOutcome) { - let end = match &outcome { - JobOutcome::Ok(_) => "ok", - JobOutcome::Err(JobError::Panicked(_)) => "panicked", - JobOutcome::Err(JobError::DeadlineExceeded) => "deadline", - }; - let (_, unsent) = self.state.notify(end, || { - let done = self.done.lock().unwrap_or_else(|p| p.into_inner()).take()?; - let envelope: Box + Send> = Box::new(TerminalEnvelope { - job: self.state.clone(), - done, - outcome, - }); - self.ctx.sender.send(MailboxItem::Message(envelope)).err() + let (_, unsent) = self + .state + .notify(end_of(&outcome), || self.enqueue_terminal(outcome)); + drop(unsent); + } + + fn enqueue_terminal(&self, outcome: JobOutcome) -> Option> { + let done = self.done.lock().unwrap_or_else(|p| p.into_inner()).take()?; + let envelope: Box + Send> = Box::new(TerminalEnvelope { + job: self.state.clone(), + done, + outcome, }); + self.ctx + .sender + .send(MailboxItem::Message(envelope)) + .err() + .map(|e| e.0) + } + + /// The work's result: notify now, or, while delayed emits are still + /// scheduled, hold it for the last of them so the terminal message + /// stays last. + fn finish(self: &Arc, outcome: JobOutcome) { + let end = end_of(&outcome); + let outcome = std::cell::Cell::new(Some(outcome)); + let unsent = self.state.finish( + end, + || self.enqueue_terminal(outcome.take()?), + || { + let run = self.clone(); + let outcome = outcome.take(); + Box::new(move || { + if let Some(outcome) = outcome { + run.notify(outcome); + } + }) + }, + ); drop(unsent); } } @@ -202,6 +235,35 @@ impl JobCtx { self.enqueue(msg) } + /// [`emit_blocking`](Self::emit_blocking) delivered at `at` instead of + /// now (OTP's `erlang:send_after` to the owner, fenced with the job). + /// Takes an emit credit now (blocking while all are in flight) and keeps + /// it until the message is dispatched. At `at` the message is enqueued + /// like an emit: dropped if the job was cancelled or replaced, or its + /// deadline passed. The job's terminal message waits for its scheduled + /// emits, so it stays last even when the work finishes first. + pub fn emit_at_blocking(&self, at: crate::Instant, msg: M) -> Result<(), JobGone> + where + A: Handler, + M: Message, + { + self.state.acquire_blocking()?; + let credit = EmitCredit(self.state.clone()); + let ctx = self.ctx.clone(); + let send: SendScheduled = Box::new(move || { + let envelope: Box + Send> = Box::new(EmitEnvelope { msg, credit }); + ctx.sender + .send(MailboxItem::Message(envelope)) + .map_err(|refused| Box::new(refused.0) as Box) + }); + JobState::schedule_emit( + &self.state, + spawned_rt::time::to_ticks(at), + send, + |at, item| self.ctx.replies.track(at, item), + ) + } + fn enqueue(&self, msg: M) -> Result<(), JobGone> where A: Handler, @@ -224,12 +286,118 @@ impl JobCtx { // Context API // --------------------------------------------------------------------------- +/// The task a native worker thread runs once its job is spawned; it gets +/// the thread's native slot so it can release it before notifying. +type NativeTask = Box) + Send>; + +/// A native thread spawned at reservation and parked until its job is +/// spawned, so spawning from a permit cannot fail. Dropped unused, the +/// thread exits and releases its native slot. +struct NativeWorker { + start: std::sync::mpsc::Sender>, +} + +/// A blocking job's admission reserved ahead of building the work: +/// `ctx.reserve_blocking_job()` checks the owner is running and takes a +/// `MAX_OUTSTANDING` credit, a native slot and the job's (parked) thread, so +/// a caller that is refused still holds what it would have moved into +/// `work` and `done`, and spawning from the permit cannot fail. Dropping an +/// unused permit returns everything. +#[must_use = "an unused permit only holds a credit and a native thread; spawn the job with it or drop it"] +pub struct BlockingJobPermit { + ctx: Context, + credit: JobCredit, + worker: NativeWorker, +} + +impl std::fmt::Debug for BlockingJobPermit { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BlockingJobPermit").finish_non_exhaustive() + } +} + +impl BlockingJobPermit { + /// [`Context::pipe_blocking_with`] with this permit; cannot fail. If the + /// owner began exiting since the reservation, nothing runs: `work` and + /// `done` are dropped and the handle reports the job cancelled (as + /// owner exit cancels every job). + pub fn pipe_blocking_with(self, opts: JobOptions, work: W, done: D) -> JobHandle + where + A: Handler, + M: Message, + T: Send + 'static, + W: FnOnce(&JobCtx) -> T + Send + 'static, + D: FnOnce(JobOutcome) -> M + Send + 'static, + { + let BlockingJobPermit { + ctx, + credit, + worker, + } = self; + if ctx.replies.is_exiting() { + drop((work, done, worker)); + let state = JobState::new(ctx.id(), opts.emit_credits); + state.never_started(); + return JobHandle::new(state); + } + let state = JobState::new(ctx.id(), opts.emit_credits); + ctx.launch_blocking::( + opts, + state, + Hold::Credit(credit), + worker, + work, + Box::new(done), + ) + } +} + impl Context { - fn admit(&self, opts: &JobOptions) -> Result, PipeErrorKind> { + fn reserve_credit(&self) -> Result { if self.replies.is_exiting() { return Err(PipeErrorKind::OwnerStopping); } - Ok(JobState::new(self.id(), opts.emit_credits)) + JobCredit::reserve(&self.replies).ok_or(PipeErrorKind::Overloaded) + } + + /// Reserve a native slot and spawn its thread, parked until the job is + /// spawned. A spawn failure releases the slot. + fn spawn_worker(&self) -> Result, PipeErrorKind> { + if !self.jobs.reserve_native() { + return Err(PipeErrorKind::NativeBudget); + } + let native = NativeSlot(self.clone()); + let (start, started) = std::sync::mpsc::channel::>(); + std::thread::Builder::new() + .name("spawned-job".into()) + .spawn(move || { + if let Ok(task) = started.recv() { + task(native); + } + }) + .map_err(PipeErrorKind::SpawnFailed)?; + Ok(NativeWorker { start }) + } + + /// Reserve a blocking job's admission (owner running, one credit, a + /// native slot and its thread) before building `work` and `done`: a + /// refusal leaves the caller everything it would have moved into them, + /// and spawning from the permit cannot fail. + /// + /// ```ignore + /// let Ok(permit) = ctx.reserve_blocking_job() else { + /// return self.report(id, Verdict::Overloaded); // `id` was not moved + /// }; + /// permit.pipe_blocking_with(opts, move |_| check(&object), move |res| Checked { id, res }); + /// ``` + pub fn reserve_blocking_job(&self) -> Result, PipeError> { + let credit = self.reserve_credit().map_err(|kind| PipeError { kind })?; + let worker = self.spawn_worker().map_err(|kind| PipeError { kind })?; + Ok(BlockingJobPermit { + ctx: self.clone(), + credit, + worker, + }) } fn arm_deadline( @@ -267,9 +435,10 @@ impl Context { opts: JobOptions, state: Arc, hold: Hold, + worker: NativeWorker, work: W, done: Done, - ) -> Result + ) -> JobHandle where A: Handler, M: Message, @@ -277,52 +446,31 @@ impl Context { R: Send + 'static, W: FnOnce(&JobCtx) -> T + Send + 'static, { - if !self.jobs.reserve_native() { - return Err(PipeErrorKind::NativeBudget); - } - let native = NativeSlot(self.clone()); + self.jobs + .admit(&state, opts.key.clone(), opts.deadline.is_some()); let run = Arc::new(Run { state: state.clone(), ctx: self.clone(), done: Mutex::new(Some(done)), }); - let thread_run = run.clone(); + let deadline = self.arm_deadline(&opts, &run); + let ended = Ended(self.clone(), state.clone()); let job = JobCtx { state: state.clone(), ctx: self.clone(), }; - // Admission happens only once the thread exists; the thread waits - // for it, so a spawn failure leaves no trace in the job set. - type Start = (Ended, Option>); - let (start, started) = std::sync::mpsc::channel::>(); - let spawned = std::thread::Builder::new() - .name("spawned-job".into()) - .spawn(move || { - let _native = native; - let _hold = hold; - let Ok((ended, deadline)) = started.recv() else { - return; - }; - let _ended = ended; - let _deadline = deadline; - let result = std::panic::catch_unwind(AssertUnwindSafe(|| work(&job))); - // Execution ended: release the native slot and credit first. - drop((_native, _hold)); - thread_run.notify(caught(result)); - }); - match spawned { - Ok(_thread) => { - self.jobs - .admit(&state, opts.key.clone(), opts.deadline.is_some()); - let deadline = self.arm_deadline(&opts, &run); - let ended = Ended(self.clone(), state.clone()); - let _ = start.send((ended, deadline)); - Ok(JobHandle::new(state)) - } - // The closure (native slot, hold, `done`) dropped with the failed - // spawn, releasing what was reserved. - Err(error) => Err(PipeErrorKind::SpawnFailed(error)), - } + let task: NativeTask = Box::new(move |native| { + let _ended = ended; + let _deadline = deadline; + let result = std::panic::catch_unwind(AssertUnwindSafe(|| work(&job))); + // Execution ended: release the native slot and credit first. + drop((native, hold)); + run.finish(caught(result)); + }); + // The worker thread is parked on `recv` until this send (or its + // sender's drop); it cannot have exited. + let _ = worker.start.send(task); + JobHandle::new(state) } /// Run blocking `work` as a job on an owned native thread, under the @@ -330,7 +478,9 @@ impl Context { /// message in a later turn. A panic arrives as /// `JobOutcome::Err(JobError::Panicked)`. `work` gets a [`JobCtx`] to /// check cancellation and `emit_blocking` fenced messages. Rejects at - /// once at a bound; never waits. + /// once at a bound; never waits. On `Err`, `work` and `done` are + /// dropped: use [`reserve_blocking_job`](Self::reserve_blocking_job) to + /// keep them. pub fn pipe_blocking_with( &self, opts: JobOptions, @@ -344,12 +494,9 @@ impl Context { W: FnOnce(&JobCtx) -> T + Send + 'static, D: FnOnce(JobOutcome) -> M + Send + 'static, { - let state = self.admit(&opts).map_err(|kind| PipeError { kind })?; - let credit = JobCredit::reserve(&self.replies).ok_or(PipeError { - kind: PipeErrorKind::Overloaded, - })?; - self.launch_blocking::(opts, state, Hold::Credit(credit), work, Box::new(done)) - .map_err(|kind| PipeError { kind }) + Ok(self + .reserve_blocking_job()? + .pipe_blocking_with(opts, work, done)) } /// Delegate a deferred reply: run blocking `work` on an owned native @@ -373,29 +520,20 @@ impl Context { D: FnOnce(JobOutcome, ReplyTo) -> M + Send + 'static, { let (state, slot, token) = self.link_reply(&opts, token)?; + let hold = Hold::Linked(LinkedJob(slot.clone())); + let worker = match self.spawn_worker() { + Ok(worker) => worker, + // Dropping the hold detaches the linked job from the slot. + Err(kind) => { + drop(hold); + return Err(PipeReplyError::Failed { token, kind }); + } + }; if opts.cancel_on_caller_gone { crate::reply::jobs::cancel_when_closed(&slot, &state); } - let hold = Hold::Linked(LinkedJob(slot)); - let cell = Arc::new(Mutex::new(Some(token))); - let shared = cell.clone(); - let done: Done = Box::new(move |outcome| { - let token = shared - .lock() - .unwrap_or_else(|p| p.into_inner()) - .take() - .expect("the token is taken once, by done or by a spawn failure"); - done(outcome, token) - }); - self.launch_blocking(opts, state, hold, work, done) - .map_err(|kind| { - let token = cell - .lock() - .unwrap_or_else(|p| p.into_inner()) - .take() - .expect("done never ran"); - PipeReplyError::Failed { token, kind } - }) + let done: Done = Box::new(move |outcome| done(outcome, token)); + Ok(self.launch_blocking(opts, state, hold, worker, work, done)) } /// Checks shared by the `pipe_reply*` calls: owner, owner exit, and the @@ -410,10 +548,13 @@ impl Context { if token.owner() != self.id() { return Err(PipeReplyError::WrongOwner(token)); } - let state = match self.admit(opts) { - Ok(state) => state, - Err(kind) => return Err(PipeReplyError::Failed { token, kind }), - }; + if self.replies.is_exiting() { + return Err(PipeReplyError::Failed { + token, + kind: PipeErrorKind::OwnerStopping, + }); + } + let state = JobState::new(self.id(), opts.emit_credits); let slot = token.slot_arc().clone(); match slot.attach(&now_ticks) { Ok(()) => Ok((state, slot, token)), diff --git a/concurrency/src/threads/mod.rs b/concurrency/src/threads/mod.rs index bc8480fe..e40b7c5f 100644 --- a/concurrency/src/threads/mod.rs +++ b/concurrency/src/threads/mod.rs @@ -17,10 +17,11 @@ pub use crate::reply::{ ReplyTo, DEFAULT_MAX_OUTSTANDING, }; pub use crate::response::Response; +pub use crate::Instant; pub use actor::{ request, send_message_on, Actor, ActorRef, ActorStart, Context, Handler, Receiver, Recipient, DEFAULT_REQUEST_TIMEOUT, }; -pub use actor::{JobCtx, DEFAULT_STOP_DEADLINE}; +pub use actor::{BlockingJobPermit, JobCtx, DEFAULT_STOP_DEADLINE}; pub use stream::spawn_listener; pub use time::{send_after, send_interval, TimerHandle}; diff --git a/concurrency/tests/deferred_api.rs b/concurrency/tests/deferred_api.rs new file mode 100644 index 00000000..dcdfb287 --- /dev/null +++ b/concurrency/tests/deferred_api.rs @@ -0,0 +1,380 @@ +//! Deferred-reply API refinements from porting ethlambda and ethrex to 0.8: +//! bounds set in `#[actor]` and per `ChildSpec`, blocking jobs on Tokio's +//! blocking pool, reserve-then-spawn permits, fenced delayed emits +//! (`emit_at`), hard job deadlines, send-only manual handlers +//! (`#[send_handler]` on an impl) and `spawned_test::in_actor`. +use spawned_concurrency::message::Message; +use std::sync::{Arc, Mutex}; + +/// What the owner's handlers observed, in dispatch order. +#[derive(Default)] +pub struct Log(pub Mutex>); +impl Log { + pub fn push(&self, entry: impl Into) { + self.0.lock().unwrap().push(entry.into()); + } + pub fn snapshot(&self) -> Vec { + self.0.lock().unwrap().clone() + } +} + +pub struct Note(pub String); +impl Message for Note { + type Result = (); +} + +/// A send-only message whose handler returns early on one path. +pub struct Poke(pub bool); +impl Message for Poke { + type Result = (); +} + +#[cfg(feature = "async")] +mod tasks_mode { + use super::*; + use spawned_concurrency::supervision::{ChildSpec, SupervisorConfig}; + use spawned_concurrency::system::ActorSystem; + use spawned_concurrency::tasks::{ + Actor, ActorRef, ActorStart, Context, Handler, Instant, JobError, JobGone, JobOptions, + JobOutcome, PipeErrorKind, + }; + use spawned_concurrency::{actor, send_handler}; + use std::time::Duration; + + /// `spawned_test::tasks::in_actor` (the dev-dependency is built without + /// its Tokio half; `testing/tests/tasks_kit.rs` covers the helper). + async fn in_actor(actor: &ActorRef, f: F) -> R + where + R: Send + 'static, + F: FnOnce(&mut Owner, &Context) -> R + Send + 'static, + { + actor.__in_turn(f).unwrap().await.unwrap() + } + + pub struct Owner { + log: Arc, + } + + #[actor(max_outstanding = 3, max_native_jobs = 2)] + impl Owner { + #[send_handler] + async fn note(&mut self, msg: Note, _: &Context) { + self.log.push(msg.0); + } + } + + #[send_handler] + impl Handler for Owner { + async fn handle(&mut self, msg: Poke, _ctx: &Context) { + if msg.0 { + return; + } + self.log.push("poked"); + } + } + + fn describe(name: &str, outcome: JobOutcome) -> Note { + Note(match outcome.into_result() { + Ok(v) => format!("{name}:ok:{v:?}"), + Err(JobError::Panicked(m)) => format!("{name}:panicked:{m}"), + Err(JobError::DeadlineExceeded) => format!("{name}:deadline"), + Err(other) => format!("{name}:{other}"), + }) + } + + fn start() -> (ActorRef, Arc) { + let log = Arc::new(Log::default()); + (Owner { log: log.clone() }.start(), log) + } + + async fn wait_for(log: &Log, n: usize) -> Vec { + for _ in 0..1000 { + let entries = log.snapshot(); + if entries.len() >= n { + return entries; + } + tokio::time::sleep(Duration::from_millis(2)).await; + } + panic!("log never reached {n} entries: {:?}", log.snapshot()); + } + + #[test] + fn actor_macro_sets_the_bounds() { + assert_eq!(::MAX_OUTSTANDING, 3); + assert_eq!(::MAX_NATIVE_JOBS, Some(2)); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn macro_bounds_apply_and_permits_keep_the_parts() { + let (actor, _log) = start(); + let kinds = in_actor(&actor, |_, ctx| { + let a = ctx.reserve_blocking_job().expect("first native slot"); + let b = ctx.reserve_blocking_job().expect("second native slot"); + // max_native_jobs = 2: the third is refused, and the caller keeps + // whatever it meant to move into the job. + let third = ctx.reserve_blocking_job().err().map(|e| e.kind); + drop((a, b)); + let c = ctx.reserve_job().expect("credit"); + let d = ctx.reserve_job().expect("credit"); + let e = ctx.reserve_job().expect("credit"); + // max_outstanding = 3. + let fourth = ctx.reserve_job().err().map(|e| e.kind); + drop((c, d, e)); + let again = ctx.reserve_job().is_ok(); + (third, fourth, again, ctx.deferred_outstanding()) + }) + .await; + assert!(matches!(kinds.0, Some(PipeErrorKind::NativeBudget))); + assert!(matches!(kinds.1, Some(PipeErrorKind::Overloaded))); + assert!(kinds.2, "dropped permits return their credits"); + assert_eq!(kinds.3, 0); + actor.context().stop(); + actor.join().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn blocking_jobs_run_on_the_tokio_blocking_pool() { + let (actor, log) = start(); + in_actor(&actor, |_, ctx| { + let permit = ctx.reserve_blocking_job().unwrap(); + permit.pipe_blocking_with( + JobOptions::default(), + |_| std::thread::current().name().map(str::to_owned), + |o| describe("thread", o), + ); + ctx.pipe_blocking_with( + JobOptions::default(), + |_| -> u32 { panic!("check boom") }, + |o| describe("panics", o), + ) + .unwrap(); + }) + .await; + let mut entries = wait_for(&log, 2).await; + entries.sort(); + assert_eq!(entries[0], "panics:panicked:check boom"); + assert!( + entries[1].starts_with("thread:ok:Some(\"tokio"), + "not on the blocking pool: {}", + entries[1] + ); + actor.context().stop(); + actor.join().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn emit_at_arrives_late_but_before_the_terminal_message() { + let (actor, log) = start(); + in_actor(&actor, |_, ctx| { + ctx.pipe_blocking_with( + JobOptions::default(), + |job| { + let at = Instant::now() + Duration::from_millis(80); + job.emit_at_blocking(at, Note("late".into()))?; + job.emit_blocking(Note("now".into()))?; + Ok::<_, JobGone>(()) + }, + |o| describe("job", o), + ) + .unwrap(); + }) + .await; + let entries = wait_for(&log, 3).await; + assert_eq!(entries, ["now", "late", "job:ok:Ok(())"]); + actor.context().stop(); + actor.join().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_cancelled_job_fences_its_scheduled_emits() { + let (actor, log) = start(); + in_actor(&actor, |_, ctx| { + let opts = JobOptions { + key: Some("k".into()), + ..JobOptions::default() + }; + ctx.pipe_with_ctx( + opts.clone(), + |job| async move { + let at = Instant::now() + Duration::from_millis(60); + job.emit_at(at, Note("stale".into())).await + }, + |o| describe("old", o), + ) + .unwrap(); + }) + .await; + // The replacement cancels the previous job: its scheduled emit and + // its held terminal message are dropped. + in_actor(&actor, |_, ctx| { + let opts = JobOptions { + key: Some("k".into()), + ..JobOptions::default() + }; + ctx.pipe_with(opts, async { 1 }, |o| describe("new", o)) + .unwrap(); + }) + .await; + tokio::time::sleep(Duration::from_millis(150)).await; + assert_eq!(log.snapshot(), ["new:ok:1"]); + let outstanding = in_actor(&actor, |_, ctx| ctx.deferred_outstanding()).await; + assert_eq!(outstanding, 0); + actor.context().stop(); + actor.join().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn job_deadlines_are_hard() { + let (actor, log) = start(); + in_actor(&actor, |_, ctx| { + let opts = JobOptions::default().deadline_after(Duration::from_millis(30)); + ctx.pipe_blocking_with( + opts, + |job| { + std::thread::sleep(Duration::from_millis(120)); + // Past the deadline: the job is over for the owner. + job.emit_blocking(Note("too late".into())).is_err() + }, + |o| describe("slow", o), + ) + .unwrap(); + }) + .await; + assert_eq!(wait_for(&log, 1).await, ["slow:deadline"]); + tokio::time::sleep(Duration::from_millis(200)).await; + assert_eq!(log.snapshot(), ["slow:deadline"]); + actor.context().stop(); + actor.join().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn send_handler_impls_answer_requests_with_unit() { + let (actor, log) = start(); + assert!(matches!(actor.request(Poke(false)).await, Ok(()))); + // The early return still answers. + assert!(matches!(actor.request(Poke(true)).await, Ok(()))); + actor.send(Poke(false)).unwrap(); + assert_eq!(wait_for(&log, 2).await, ["poked", "poked"]); + actor.context().stop(); + actor.join().await; + assert!(actor.request(Poke(false)).await.is_err()); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn child_spec_overrides_the_actor_bounds() { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("bounds", 1).unwrap(); + let mut builder = system.supervisor(SupervisorConfig::default()).unwrap(); + let log = Arc::new(Log::default()); + let child = builder + .add( + ChildSpec::new("bounded", &group, move |_| { + let log = log.clone(); + async move { Ok::<_, String>(Owner { log }) } + }) + .max_outstanding(2) + .max_native_jobs(1), + ) + .unwrap(); + let _supervisor = builder.start().await.unwrap(); + let actor = child.current().unwrap(); + let refused = in_actor(&actor, |_, ctx| { + let native = ctx.reserve_blocking_job().expect("one native slot"); + let second_native = ctx.reserve_blocking_job().err().map(|e| e.kind); + drop(native); + let credits = (ctx.reserve_job(), ctx.reserve_job()); + assert!(credits.0.is_ok() && credits.1.is_ok(), "two credits"); + let second_credit = ctx.reserve_job().err().map(|e| e.kind); + drop(credits); + (second_native, second_credit) + }) + .await; + assert!(matches!(refused.0, Some(PipeErrorKind::NativeBudget))); + assert!(matches!(refused.1, Some(PipeErrorKind::Overloaded))); + let _ = system.shutdown(Duration::from_secs(5)).await; + } +} + +mod threads_mode { + use super::*; + use spawned_concurrency::threads::{ + Actor, ActorStart, Context, Handler, Instant, JobGone, JobOptions, JobOutcome, + PipeErrorKind, + }; + use spawned_concurrency::{actor, send_handler}; + use spawned_test::threads::in_actor; + use std::time::Duration; + + pub struct Owner { + log: Arc, + } + + #[actor(max_native_jobs = 1)] + impl Owner { + #[send_handler] + fn note(&mut self, msg: Note, _: &Context) { + self.log.push(msg.0); + } + } + + #[send_handler] + impl Handler for Owner { + fn handle(&mut self, msg: Poke, _ctx: &Context) { + if msg.0 { + return; + } + self.log.push("poked"); + } + } + + fn wait_for(log: &Log, n: usize) -> Vec { + for _ in 0..1000 { + let entries = log.snapshot(); + if entries.len() >= n { + return entries; + } + std::thread::sleep(Duration::from_millis(2)); + } + panic!("log never reached {n} entries: {:?}", log.snapshot()); + } + + fn label(name: &str, outcome: JobOutcome>) -> Note { + Note(format!("{name}:{:?}", outcome.into_result())) + } + + #[test] + fn in_actor_permits_and_emit_at() { + let log = Arc::new(Log::default()); + let actor = Owner { log: log.clone() }.start(); + let refused = in_actor(&actor, |_, ctx| { + let permit = ctx.reserve_blocking_job().expect("one native slot"); + // max_native_jobs = 1. + let second = ctx.reserve_blocking_job().err().map(|e| e.kind); + permit.pipe_blocking_with( + JobOptions::default(), + |job| { + let at = Instant::now() + Duration::from_millis(60); + job.emit_at_blocking(at, Note("late".into()))?; + job.emit_blocking(Note("now".into())) + }, + |o| label("job", o), + ); + second + }); + assert!(matches!(refused, Some(PipeErrorKind::NativeBudget))); + assert_eq!(wait_for(&log, 3), ["now", "late", "job:Ok(Ok(()))"]); + actor.context().stop(); + actor.join(); + } + + #[test] + fn send_handler_impls_answer_requests_with_unit() { + let log = Arc::new(Log::default()); + let actor = Owner { log: log.clone() }.start(); + assert!(matches!(actor.request(Poke(false)), Ok(()))); + assert!(matches!(actor.request(Poke(true)), Ok(()))); + assert_eq!(log.snapshot(), ["poked"]); + actor.context().stop(); + actor.join(); + } +} diff --git a/concurrency/tests/deferred_jobs.rs b/concurrency/tests/deferred_jobs.rs index 092e5929..9495ffee 100644 --- a/concurrency/tests/deferred_jobs.rs +++ b/concurrency/tests/deferred_jobs.rs @@ -78,10 +78,11 @@ mod tasks_mode { } fn describe(name: &str, outcome: JobOutcome) -> Done { - Done(match outcome { - JobOutcome::Ok(v) => format!("{name}:ok:{v:?}"), - JobOutcome::Err(JobError::Panicked(m)) => format!("{name}:panicked:{m}"), - JobOutcome::Err(JobError::DeadlineExceeded) => format!("{name}:deadline"), + Done(match outcome.into_result() { + Ok(v) => format!("{name}:ok:{v:?}"), + Err(JobError::Panicked(m)) => format!("{name}:panicked:{m}"), + Err(JobError::DeadlineExceeded) => format!("{name}:deadline"), + Err(other) => format!("{name}:{other}"), }) } @@ -648,15 +649,15 @@ mod threads_mode { } impl Handler for Owner { fn handle(&mut self, msg: Computed, _: PendingReply<'_, ()>, _: &Context) { - match msg.res { - JobOutcome::Ok(v) => { + match msg.res.into_result() { + Ok(v) => { msg.token.reply(v); } - JobOutcome::Err(JobError::Panicked(m)) => { + Err(JobError::Panicked(m)) => { self.log.push(format!("panicked {m}")); msg.token.overloaded(); } - JobOutcome::Err(JobError::DeadlineExceeded) => msg.token.overloaded(), + Err(_) => msg.token.overloaded(), } } } diff --git a/concurrency/tests/job_core_loom.rs b/concurrency/tests/job_core_loom.rs index 958ea2a2..8d82170f 100644 --- a/concurrency/tests/job_core_loom.rs +++ b/concurrency/tests/job_core_loom.rs @@ -375,6 +375,85 @@ fn emit_vs_notification_and_cancel() { } } +/// A delayed emit (`emit_at`) scheduled by the work fires on the deadline +/// service while the work finishes and, optionally, the job is cancelled. +/// The terminal message is held until the scheduled emit has fired, so it is +/// always last; it is delivered exactly once unless cancelled; a cancel drops +/// a held result; every credit comes back. +#[test] +fn scheduled_emit_vs_finish_and_cancel() { + for cancel in [false, true] { + model(move || { + let mailbox = Arc::new(Mailbox::default()); + let destroyed = Arc::new(AtomicUsize::new(0)); + let job = TestJob::new(0, 1, None, &destroyed); + // The work schedules one delayed emit (its credit taken now). + job.core.acquire_blocking().unwrap(); + job.core.schedule().unwrap(); + let timer = { + let (job, mailbox) = (job.clone(), mailbox.clone()); + loom::thread::spawn(move || { + let (result, held) = job.core.fire(Some(|| { + mailbox.push(Msg::Emit(0, 0)); + Ok::<(), ()>(()) + })); + if result.is_err() { + // Not enqueued: the credit guard returns it. + job.core.return_emit_credit(); + } + if let Some(held) = held { + held(); + } + }) + }; + let canceller = cancel.then(|| { + let job = job.clone(); + loom::thread::spawn(move || job.core.cancel()) + }); + // The work ends: notify now, or hold for the scheduled emit. + let later_job = job.clone(); + let later_mailbox = mailbox.clone(); + job.core.finish( + || { + mailbox.push(Msg::Terminal(0)); + None::<()> + }, + move || { + Box::new(move || { + later_job.core.notify(|| { + later_mailbox.push(Msg::Terminal(0)); + None::<()> + }); + }) + }, + ); + timer.join().unwrap(); + if let Some(canceller) = canceller { + canceller.join().unwrap(); + } + let mut terminals = 0; + for msg in mailbox.take() { + match msg { + Msg::Terminal(_) => terminals += 1, + Msg::Emit(..) => { + assert_eq!(terminals, 0, "the delayed emit followed the terminal"); + job.core.return_emit_credit(); + } + } + } + assert!(terminals <= 1, "two terminal messages"); + assert_eq!(job.core.scheduled(), 0); + if !cancel { + assert_eq!(terminals, 1, "the held terminal was lost"); + } + if terminals == 0 { + assert_eq!(job.core.notification(), Notification::Suppressed); + } + assert_eq!(job.core.emit_credits(), 1, "an emit credit was lost"); + }); + } +} + /// A native job completes (native slot released, terminal notification, /// execution ended) while the owner exits: cancel every job, then join. /// The join is always woken, and once it returns no job runs, the native diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 7a09b32a..9e2feb67 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -91,7 +91,7 @@ Stopping discards messages still queued: once the stop is observed no further me | `ctx.send(msg)` | Send a fire-and-forget message to this actor (self-send) | | `ctx.request(msg)` | Send a request and wait for the reply (tasks: async, threads: blocking) | | `ctx.request_with_timeout(msg, duration)` | Send a request with a custom timeout | -| `ctx.request_raw(msg)` | Send a request and get a raw oneshot receiver | +| `ctx.request_raw(msg)` / `ctx.request_raw_until(msg, deadline)` | Send a request and get its `ReplyReceiver` (tasks: a future, threads: `recv()`/`try_recv()`) yielding `Result`; the deadline (5 s default, or `Deadline::At`/`Never`) is fixed at send | | `ctx.recipient::()` | Get a type-erased `Recipient` for this actor | | `ctx.actor_ref()` | Get an `ActorRef` from the context | | `ctx.id()` | This actor's `ActorId` | @@ -129,7 +129,7 @@ async fn handle_tick(&mut self, _msg: Tick, ctx: &Context) { | `actor_ref.request(msg)` | Request with default 5s timeout. Tasks: `.await`, Threads: blocking | | `actor_ref.request_with_timeout(msg, duration)` | Request with custom timeout | | `actor_ref.with_request_timeout(duration)` | Clone a configured handle for generated protocol methods (v0.7) | -| `actor_ref.request_raw(msg)` | Returns a raw oneshot receiver | +| `actor_ref.request_raw(msg)` / `actor_ref.request_raw_until(msg, deadline)` | Returns the request's `ReplyReceiver` (as `ctx.request_raw`) | | `actor_ref.recipient::()` | Get a type-erased `Recipient` | | `actor_ref.context()` | Get a `Context` (for timer setup, etc.) | | `actor_ref.join()` | Wait until the actor has fully stopped (tasks: `.await`, threads: blocking) | @@ -565,14 +565,61 @@ Rules (the [design](design/deferred-replies.md) has the full contract): - `PendingReplies` is a bounded table of tokens by key (purges terminal entries on insert; `InsertError::{Duplicate, Full}` hands the token back). +**Answering with a reason.** The framework's errors (`ReplyDropped`, +`ActorStopped`, `Overloaded`, `RequestTimeout`) say what happened to the +request, not why your actor could not serve it. To carry your own reason, make +the reply type a `Result`, as `gen_server` replies `{error, Reason}`: + +```rust +impl Message for GetPeer { type Result = Result; } + +// a ReplyTo> answers either way, exactly once: +token.reply(Err(PeerError::Disconnected)); +// the caller sees Ok(Err(PeerError::Disconnected)): the request worked, +// the answer is "no"; Err(ActorError::..) stays the transport's. +``` + +**Send-only handlers.** A manual `impl Handler` for a message whose +`Result` is `()` can leave out the reply parameter with `#[send_handler]` on +the impl; the impl answers `()` once `handle` returns (also after an early +`return`), so a `request` gets `Ok(())` rather than `ReplyDropped`. This is +the manual form of a `#[send_handler]` method in an `#[actor]` block: + +```rust +#[send_handler] +impl Handler for Poller { + async fn handle(&mut self, _msg: Tick, ctx: &Context) { + self.poll().await; + } +} +``` + **Pipe-to-self jobs** run work owned by the actor and deliver the result as a message in a later turn (OTP's `Task.Supervisor.async_nolink` plus -`handle_info`): +`handle_info`). A gossip verdict checked off the actor, with a deadline, that +reports exactly one outcome per object even when the actor is at a bound: ```rust -// CPU offload on an owned native thread, with a deadline: -let opts = JobOptions { deadline: Some(Instant::now() + VERDICT_DEADLINE), ..Default::default() }; -ctx.pipe_blocking_with(opts, move |_job| check(&object), move |res| Verdict { id, res })?; +#[send_handler] +async fn on_gossip(&mut self, msg: Gossip, ctx: &Context) { + // Reserve first: on refusal nothing has been moved into the job yet. + let Ok(permit) = ctx.reserve_blocking_job() else { + return self.report(msg.id, Verdict::Ignore(Overloaded)); + }; + let Gossip { id, object } = msg; + let opts = JobOptions::default().deadline_after(VERDICT_DEADLINE); + permit.pipe_blocking_with(opts, move |_job| check(&object), move |res| Checked { id, res }); +} + +#[send_handler] +async fn on_checked(&mut self, msg: Checked, _ctx: &Context) { + let verdict = match msg.res.into_result() { + Ok(outcome) => outcome, + Err(JobError::DeadlineExceeded) => Verdict::Ignore(TimedOut), + Err(_) => Verdict::Reject(CheckFailed), // a panic in `check` + }; + self.report(msg.id, verdict); +} // Delegation: the obligation and its credit stay here. let token = reply.defer()?; @@ -581,21 +628,67 @@ ctx.pipe_reply(token, async move { downstream?.await }, |res, token| Delegated { ``` - `done(outcome)` runs in the owner's turn; `JobOutcome::Err` carries - `JobError::Panicked` or `JobError::DeadlineExceeded` (the work keeps running - and keeps its credit). + `JobError::Panicked` or `JobError::DeadlineExceeded`. `JobOutcome`, + `JobError` and `PipeErrorKind` are `#[non_exhaustive]`: match with a + wildcard arm (or `outcome.into_result()`). +- `ctx.pipe*` reject at once at a bound (`PipeError`, never waiting) and drop + `work` and `done`. When the caller must still produce a result, + `ctx.reserve_job()` / `ctx.reserve_blocking_job()` take the admission + first (owner running, a credit, and for blocking jobs a native slot) and + return a permit whose `pipe*` cannot fail. `pipe_reply*` errors hand the + token back. - `pipe_keyed` / `JobOptions::key` cancel the previous job with that key and fence its queued messages; `JobHandle::cancel` fences one job. - `JobCtx::emit_blocking` (and `emit` for async jobs) sends fenced messages, bounded by `JobOptions::emit_credits`; the terminal message is always last. -- Each unlinked job holds one credit; native jobs also count against - `Actor::MAX_NATIVE_JOBS`. Every bound rejects at once (`PipeError`), never - waits. `pipe_reply*` errors hand the token back. + `JobCtx::emit_at(at, msg)` (`emit_at_blocking` in blocking jobs) delivers + at `at`, fenced and credit-accounted the same way: it never reaches a newer + job's turn or a newer instance, it is dropped if the job is cancelled or + replaced first, and the job's terminal message waits for it. +- **Job deadlines are hard.** When `JobOptions::deadline` (or + `deadline_after(d)`) passes, `done` gets `DeadlineExceeded` at once, later + emits fail with `JobGone` and the work's eventual result is dropped. The + work itself keeps running (cooperative cancellation) and keeps its credit + and native slot until it returns. For a **soft deadline** (stop starting + new work, finish and deliver what is in hand), leave `JobOptions::deadline` + unset and check the time in the work: + + ```rust + let stop_starting_at = Instant::now() + SOFT_DEADLINE; + ctx.pipe_blocking_with(JobOptions::default(), move |job| { + let mut done = Vec::new(); + for item in batch { + if job.is_cancelled() || Instant::now() >= stop_starting_at { break; } + let out = prove(item); // a started item finishes + let _ = job.emit_blocking(Proven(out.id)); // and is delivered + done.push(out); + } + Summary::from(done) + }, |res| BatchDone { res })?; + ``` + + `spawned_concurrency::Instant` (also `tasks::Instant`, `threads::Instant`) + is the deadline clock: `std::time::Instant`, or the simulated clock under + `--cfg spawned_sim`, so code need not depend on `spawned-rt`. +- **Bounds.** Each unlinked job holds one `MAX_OUTSTANDING` credit; blocking + jobs also count against `Actor::MAX_NATIVE_JOBS` (default + `min(4, available_parallelism)`). Set both in the macro, + `#[actor(max_native_jobs = 64, max_outstanding = 4096)]` (or as `Actor` + constants in a manual impl), and override them per supervised child with + `ChildSpec::max_native_jobs(n)` / `ChildSpec::max_outstanding(n)`. +- **Executors.** In `tasks` mode async jobs run on the owner's runtime and + blocking jobs on Tokio's blocking pool (`spawn_blocking` of the actor's + group runtime, or the current runtime), still bounded by the native + budget; a runtime that is dropped waits for blocking jobs still running, + as with any `spawn_blocking`. `threads` mode offers `pipe_blocking_with` + and `pipe_reply_blocking_with` on owned native threads (a reservation + spawns the thread, parked until the job starts). - Owner exit cancels every job; after death the jobs are joined, bounded by - `DEFAULT_STOP_DEADLINE`, and stragglers are reported (`unfinished_at_stop`). -- `threads` mode offers `pipe_blocking_with` and `pipe_reply_blocking_with` - (one owned native thread per job). + the cleanup deadline, and stragglers are reported (`unfinished_at_stop`). -Test a manual handler directly with `spawned_test::reply_pair()`. Run the +Test a manual handler directly with `spawned_test::reply_pair()`, and run +code inside a live actor's turn (to start a job or inspect state) with +`spawned_test::tasks::in_actor` / `spawned_test::threads::in_actor`. Run the [deferred replies example](../examples/capabilities/src/bin/deferred_replies.rs). ## Message Trait @@ -1230,11 +1323,12 @@ waits retain their existing first-poll timeout behavior. Synchronous protocol requests are observed by their blocking request; `Response::ready` does not count that result a second time. Manually constructed ready results are not requests. -`request_raw` returns the existing mode-specific raw receiver. It cannot observe -custom timeouts or how the caller waits without changing that API. Such requests +`request_raw` returns a `ReplyReceiver`; it does not observe how the caller +waits on it, so it is not a caller-observed request. Such requests still contribute message, handler, queue and reply-delivery metrics, but only a framework request/Response wrapper contributes caller-outcome/latency metrics. -Reply delivery means a successful channel send, not proof the caller consumed it. +Reply delivery means the reply was stored in the request's slot, not proof the +caller consumed it. Dropping/timing out a caller does not cancel accepted actor work. ### Collection and exporting limits @@ -1364,10 +1458,12 @@ These helpers fail with a report instead of hanging the test: | `wait_for_generation(&child, after, deadline)` | a supervised child does not reach a new generation, printing its snapshot | | `shutdown_supervisor(&supervisor, deadline)` | the supervisor report is incomplete, listing its children | | `shutdown_system(&system, deadline)` / `TestSystem::shutdown` | the `ActorSystem` report is incomplete, listing unfinished groups, actors, exits and failures | +| `in_actor(&actor, \|state, ctx\| ..)` | the actor is gone or the closure panicked; otherwise runs the closure in one of the actor's turns (after earlier sends) and returns its result, so a test can reach `ctx` (start a job, defer a reply) or inspect state without an ad hoc message or a `cfg(test)` bypass | `TestSystem::new()` owns an `ActorSystem` with a two-worker async group named `test`, and starts actors and mocks on it. The `threads` module has the -blocking versions of `settle`, `exit_within`, `join_within` and `wait_until`. +blocking versions of `settle`, `exit_within`, `join_within`, `wait_until` and +`in_actor`. There is no generic "actor is idle" check: an actor's mailbox can receive from any sender at any time, so `settle` fences the senders the test controls. Every async wait in `tasks` (`expect`, `expect_no_message`, `wait_until`, diff --git a/docs/TESTING.md b/docs/TESTING.md index 1ea47649..b551629f 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -538,7 +538,9 @@ abandon, close, defer at the bound, linked jobs, parked receivers); `registry_turn_loom` the in-turn drop commit against turn end, exit and death; `job_core_loom` job admission and rollback at the credit and native bounds, keyed replacement of a completed job, emits against the terminal notification -and cancel, native completion against the owner's cleanup join, and a job's +and cancel, a delayed emit (`emit_at`) firing against the work's finish (the +terminal held until it fires) and a cancel, native completion against the +owner's cleanup join, and a job's final destruction after a `Weak` upgrade (modelled `Arc`/`Weak`; the destruction takes the job set's locks, so running it under one is a Loom deadlock). Each model was checked against a planted bug. diff --git a/examples/capabilities/src/bin/deferred_replies.rs b/examples/capabilities/src/bin/deferred_replies.rs index 1c172360..47d6b52e 100644 --- a/examples/capabilities/src/bin/deferred_replies.rs +++ b/examples/capabilities/src/bin/deferred_replies.rs @@ -142,10 +142,12 @@ impl Verifier { #[send_handler] async fn verdict(&mut self, msg: Verdict, _: &Context) { - self.verdicts.push(match msg.res { - JobOutcome::Ok(valid) => format!("{}: valid={valid}", msg.id), - JobOutcome::Err(JobError::DeadlineExceeded) => format!("{}: timed out", msg.id), - JobOutcome::Err(JobError::Panicked(m)) => format!("{}: panicked {m}", msg.id), + // `JobOutcome` and `JobError` are `#[non_exhaustive]`. + self.verdicts.push(match msg.res.into_result() { + Ok(valid) => format!("{}: valid={valid}", msg.id), + Err(JobError::DeadlineExceeded) => format!("{}: timed out", msg.id), + Err(JobError::Panicked(m)) => format!("{}: panicked {m}", msg.id), + Err(other) => format!("{}: {other}", msg.id), }); } diff --git a/macros/src/lib.rs b/macros/src/lib.rs index a187a6e7..4850e4dd 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1116,6 +1116,18 @@ fn is_actor_method_attribute(attr: &Attribute) -> bool { /// impl ChatUser { /* ... */ } /// ``` /// +/// # Deferred-reply bounds +/// +/// `max_outstanding = N` sets `Actor::MAX_OUTSTANDING` (deferred credits per +/// instance) and `max_native_jobs = N` sets `Actor::MAX_NATIVE_JOBS` (blocking +/// jobs running at once). Both take any constant expression and combine with +/// `protocol`; a supervised child can still override them per `ChildSpec`: +/// +/// ```ignore +/// #[actor(protocol = P2PProtocol, max_native_jobs = 64, max_outstanding = 4096)] +/// impl P2PServer { /* ... */ } +/// ``` +/// /// # Handler Attributes /// /// | Attribute | Use for | @@ -1215,7 +1227,10 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { let self_ty = &impl_block.self_ty; let (impl_generics, _, where_clause) = impl_block.generics.split_for_impl(); - // --- Parse named parameters from #[actor(protocol = X)] or #[actor(protocol(X, Y))] --- + // --- Parse named parameters: #[actor(protocol = X)], #[actor(protocol(X, Y))], + // and the deferred-reply bounds `max_outstanding = N`, `max_native_jobs = N` --- + let mut max_outstanding: Option = None; + let mut max_native_jobs: Option = None; let bridge_traits: Vec = if attr.is_empty() { Vec::new() } else { @@ -1235,13 +1250,23 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { let mut protocols = Vec::new(); while !input.is_empty() { let key: Ident = input.parse()?; - if key != "protocol" { + if key == "max_outstanding" || key == "max_native_jobs" { + let _: syn::Token![=] = input.parse()?; + let value: syn::Expr = input.parse()?; + let slot = if key == "max_outstanding" { + &mut max_outstanding + } else { + &mut max_native_jobs + }; + if slot.replace(value).is_some() { + return Err(syn::Error::new(key.span(), format!("`{key}` given twice"))); + } + } else if key != "protocol" { return Err(syn::Error::new( key.span(), - "unknown parameter, expected `protocol`", + "unknown parameter, expected `protocol`, `max_outstanding` or `max_native_jobs`", )); - } - if input.peek(syn::Token![=]) { + } else if input.peek(syn::Token![=]) { // protocol = TraitName let _: syn::Token![=] = input.parse()?; protocols.push(protocol_name(input)?); @@ -1638,9 +1663,17 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { quote! { #[doc = #doc_body] } }; + let bound_consts = { + let outstanding = max_outstanding.map(|n| quote! { const MAX_OUTSTANDING: usize = #n; }); + let native = max_native_jobs + .map(|n| quote! { const MAX_NATIVE_JOBS: ::core::option::Option = ::core::option::Option::Some(#n); }); + quote! { #outstanding #native } + }; + let actor_impl = quote! { #protocol_doc impl #impl_generics Actor for #self_ty #where_clause { + #bound_consts #(#lifecycle_methods)* } }; @@ -1678,6 +1711,95 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { // --- #[sim_test] --- +/// Send-only form of a manual `Handler` impl: write `handle` without the +/// reply parameter, and the impl answers `()` once `handle` returns. +/// +/// ```ignore +/// #[send_handler] +/// impl Handler for Poller { +/// async fn handle(&mut self, _msg: Tick, ctx: &Context) { +/// self.poll().await; +/// } +/// } +/// ``` +/// +/// expands to the full `handle(&mut self, msg, reply: PendingReply<'_, ()>, +/// ctx)` whose body runs yours and then calls `reply.reply(())`, also after +/// an early `return`. The automatic `()` is a real reply, as with +/// `#[send_handler]` methods in an `#[actor]` block: a `request` gets +/// `Ok(())`, not `ReplyDropped`, and a panic in the body answers +/// `ActorStopped` as for any handler. The message's `Result` must be `()`. +/// Works for `tasks` (`async fn`) and `threads` (`fn`) handlers. +#[proc_macro_attribute] +pub fn send_handler(attr: TokenStream, item: TokenStream) -> TokenStream { + if !attr.is_empty() { + return syn::Error::new( + proc_macro2::Span::call_site(), + "#[send_handler] takes no arguments", + ) + .to_compile_error() + .into(); + } + let mut impl_block = parse_macro_input!(item as ItemImpl); + let is_handler = impl_block + .trait_ + .as_ref() + .and_then(|(_, path, _)| path.segments.last()) + .is_some_and(|segment| segment.ident == "Handler"); + if !is_handler { + return syn::Error::new_spanned( + &impl_block.self_ty, + "#[send_handler] on an item applies to an `impl Handler for Actor` block; \ + inside an #[actor] block, put it on the handler method instead", + ) + .to_compile_error() + .into(); + } + let Some(method) = impl_block.items.iter_mut().find_map(|item| match item { + ImplItem::Fn(method) if method.sig.ident == "handle" => Some(method), + _ => None, + }) else { + return syn::Error::new_spanned( + &impl_block.self_ty, + "#[send_handler] impl must define `handle`", + ) + .to_compile_error() + .into(); + }; + if method.sig.inputs.len() != 3 { + return syn::Error::new_spanned( + &method.sig, + "#[send_handler] `handle` takes (&mut self, msg: M, ctx: &Context), without the reply parameter", + ) + .to_compile_error() + .into(); + } + if let ReturnType::Type(_, ty) = &method.sig.output { + if !is_unit_type(ty) { + return syn::Error::new_spanned(ty, "#[send_handler] `handle` returns nothing") + .to_compile_error() + .into(); + } + } + let reply = format_ident!("__spawned_reply"); + let reply_param: FnArg = syn::parse_quote! { + #reply: spawned_concurrency::reply::PendingReply<'_, ()> + }; + method.sig.inputs.insert(2, reply_param); + method.sig.output = ReturnType::Default; + let body = &method.block; + let run = if method.sig.asyncness.is_some() { + quote! { async move #body.await } + } else { + quote! { (move || #body)() } + }; + method.block = syn::parse_quote! {{ + let () = #run; + let _ = #reply.reply(()); + }}; + quote! { #impl_block }.into() +} + /// Seeded async test: on Tokio in normal builds, over a seed range on the /// deterministic simulator under `--cfg spawned_sim` (see /// `spawned_rt::tasks::SimTest`). Accepts `#[tokio::test]`'s `flavor` and diff --git a/testing/src/tasks.rs b/testing/src/tasks.rs index a7c84ba2..fcb938f6 100644 --- a/testing/src/tasks.rs +++ b/testing/src/tasks.rs @@ -252,6 +252,39 @@ where } } +/// Run `f` inside one of `actor`'s turns, with its state and `Context`, and +/// return its result: a test reaches `ctx` (to start a job, defer a reply, +/// arm a timer) or inspects state without an ad hoc message or a +/// `cfg(test)` bypass. Messages sent to `actor` earlier are handled first. +/// Panics if the actor is gone or `f` panicked (which crashes the actor, as +/// any handler panic does). +/// +/// ```ignore +/// let handle = in_actor(&server, |server, ctx| server.start_checks(ctx, id)).await; +/// ``` +#[track_caller] +pub fn in_actor(actor: &ActorRef, f: F) -> impl Future + '_ +where + A: Actor, + R: Send + 'static, + F: FnOnce(&mut A, &Context) -> R + Send + 'static, +{ + let at = Location::caller(); + async move { + let reply = match actor.__in_turn(f) { + Ok(rx) => rx.await, + Err(error) => Err(error), + }; + reply.unwrap_or_else(|error| { + panic!( + "in_actor on {:?} failed: {error}; exit reason {:?} (called at {at})", + actor.id(), + actor.exit_reason() + ) + }) + } +} + /// Wait for `handle` to exit, or panic after `deadline` instead of hanging. /// Works for actors of either mode. #[track_caller] diff --git a/testing/src/threads.rs b/testing/src/threads.rs index 51569498..edc25747 100644 --- a/testing/src/threads.rs +++ b/testing/src/threads.rs @@ -169,6 +169,29 @@ where } } +/// Run `f` inside one of `actor`'s turns, with its state and `Context`, and +/// return its result: a test reaches `ctx` (to start a job, defer a reply, +/// arm a timer) or inspects state without an ad hoc message or a +/// `cfg(test)` bypass. Messages sent to `actor` earlier are handled first. +/// Panics if the actor is gone or `f` panicked (which crashes the actor, as +/// any handler panic does). +#[track_caller] +pub fn in_actor(actor: &ActorRef, f: F) -> R +where + A: Actor, + R: Send + 'static, + F: FnOnce(&mut A, &Context) -> R + Send + 'static, +{ + let reply = actor.__in_turn(f).and_then(|rx| rx.recv()); + reply.unwrap_or_else(|error| { + panic!( + "in_actor on {:?} failed: {error}; exit reason {:?}", + actor.id(), + actor.exit_reason() + ) + }) +} + /// Block until `handle` exits, or panic after `deadline` instead of hanging. /// /// Polls every millisecond: `ChildHandle` has no blocking wait with a diff --git a/testing/tests/tasks_kit.rs b/testing/tests/tasks_kit.rs index 5520fff9..cba615f5 100644 --- a/testing/tests/tasks_kit.rs +++ b/testing/tests/tasks_kit.rs @@ -609,3 +609,31 @@ async fn direct_kill_and_shutdown_stop_trapping_targets_without_killing_trapping assert_eq!(join_within(&observer, WAIT).await, ExitReason::Normal); } } + +/// `in_actor` runs a closure in one of the actor's turns, after the messages +/// sent before it, with the state and the `Context`. +#[tokio::test] +async fn in_actor_runs_a_closure_in_a_turn() { + struct Tally(u32); + #[actor] + impl Tally { + #[send_handler] + async fn add(&mut self, msg: AddOne, _: &Context) { + self.0 += msg.0; + } + } + struct AddOne(u32); + impl Message for AddOne { + type Result = (); + } + let tally = Tally(0).start(); + tally.send(AddOne(2)).unwrap(); + let (seen, own) = spawned_test::tasks::in_actor(&tally, |state, ctx| { + state.0 += 1; + (state.0, ctx.id()) + }) + .await; + assert_eq!((seen, own), (3, tally.id())); + tally.context().stop(); + tally.join().await; +} diff --git a/testing/tests/threads_kit.rs b/testing/tests/threads_kit.rs index 4727d4f3..07518fa5 100644 --- a/testing/tests/threads_kit.rs +++ b/testing/tests/threads_kit.rs @@ -428,3 +428,30 @@ fn direct_kill_and_shutdown_stop_trapping_targets_without_killing_trapping_peers assert_eq!(join_within(&observer, WAIT), ExitReason::Normal); } } + +/// `in_actor` runs a closure in one of the actor's turns, after the messages +/// sent before it, with the state and the `Context`. +#[test] +fn native_in_actor_runs_a_closure_in_a_turn() { + struct Tally(u32); + #[actor] + impl Tally { + #[send_handler] + fn add(&mut self, msg: AddOne, _: &Context) { + self.0 += msg.0; + } + } + struct AddOne(u32); + impl Message for AddOne { + type Result = (); + } + let tally = Tally(0).start(); + tally.send(AddOne(2)).unwrap(); + let (seen, own) = spawned_test::threads::in_actor(&tally, |state, ctx| { + state.0 += 1; + (state.0, ctx.id()) + }); + assert_eq!((seen, own), (3, tally.id())); + tally.context().stop(); + tally.join(); +} From 576d7f3f730ac94faa5a803c7fb7c8e0cbe04cfb Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 03:01:12 -0300 Subject: [PATCH 090/208] feat(exit): client-port feedback: error classification, stopped() skip log, exit_pending From porting ethrex and ethlambda to the 0.8 stack: - ActorError::{target_gone, is_timeout, is_transient, exit_reason}, so clients need no exhaustive match on the #[non_exhaustive] enum; unit tests enumerate every variant (adding one fails to compile until listed) - a skipped stopped() (kill, untrapped signal or link notice, parent shutdown of a non-trapping actor, runtime drop) logs event = "stopped_skipped" at debug with actor, type, cause and reason - Context::exit_pending() in both modes: exit began, or a parent exit is queued for a trapping actor (set under the request lock before the envelope is queued); a trapped peer Exit is not pending; advisory - canary_feedback: let the system kill at grace land before releasing stopped() (same E3 race as the earlier fixes) - GUIDE (when stopped() does not run, error helpers), MIGRATION, CHANGELOG --- CHANGELOG.md | 8 + concurrency/src/actor_cleanup.rs | 33 ++- concurrency/src/error.rs | 151 +++++++++++- concurrency/src/link.rs | 75 ++++++ concurrency/src/tasks/actor.rs | 42 +++- concurrency/src/threads/actor.rs | 42 +++- concurrency/tests/canary_feedback.rs | 3 + concurrency/tests/otp_exit_signals.rs | 339 ++++++++++++++++++++++++++ docs/GUIDE.md | 47 +++- docs/MIGRATION.md | 26 +- 10 files changed, 744 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c98f1bc4..fdfdb7c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,14 @@ (both modes), all accepted, and eleven edited traces (stopped() after a kill, `Kill` for `Killed`, publication before D, real reason instead of `NoProc`, notices after unlink/demonitor, ...) are rejected. + Client feedback (ethrex, ethlambda): `ActorError` classification helpers + `target_gone()`, `is_timeout()`, `is_transient()` and `exit_reason()`, so + clients need no exhaustive match on the `#[non_exhaustive]` enum; a + skipped `stopped()` logs `event = "stopped_skipped"` at `debug` (actor, + cause `kill`/`exit_signal`/`runtime_drop`, reason); + `Context::exit_pending()` (both modes) tells a trapping actor that its exit + began or a parent exit is queued behind the current message, so it can + refuse late work (advisory, read-only, no per-message cost). Remaining: Loom and simulator coverage, GUIDE and examples. - **Breaking (0.8), in progress:** deferred replies and pipe-to-self diff --git a/concurrency/src/actor_cleanup.rs b/concurrency/src/actor_cleanup.rs index f1fb9c39..4343abf3 100644 --- a/concurrency/src/actor_cleanup.rs +++ b/concurrency/src/actor_cleanup.rs @@ -15,6 +15,25 @@ pub(crate) fn log_safely(log: impl FnOnce()) { } } +/// `stopped()` did not run for this exit (decision 2: a kill, an untrapped +/// exit signal or link notice, or the runtime dropping the actor). Logged at +/// `debug` for every skip, whether or not the actor overrides `stopped()` +/// (there is no cheap way to know): an application whose cleanup must run +/// traps exits (`ctx.trap_exit(true)`) so a parent's shutdown runs it. +pub(crate) fn log_stopped_skipped( + actor: ActorId, + actor_type: &'static str, + cause: &'static str, + reason: Option<&ExitReason>, +) { + let reason = reason.cloned().unwrap_or(ExitReason::Killed); + log_safely(|| { + tracing::debug!(event = "stopped_skipped", actor = %actor, actor_type, cause, + reason = %reason, "stopped() skipped: the exit was not begun by the actor, \ + an administrative stop or its parent"); + }); +} + /// Constructed before dispatch, so even a never-polled actor has cleanup. /// [`die`](Self::die) is the death point D; dropping it is publication. pub(crate) struct ActorCleanup { @@ -117,13 +136,21 @@ impl Drop for ActorCleanup { Some(reason) => reason, None => { let id = self.id.raw(); - trace::life_section(id, || { - let reason = crate::link::lock_request(&self.cause).runtime_drop(); + let mut skipped = false; + let reason = trace::life_section(id, || { + let mut request = crate::link::lock_request(&self.cause); + skipped = !request.hook_begun(); + let reason = request.runtime_drop(); + drop(request); trace::life_emit(id, || { trace::LifeEvent::new("Drop", id).x(trace::life_reason(&reason)) }); self.close(reason) - }) + }); + if skipped { + log_stopped_skipped(self.id, "", "runtime_drop", Some(&reason)); + } + reason } }; if let Some(message) = self.failures.as_ref().and_then(|f| f.first()) { diff --git a/concurrency/src/error.rs b/concurrency/src/error.rs index ed133ee8..5a012aa0 100644 --- a/concurrency/src/error.rs +++ b/concurrency/src/error.rs @@ -29,8 +29,17 @@ pub enum ExitReason { impl ExitReason { /// Returns `true` for exit reasons that should trigger a restart - /// of `Transient` or `Permanent` children. - /// `Shutdown` is considered normal (no restart) matching Erlang semantics. + /// of `Transient` or `Permanent` children, and that end a non-trapping + /// linked actor (OTP: every reason except `normal` and `shutdown`). + /// + /// | Reason | `is_abnormal()` | + /// |---|---| + /// | `Normal` | `false` | + /// | `Shutdown` | `false` (an ordered stop, as in Erlang) | + /// | `Panic(_)` | `true` | + /// | `Kill` | `true` | + /// | `Killed` | `true` (OTP `killed`: a kill, a supervisor's kill escalation or a runtime drop) | + /// | `NoProc` | `true` (OTP `noproc`: the target was already gone) | pub fn is_abnormal(&self) -> bool { match self { ExitReason::Normal | ExitReason::Shutdown => false, @@ -108,6 +117,53 @@ pub enum ActorError { Overloaded, } +/// Classification helpers, so callers need not match every variant (the enum +/// is `#[non_exhaustive]`; a variant added later is classified here). +/// +/// | Variant | `target_gone` | `is_timeout` | `is_transient` | `exit_reason` | +/// |---|---|---|---|---| +/// | `ActorStopped(r)` | `true` | `false` | `false` | `Some(r)` | +/// | `NoProc` | `true` | `false` | `false` | `None` | +/// | `RequestTimeout` | `false` | `true` | `true` | `None` | +/// | `ReplyDropped` | `false` | `false` | `false` | `None` | +/// | `Overloaded` | `false` | `false` | `true` | `None` | +impl ActorError { + /// The target actor is gone: it died before answering + /// (`ActorStopped`) or was already past its death point (`NoProc`). + /// Retrying the same `ActorRef` cannot succeed; look the actor up again + /// (a supervisor may have restarted it under a new reference). + pub fn target_gone(&self) -> bool { + matches!(self, ActorError::ActorStopped(_) | ActorError::NoProc) + } + + /// The request was not answered within its timeout + /// (`RequestTimeout`). The actor may still be running and may still + /// handle the request: a timeout does not cancel it. + pub fn is_timeout(&self) -> bool { + matches!(self, ActorError::RequestTimeout) + } + + /// Retrying the same target may succeed: the actor was alive but did + /// not answer in time (`RequestTimeout`) or rejected the request at a + /// bound (`Overloaded`). `false` for a gone target, and for + /// `ReplyDropped`, where the actor is alive but its handler chose not to + /// answer (a decision or a bug, not a load condition): a retry would + /// usually get the same result. + pub fn is_transient(&self) -> bool { + matches!(self, ActorError::RequestTimeout | ActorError::Overloaded) + } + + /// The final exit reason of an actor that died before answering + /// (`ActorStopped`). `None` otherwise, including `NoProc` (the target + /// was already gone; its reason is not known to the caller). + pub fn exit_reason(&self) -> Option<&ExitReason> { + match self { + ActorError::ActorStopped(reason) => Some(reason), + _ => None, + } + } +} + impl From> for ActorError { fn from(_value: spawned_rt::threads::mpsc::SendError) -> Self { Self::NoProc @@ -132,14 +188,93 @@ mod tests { assert_eq!(ActorError::NoProc.to_string(), "noproc"); } + /// Every `ExitReason`; the exhaustive match (allowed inside the crate) + /// makes adding a variant fail to compile until it is listed here. + fn all_exit_reasons() -> Vec { + let all = vec![ + ExitReason::Normal, + ExitReason::Shutdown, + ExitReason::Panic("boom".into()), + ExitReason::Kill, + ExitReason::Killed, + ExitReason::NoProc, + ]; + for reason in &all { + match reason { + ExitReason::Normal + | ExitReason::Shutdown + | ExitReason::Panic(_) + | ExitReason::Kill + | ExitReason::Killed + | ExitReason::NoProc => {} + } + } + all + } + + /// Every `ActorError`, with the same compile-time completeness check. + fn all_actor_errors() -> Vec { + let all = vec![ + ActorError::ActorStopped(ExitReason::Killed), + ActorError::NoProc, + ActorError::RequestTimeout, + ActorError::ReplyDropped, + ActorError::Overloaded, + ]; + for error in &all { + match error { + ActorError::ActorStopped(_) + | ActorError::NoProc + | ActorError::RequestTimeout + | ActorError::ReplyDropped + | ActorError::Overloaded => {} + } + } + all + } + #[test] fn exit_reason_is_abnormal_classification() { - assert!(!ExitReason::Normal.is_abnormal()); - assert!(!ExitReason::Shutdown.is_abnormal()); - assert!(ExitReason::Panic("boom".into()).is_abnormal()); - assert!(ExitReason::Kill.is_abnormal()); - assert!(ExitReason::Killed.is_abnormal()); - assert!(ExitReason::NoProc.is_abnormal()); + for reason in all_exit_reasons() { + let expected = match reason { + ExitReason::Normal | ExitReason::Shutdown => false, + ExitReason::Panic(_) + | ExitReason::Kill + | ExitReason::Killed + | ExitReason::NoProc => true, + }; + assert_eq!(reason.is_abnormal(), expected, "{reason}"); + } + } + + #[test] + fn actor_error_classification_covers_every_variant() { + for error in all_actor_errors() { + // (target_gone, is_timeout, is_transient, exit_reason) + let expected = match &error { + ActorError::ActorStopped(r) => (true, false, false, Some(r.clone())), + ActorError::NoProc => (true, false, false, None), + ActorError::RequestTimeout => (false, true, true, None), + ActorError::ReplyDropped => (false, false, false, None), + ActorError::Overloaded => (false, false, true, None), + }; + let actual = ( + error.target_gone(), + error.is_timeout(), + error.is_transient(), + error.exit_reason().cloned(), + ); + assert_eq!(actual, expected, "{error:?}"); + } + } + + #[test] + fn actor_stopped_carries_every_reason() { + for reason in all_exit_reasons() { + let error = ActorError::ActorStopped(reason.clone()); + assert!(error.target_gone() && !error.is_transient() && !error.is_timeout()); + assert_eq!(error.exit_reason(), Some(&reason)); + } } #[test] diff --git a/concurrency/src/link.rs b/concurrency/src/link.rs index aa9e8d9b..517fefa0 100644 --- a/concurrency/src/link.rs +++ b/concurrency/src/link.rs @@ -62,6 +62,11 @@ pub(crate) struct ExitRequest { /// Held signal envelopes become `Exit`s (the actor trapped when /// `started()` returned); otherwise they are inert. held_trapped: bool, + /// A parent signal is among the held ones. + held_parent: bool, + /// A parent exit is queued behind earlier messages of a trapping actor + /// (E3): when dispatched it ends the actor whatever its reason. + parent_exit_queued: bool, } impl ExitRequest { @@ -72,6 +77,25 @@ impl ExitRequest { pub(crate) fn is_dead(&self) -> bool { self.dead } + /// `Context::exit_pending`: exit has begun (a reason is requested, which + /// includes a kill and the dispatch of a parent exit) or a parent exit + /// is queued for a trapping actor. Held phase-0 signals count once they + /// are resolved when `started()` returns. + pub(crate) fn exit_pending(&self) -> bool { + self.reason.is_some() || self.parent_exit_queued + } + /// What began an exit that skips `stopped()`, for its debug log. + pub(crate) fn skip_cause(&self) -> &'static str { + if self.killed { + "kill" + } else { + "exit_signal" + } + } + /// `stopped()` has begun (or was skipped by the run loop). + pub(crate) fn hook_begun(&self) -> bool { + self.hook_begun + } /// A self-exit or an administrative stop: the first requested reason /// wins. Returns whether the run loop must be woken. pub(crate) fn request(&mut self, reason: ExitReason) -> bool { @@ -148,6 +172,10 @@ impl ExitRequest { self.starting = false; let held = std::mem::take(&mut self.held); self.held_trapped = trapping; + if trapping && std::mem::take(&mut self.held_parent) { + // The held parent envelope is dispatched as a parent exit. + self.parent_exit_queued = true; + } if !trapping { if let Some(reason) = held.into_iter().find(|r| *r != ExitReason::Normal) { return self.untrapped(reason); @@ -250,11 +278,17 @@ fn apply_signal( }); // Held in arrival order, and as an envelope in its mailbox place. request.held.push(exit.reason.clone()); + request.held_parent |= from_parent; let _ = (target.send_held)(exit, from_parent); return; } // The trap flag is read when the signal arrives. if target.trap.load(Ordering::Acquire) { + if from_parent { + // Set before the envelope is queued, under the request lock, so + // `exit_pending` reports it to every handler that runs before it. + request.parent_exit_queued = true; + } drop(request); let send = if from_parent { &target.send_parent_exit @@ -827,6 +861,47 @@ mod tests { assert_eq!(lock_request(&t.request).reason(), Some(&ExitReason::Killed)); } #[test] + fn exit_pending_counts_begun_exits_and_queued_parent_exits() { + let seen = Arc::new(AtomicUsize::new(0)); + let from = ActorId::next(); + let pending = |t: &SignalTarget| lock_request(&t.request).exit_pending(); + // Trapping: a peer's Exit is not pending; a parent's signal is. + let t = target(&seen); + t.trap.store(true, Ordering::Release); + deliver_signal(&t, from, ExitReason::Shutdown, Origin::Link, false); + assert!(!pending(&t)); + deliver_signal(&t, from, ExitReason::Normal, Origin::Direct, true); + assert!(pending(&t)); + assert_eq!(lock_request(&t.request).reason(), None); + // Non-trapping: an untrapped signal begins exit; Normal is ignored. + let t = target(&seen); + deliver_signal(&t, from, ExitReason::Normal, Origin::Direct, true); + assert!(!pending(&t)); + deliver_signal(&t, from, ExitReason::Shutdown, Origin::Direct, false); + assert!(pending(&t)); + // Phase 0: held signals count when started() returns. + let t = target(&seen); + lock_request(&t.request).begin_starting(); + deliver_signal(&t, from, ExitReason::Normal, Origin::Direct, true); + assert!(!pending(&t)); + lock_request(&t.request).end_starting(true); + assert!(pending(&t), "a held parent exit of a trapping actor"); + let t = target(&seen); + lock_request(&t.request).begin_starting(); + deliver_signal(&t, from, ExitReason::Normal, Origin::Direct, true); + deliver_signal(&t, from, ExitReason::Shutdown, Origin::Direct, false); + lock_request(&t.request).end_starting(false); + assert!(pending(&t), "a held untrapped signal begins exit"); + let t = target(&seen); + lock_request(&t.request).begin_starting(); + deliver_signal(&t, from, ExitReason::Normal, Origin::Direct, true); + lock_request(&t.request).end_starting(false); + assert!( + !pending(&t), + "a held Normal parent signal is ignored untrapped" + ); + } + #[test] fn reason_precedence_until_death() { // A late non-kill signal replaces a requested reason (decision 8); // a kill supersedes it and nothing replaces Killed. diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 78d02737..8624f5a4 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -829,6 +829,31 @@ impl Context { ); } + /// Whether this actor's exit has begun or a signal that will end it is + /// already pending, for a trapping actor that wants to refuse late work. + /// + /// `true` once: + /// - exit has begun: `ctx.stop()`, an administrative stop + /// ([`ChildHandle::stop`](crate::ChildHandle::stop)), a kill, an untrapped exit + /// signal or link notice, a panic; it stays `true` in `stopped()`; + /// - a parent exit is queued for this trapping actor (E3: the + /// supervisor's or system's shutdown, or a signal from its + /// `start_linked` parent). The parent exit is dispatched after the + /// messages queued before it, and handlers of those messages see + /// `true`. + /// + /// A trapped [`Exit`](crate::Exit) from a linked peer or a non-parent + /// `exit` is not a pending exit: `exit_received` decides. Signals that + /// arrive while `started()` runs count once `started()` returns, when + /// they are resolved. Without trapping, a pending signal has already + /// begun the exit, so this is also `true`. + /// + /// Advisory: a signal can arrive right after it returns `false`. It + /// takes the actor's exit-request lock; nothing is added per message. + pub fn exit_pending(&self) -> bool { + link::lock_request(&self.shared.linked_reason).exit_pending() + } + /// Control how exit signals (link notices and `exit` signals) are /// handled; read when each signal arrives (OTP `trap_exit`). /// @@ -1780,8 +1805,11 @@ async fn run_actor_lifecycle( ctx.shared.jobs.cancel_all(); // stopped() is skipped when a kill, an untrapped exit signal or link // notice began the exit (decision 2); one that already began finishes. - let run_hook = crate::lifecycle_trace::life_section(trace_id, || { - let run = link::lock_request(&request).begin_hook(); + let (run_hook, skipped) = crate::lifecycle_trace::life_section(trace_id, || { + let mut req = link::lock_request(&request); + let run = req.begin_hook(); + let skipped = (!run).then(|| (req.skip_cause(), req.reason().cloned())); + drop(req); crate::lifecycle_trace::life_emit(trace_id, || { crate::lifecycle_trace::LifeEvent::new("Hook", trace_id).x(if run { "run" @@ -1789,8 +1817,16 @@ async fn run_actor_lifecycle( "skip" }) }); - run + (run, skipped) }); + if let Some((cause, reason)) = skipped { + crate::actor_cleanup::log_stopped_skipped( + ctx.shared.id, + std::any::type_name::(), + cause, + reason.as_ref(), + ); + } if !run_hook { return; } diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 13689acb..35a27902 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -719,6 +719,31 @@ impl Context { link::unregister_link(self.id, &self.links, target.id(), target.links()); } + /// Whether this actor's exit has begun or a signal that will end it is + /// already pending, for a trapping actor that wants to refuse late work. + /// + /// `true` once: + /// - exit has begun: `ctx.stop()`, an administrative stop + /// ([`ChildHandle::stop`](crate::ChildHandle::stop)), a kill, an untrapped exit + /// signal or link notice, a panic; it stays `true` in `stopped()`; + /// - a parent exit is queued for this trapping actor (E3: the + /// supervisor's or system's shutdown, or a signal from its + /// `start_linked` parent). The parent exit is dispatched after the + /// messages queued before it, and handlers of those messages see + /// `true`. + /// + /// A trapped [`Exit`](crate::Exit) from a linked peer or a non-parent + /// `exit` is not a pending exit: `exit_received` decides. Signals that + /// arrive while `started()` runs count once `started()` returns, when + /// they are resolved. Without trapping, a pending signal has already + /// begun the exit, so this is also `true`. + /// + /// Advisory: a signal can arrive right after it returns `false`. It + /// takes the actor's exit-request lock; nothing is added per message. + pub fn exit_pending(&self) -> bool { + link::lock_request(&self.linked_reason).exit_pending() + } + /// Control how exit signals (link notices and `exit` signals) are /// handled; read when each signal arrives (OTP `trap_exit`). /// @@ -1269,8 +1294,11 @@ fn run_actor_receiving( ctx.jobs.cancel_all(); // stopped() is skipped when a kill, an untrapped exit signal or link // notice began the exit (decision 2); one that already began finishes. - let run_hook = crate::lifecycle_trace::life_section(trace_id, || { - let run = link::lock_request(&request).begin_hook(); + let (run_hook, skipped) = crate::lifecycle_trace::life_section(trace_id, || { + let mut req = link::lock_request(&request); + let run = req.begin_hook(); + let skipped = (!run).then(|| (req.skip_cause(), req.reason().cloned())); + drop(req); crate::lifecycle_trace::life_emit(trace_id, || { crate::lifecycle_trace::LifeEvent::new("Hook", trace_id).x(if run { "run" @@ -1278,8 +1306,16 @@ fn run_actor_receiving( "skip" }) }); - run + (run, skipped) }); + if let Some((cause, reason)) = skipped { + crate::actor_cleanup::log_stopped_skipped( + ctx.id, + std::any::type_name::(), + cause, + reason.as_ref(), + ); + } if run_hook { let stop_result = catch_unwind(AssertUnwindSafe(|| { fail_point!("actor.before_stopped"); diff --git a/concurrency/tests/canary_feedback.rs b/concurrency/tests/canary_feedback.rs index 2ca94dee..b31db1ca 100644 --- a/concurrency/tests/canary_feedback.rs +++ b/concurrency/tests/canary_feedback.rs @@ -439,6 +439,9 @@ async fn panicked_turn_is_cleared_before_stopped_hook() { .unwrap(); assert_eq!(detail.message_type, None); assert_eq!(detail.turn_age, None); + // Let the system's kill at grace land before stopped() returns: the + // report can be ready just before the kill is sent. + spawned_rt::tasks::sleep(Duration::from_millis(100)).await; release.send(()).unwrap(); assert!(report.wait(LIMIT).await.complete); // The system's kill at grace reached it inside stopped(), before its diff --git a/concurrency/tests/otp_exit_signals.rs b/concurrency/tests/otp_exit_signals.rs index 0a0597b2..b9b392ec 100644 --- a/concurrency/tests/otp_exit_signals.rs +++ b/concurrency/tests/otp_exit_signals.rs @@ -614,6 +614,250 @@ mod tasks_mode { assert!(seen.stopped()); } + // Context::exit_pending: a trapping supervised child sees the + // supervisor's queued parent exit in the handlers of messages queued + // before it; a trapped Exit from a peer is not a pending exit. + mod exit_pending { + use super::*; + use spawned_concurrency::supervision::{ChildSpec, SupervisorConfig}; + use spawned_concurrency::system::ActorSystem; + + type Log = Arc>>; + pub struct Late(Log); + impl Actor for Late { + async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); + self.0.lock().unwrap().push(("started", ctx.exit_pending())); + } + async fn exit_received(&mut self, _: Exit, ctx: &Context) { + self.0.lock().unwrap().push(("exit", ctx.exit_pending())); + } + async fn stopped(&mut self, ctx: &Context) { + self.0.lock().unwrap().push(("stopped", ctx.exit_pending())); + } + } + struct Probe; + impl Message for Probe { + type Result = (); + } + impl Handler for Late { + async fn handle( + &mut self, + _: Probe, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + self.0.lock().unwrap().push(("probe", ctx.exit_pending())); + reply.reply(()); + } + } + /// Wait (bounded) until a shutdown is pending, then record it. + struct AwaitShutdown; + impl Message for AwaitShutdown { + type Result = (); + } + impl Handler for Late { + async fn handle( + &mut self, + _: AwaitShutdown, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(3); + while !ctx.exit_pending() && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(5)).await; + } + self.0.lock().unwrap().push(("await", ctx.exit_pending())); + reply.reply(()); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn queued_work_sees_the_supervisors_pending_shutdown() { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("pending", 1).unwrap(); + let log: Log = Arc::default(); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let l = log.clone(); + let child = tree + .add( + ChildSpec::new("late", &group, move |_| { + let log = l.clone(); + async move { Ok::<_, String>(Late(log)) } + }) + // No kill escalation while the test waits. + .shutdown_timeout(LIMIT), + ) + .unwrap(); + let supervisor = tree.start().await.unwrap(); + let child = child.current().unwrap(); + // Normal operation, and a trapped Exit from outside: not pending. + child.request(Probe).await.unwrap(); + child.child_handle().exit(ExitReason::Shutdown); + child.request(Probe).await.unwrap(); + // Work queued before the supervisor's shutdown. + child.send(AwaitShutdown).unwrap(); + child.send(Probe).unwrap(); + child.send(Probe).unwrap(); + let handle = child.child_handle(); + assert!(supervisor.shutdown(LIMIT).await.complete); + assert_eq!(ended(&handle).await, ExitReason::Shutdown); + assert_eq!( + *log.lock().unwrap(), + vec![ + ("started", false), + ("probe", false), + ("exit", false), + ("probe", false), + ("await", true), + ("probe", true), + ("probe", true), + ("stopped", true), + ] + ); + assert!(system.shutdown(LIMIT).await.complete); + } + + // A non-trapping actor: an untrapped signal begins exit, so the + // handler that is running when it arrives sees it pending; ctx.stop() + // is pending at once. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn begun_exits_are_pending() { + let seen = Seen::default(); + let actor = Node::new(&seen, false).start(); + let pending = Arc::new(Mutex::new(Vec::new())); + let p = pending.clone(); + on(&actor, move |ctx| { + p.lock().unwrap().push(ctx.exit_pending()); + ctx.stop(); + p.lock().unwrap().push(ctx.exit_pending()); + }) + .await; + assert_eq!(ended(&actor.child_handle()).await, ExitReason::Normal); + assert_eq!(*pending.lock().unwrap(), vec![false, true]); + let actor = Node::new(&seen, false).start(); + let handle = actor.child_handle(); + let p = pending.clone(); + on(&actor, move |ctx| { + handle.exit(ExitReason::Shutdown); + p.lock().unwrap().push(ctx.exit_pending()); + }) + .await; + assert_eq!(ended(&actor.child_handle()).await, ExitReason::Shutdown); + assert_eq!(*pending.lock().unwrap(), vec![false, true, true]); + } + } + + // Decision 2 is easy to miss: a skipped stopped() is logged at debug + // (`event = "stopped_skipped"`), for a kill, an untrapped signal and a + // runtime drop. A current-thread runtime keeps the scoped subscriber on + // the actors' thread. + mod stopped_skipped_log { + use super::*; + use std::collections::BTreeMap; + type Fields = BTreeMap; + #[derive(Clone, Default)] + struct Capture(Arc>>); + impl tracing::Subscriber for Capture { + fn enabled(&self, _: &tracing::Metadata<'_>) -> bool { + true + } + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id { + tracing::span::Id::from_u64(1) + } + fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {} + fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {} + fn enter(&self, _: &tracing::span::Id) {} + fn exit(&self, _: &tracing::span::Id) {} + fn event(&self, event: &tracing::Event<'_>) { + struct Visitor(Fields); + impl tracing::field::Visit for Visitor { + fn record_debug( + &mut self, + field: &tracing::field::Field, + value: &dyn std::fmt::Debug, + ) { + self.0.insert(field.name().to_owned(), format!("{value:?}")); + } + fn record_str(&mut self, field: &tracing::field::Field, value: &str) { + self.0.insert(field.name().to_owned(), value.to_owned()); + } + } + let mut visitor = Visitor(Fields::new()); + event.record(&mut visitor); + self.0 + .lock() + .unwrap() + .push((*event.metadata().level(), visitor.0)); + } + } + impl Capture { + /// (cause, reason) of each `stopped_skipped` event for `actor`. + fn skipped(&self, actor: ActorId) -> Vec<(String, String)> { + self.0 + .lock() + .unwrap() + .iter() + .filter(|(level, f)| { + *level == tracing::Level::DEBUG + && f.get("event").map(String::as_str) == Some("stopped_skipped") + && f.get("actor") == Some(&actor.to_string()) + }) + .map(|(_, f)| (f["cause"].clone(), f["reason"].clone())) + .collect() + } + } + + #[test] + fn skipped_stopped_is_logged_at_debug() { + let capture = Capture::default(); + let _subscriber = tracing::subscriber::set_default(capture.clone()); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (killed, signalled, trapping) = runtime.block_on(async { + let seen = Seen::default(); + let killed = Node::new(&seen, true).start(); + settle(&killed).await; + killed.child_handle().kill(); + assert_eq!(ended(&killed.child_handle()).await, ExitReason::Killed); + let signalled = Node::new(&seen, false).start(); + settle(&signalled).await; + signalled.child_handle().exit(ExitReason::Shutdown); + assert_eq!(ended(&signalled.child_handle()).await, ExitReason::Shutdown); + assert!(!seen.stopped()); + // stopped() runs: nothing logged. + let trapping = Node::new(&seen, true).start(); + settle(&trapping).await; + trapping.child_handle().stop(); + assert_eq!(ended(&trapping.child_handle()).await, ExitReason::Normal); + assert!(seen.stopped()); + (killed.id(), signalled.id(), trapping.id()) + }); + assert_eq!( + capture.skipped(killed), + vec![("kill".into(), "killed".into())] + ); + assert_eq!( + capture.skipped(signalled), + vec![("exit_signal".into(), "shutdown".into())] + ); + assert!(capture.skipped(trapping).is_empty()); + // Runtime drop before stopped() began. + let dropped = runtime.block_on(async { + let actor = Node::new(&Seen::default(), true).start(); + settle(&actor).await; + actor.id() + }); + drop(runtime); + assert_eq!( + capture.skipped(dropped), + vec![("runtime_drop".into(), "killed".into())] + ); + } + } + // Rank 11 x 5e ("Required changes to rank 11"): callers, `Down` and // `wait_exit` see one reason. Each test pins one rule of the design. mod caller_errors { @@ -1185,4 +1429,99 @@ mod threads_mode { assert!(matches!(actor.request_raw(Ping), Err(ActorError::NoProc))); } } + + // Context::exit_pending in threads mode: the start_linked parent's exit + // to a trapping child is pending in the handlers of messages queued + // before it. + mod exit_pending { + use super::*; + use spawned_concurrency::threads::ActorRef; + use std::sync::mpsc; + + type Log = Arc>>; + pub struct Late(Log); + impl Actor for Late { + fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); + } + fn stopped(&mut self, ctx: &Context) { + self.0.lock().unwrap().push(("stopped", ctx.exit_pending())); + } + } + struct Probe; + impl Message for Probe { + type Result = (); + } + impl Handler for Late { + fn handle( + &mut self, + _: Probe, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + self.0.lock().unwrap().push(("probe", ctx.exit_pending())); + reply.reply(()); + } + } + struct Block(mpsc::Receiver<()>); + impl Message for Block { + type Result = (); + } + impl Handler for Late { + fn handle( + &mut self, + msg: Block, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + let _ = msg.0.recv(); + self.0.lock().unwrap().push(("block", ctx.exit_pending())); + reply.reply(()); + } + } + struct Parent(Log); + impl Actor for Parent {} + struct Spawn; + impl Message for Spawn { + type Result = ActorRef; + } + impl Handler for Parent { + fn handle( + &mut self, + _: Spawn, + reply: spawned_concurrency::reply::PendingReply<'_, ActorRef>, + ctx: &Context, + ) { + reply.reply(Late(self.0.clone()).start_linked(ctx)); + } + } + + #[test] + fn queued_work_sees_the_parents_pending_exit() { + let log: Log = Arc::default(); + let parent = Parent(log.clone()).start(); + let child = parent.request(Spawn).unwrap(); + child.request(Probe).unwrap(); + let (release, gate) = mpsc::channel(); + child.send(Block(gate)).unwrap(); + child.send(Probe).unwrap(); + child.send(Probe).unwrap(); + parent.child_handle().stop_with_reason(ExitReason::Shutdown); + // Link notices are delivered before the parent's exit is + // published: the child's parent exit is queued now. + assert_eq!(ended(&parent.child_handle()), ExitReason::Shutdown); + release.send(()).unwrap(); + assert_eq!(ended(&child.child_handle()), ExitReason::Shutdown); + assert_eq!( + *log.lock().unwrap(), + vec![ + ("probe", false), + ("block", true), + ("probe", true), + ("probe", true), + ("stopped", true), + ] + ); + } + } } diff --git a/docs/GUIDE.md b/docs/GUIDE.md index a4642af9..24f3c2cb 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -38,7 +38,7 @@ Every actor goes through three phases: 1. **`started()`** — called once before the actor begins processing messages. Use this for initialization: starting timers, registering with the registry, etc. 2. **Message loop** — the actor processes messages one at a time via `Handler` implementations. -3. **`stopped()`** — called once after the message loop exits. Use this for cleanup. +3. **`stopped()`** — called once after the message loop exits, unless the exit was a kill or an exit signal (see [When `stopped()` does not run](#when-stopped-does-not-run)). Use this for cleanup. ```rust #[actor(protocol = MyProtocol)] @@ -79,6 +79,39 @@ From outside, use the actor's `ChildHandle` (`actor_ref.child_handle()`): `stop( Stopping discards messages still queued: once the stop is observed no further message is handled, and queued requests fail with `ActorStopped(reason)`, the actor's final exit reason. +### When `stopped()` does not run + +As in OTP (`terminate/2` runs only for exits the process takes itself), `stopped()` runs for `ctx.stop()`, an administrative stop (`ChildHandle::stop`/`shutdown`/`stop_with_reason`), a panic in a handler, and a parent's exit to a **trapping** actor. It is **skipped** when the exit is begun by: + +- a kill (`ChildHandle::kill()`, `exit(Kill)`, a supervisor's kill escalation at `shutdown_timeout`), unless `stopped()` already began; +- an untrapped exit signal or link notice (`ChildHandle::exit(reason)`, a linked peer's abnormal exit); +- a supervisor's or system's shutdown, or a `start_linked` parent's exit, to an actor that does **not** trap exits: the parent's exit is an exit signal (design E3); +- the runtime dropping the actor (`Killed`). + +Every skip is logged at `debug` with `event = "stopped_skipped"`, the actor id and type, `cause` (`kill`, `exit_signal` or `runtime_drop`) and `reason`. The log is emitted whether or not the actor overrides `stopped()` (there is no cheap way to tell), so filter on the actor type. The lifecycle trace (feature `lifecycle-trace`) records the same decision as `Hook` `skip`. + +If cleanup must run on a supervisor's shutdown, trap exits first thing in `started()`; the parent's exit then ends the actor through `stopped()`, after the messages queued before it: + +```rust +async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); // shutdown runs stopped(); linked peers' deaths arrive as `Exit` +} +``` + +A trapping actor still handles the messages queued before the parent's exit. To refuse late work, check `ctx.exit_pending()`: it is `true` once the actor's exit has begun (`ctx.stop()`, an administrative stop, a kill, an untrapped signal, a panic; also in `stopped()`) or a parent exit is queued behind the current message. A trapped `Exit` from a linked peer or a non-parent `exit` is not a pending exit. It is advisory: a signal can arrive right after it returns `false`. + +```rust +#[request_handler] +async fn handle_work(&mut self, msg: Work, ctx: &Context) -> Result<(), Refused> { + if ctx.exit_pending() { + return Err(Refused::ShuttingDown); + } + // ... +} +``` + +State that must be released on every exit, including kills and runtime drops, belongs in `Drop`. + --- ## Context @@ -644,6 +677,18 @@ pub enum ActorError { - `request()` returns `Err(ActorStopped(reason))` if the actor dies before replying, with the reason `wait_exit()` and monitors report (`Killed`, `Panic(..)`, the exit signal's reason, ...), `Err(NoProc)` if it was already gone, or `Err(RequestTimeout)` if the reply doesn't arrive in time. A request whose handler panicked is answered when the actor dies (after `stopped()`), not at once. - `request_with_timeout()` lets you specify a custom timeout. +`ActorError` is `#[non_exhaustive]`. Classify it with the helpers instead of an exhaustive match: + +| Variant | `target_gone()` | `is_timeout()` | `is_transient()` | `exit_reason()` | +|---|---|---|---|---| +| `ActorStopped(r)` | `true` | `false` | `false` | `Some(&r)` | +| `NoProc` | `true` | `false` | `false` | `None` | +| `RequestTimeout` | `false` | `true` | `true` | `None` | +| `ReplyDropped` | `false` | `false` | `false` | `None` | +| `Overloaded` | `false` | `false` | `true` | `None` | + +`target_gone()`: retrying the same reference cannot succeed (look the actor up again; a supervisor may have restarted it). `is_transient()`: retrying the same target may succeed. `ReplyDropped` is neither: the actor is alive but its handler chose not to answer. `ExitReason::is_abnormal()` is `false` only for `Normal` and `Shutdown` (`Killed` and `NoProc` are abnormal). + --- ## spawned-rt diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 09d82b41..b483f16d 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -18,7 +18,13 @@ table is the full list). What to edit: - **`stopped()` no longer always runs.** It is skipped when the exit is begun by a kill, an untrapped exit signal or link notice (a linked peer's abnormal death), or a runtime drop (decision 2, as OTP skips `terminate`). - Cleanup that must run on every exit belongs in `Drop`. + Cleanup that must run on every exit belongs in `Drop`. **To notice a + skip**, enable `debug` logs for `spawned_concurrency`: every skipped + `stopped()` logs `event = "stopped_skipped"` with the actor id and type, + `cause` (`kill`, `exit_signal`, `runtime_drop`) and `reason`. It is logged + for every skip, also for actors that do not override `stopped()`. The + remedy for a supervised actor is `ctx.trap_exit(true)` in `started()` + (below). > **Supervised and system-owned actors: `stopped()` is skipped on shutdown > unless the actor traps exits.** Supervisor and system shutdown send the @@ -51,6 +57,12 @@ table is the full list). What to edit: > A trapping actor also receives linked peers' deaths as `Exit` messages > (`exit_received`) instead of dying with them; trap only where that is > intended, or move the cleanup into `Drop`. +> +> A trapping actor still handles the messages queued before the shutdown. To +> refuse that late work, check the new `ctx.exit_pending()` (both modes): it +> is `true` once the actor's exit has begun or the parent's exit is queued +> behind the current message (a peer's trapped `Exit` does not count). It is +> advisory: a signal can arrive right after it returns `false`. - **Runtime drop** reports `Killed` (was `Kill` in tasks mode, `Panic("unexpected framework panic")` in threads mode). @@ -113,9 +125,15 @@ table is the full list). What to edit: while the actor exits, is no longer answered at once: it is answered at the death point (after `stopped()`) with the final reason, or by the caller's own deadline first. With `metrics`, rejected requests count in the new - `requests_noproc_total`. **ethrex**: `peer_handler.rs:1251` matches - `ActorError` exhaustively on purpose, so it no longer compiles: add arms - for `ActorStopped(_)` and `NoProc`; two tests that construct + `requests_noproc_total`. `ActorError` stays `#[non_exhaustive]`; instead + of an exhaustive match, classify with `target_gone()` (`ActorStopped(_)`, + `NoProc`), `is_timeout()` (`RequestTimeout`), `is_transient()` + (`RequestTimeout`, `Overloaded`: retrying the same target may succeed; + not `ReplyDropped`) and `exit_reason()` (the reason of `ActorStopped`). + A variant added later is classified by these helpers. **ethrex**: + `peer_handler.rs:1251` matches `ActorError` exhaustively on purpose, so it + no longer compiles: add arms for `ActorStopped(_)` and `NoProc`, or replace + the match with the helpers above; two tests that construct `ActorError::ActorStopped` must pass a reason (for example `ActorStopped(ExitReason::Normal)`). **ethlambda**: one test mock constructs `ActorStopped`; pass a reason the same way. A `spawned-test` From a54b6ee300da18db713d7f0cce3d603f72b32832 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 03:08:30 -0300 Subject: [PATCH 091/208] docs/test: client-port refinements in GUIDE, MIGRATION, CHANGELOG, design; sim-aware deferred_api; Loom model checks delayed-emit delivery - GUIDE: reserve-then-spawn verdict example on the blocking pool, hard job deadlines and the soft-deadline pattern, replies carrying a reason via R = Result<..>, #[send_handler] on manual impls, bounds in #[actor] and ChildSpec, in_actor, request_raw rows (ReplyReceiver, not a raw oneshot) - design: permits, emit_at (held terminal), executors, macro/ChildSpec bounds - example: verifier reserves before spawning, deadline_after - trybuild stderr for the new #[actor] parameters --- CHANGELOG.md | 23 ++++++++++ concurrency/src/reply/job_core.rs | 6 ++- concurrency/src/reply/jobs.rs | 2 + concurrency/tests/deferred_api.rs | 42 +++++++++++++++---- concurrency/tests/job_core_loom.rs | 4 +- concurrency/tests/job_trace.rs | 6 +-- .../tests/ui/actor_unknown_param.stderr | 2 +- docs/MIGRATION.md | 38 +++++++++++++++++ docs/design/deferred-replies.md | 37 +++++++++++++--- .../capabilities/src/bin/deferred_replies.rs | 19 +++++---- 10 files changed, 151 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0bac4f9..653d98cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,29 @@ `ReplyDropped` and `Overloaded`; `metrics::Snapshot` gains `requests_reply_dropped_total` and `requests_overloaded_total`. Example: `deferred_replies`. See [migration](docs/MIGRATION.md#unreleased). +- Rank 11 API refinements from porting ethlambda and ethrex to 0.8: + `#[actor(max_outstanding = N, max_native_jobs = N)]` sets the bounds, and + `ChildSpec::max_outstanding`/`max_native_jobs` override them per + supervised child (the design's per-spec override). `tasks`-mode blocking + jobs (`pipe_blocking*`) run on Tokio's blocking pool instead of a fresh OS + thread each, still under `MAX_NATIVE_JOBS`, with panics as + `JobError::Panicked` and cancellation and fencing unchanged (threads mode + keeps owned threads; a reservation pre-spawns the job's thread). + Reserve-then-spawn: `ctx.reserve_job()` / `ctx.reserve_blocking_job()` + return a `JobPermit` / `BlockingJobPermit` whose `pipe*` cannot fail, so a + refused caller keeps what it would have moved into `work` and `done`. + `JobCtx::emit_at` / `emit_at_blocking` deliver a fenced, credit-accounted + message at an instant; the job's terminal message waits for its scheduled + emits (new Loom model `scheduled_emit_vs_finish_and_cancel`). + `JobOptions::deadline_after(Duration)`; `spawned_concurrency::Instant` + (also `tasks::`/`threads::Instant`) re-exports the deadline clock. + `#[send_handler]` on a manual `impl Handler` (for `Result = ()`) drops + the reply parameter and answers `()` when `handle` returns. + `spawned_test::tasks::in_actor` / `threads::in_actor` run a closure in a + live actor's turn. `JobOutcome`, `JobError` and `PipeErrorKind` are + `#[non_exhaustive]`. GUIDE: hard job deadlines and the soft-deadline + pattern, replies carrying a reason through `R = Result<..>`, the + `request_raw` rows. - Design (not implementation) for rank 11, deferred typed replies and pipe-to-self, revised over four passes after independent reviews: `docs/design/deferred-replies.md`, client mapping for ethrex and ethlambda, diff --git a/concurrency/src/reply/job_core.rs b/concurrency/src/reply/job_core.rs index 6ca487c8..168014fa 100644 --- a/concurrency/src/reply/job_core.rs +++ b/concurrency/src/reply/job_core.rs @@ -58,6 +58,10 @@ impl Inner { } } +/// An emit's step 2: enqueued, or `JobGone` with the envelope the mailbox +/// refused (to drop outside locks). +pub(crate) type EmitResult = Result<(), (JobGone, Option)>; + /// What [`JobCore::finish`] did with the work's result. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum Finish { @@ -267,7 +271,7 @@ impl JobCore { pub(crate) fn fire( &self, enqueue: Option Result<(), X>>, - ) -> (Result<(), (JobGone, Option)>, Option) { + ) -> (EmitResult, Option) { let mut inner = lock(&self.inner); debug_assert!(inner.scheduled > 0, "scheduled emit underflow"); inner.scheduled = inner.scheduled.saturating_sub(1); diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 39063eff..a162e362 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -37,12 +37,14 @@ pub fn default_max_native_jobs() -> usize { /// Per-instance overrides of `Actor::MAX_OUTSTANDING` and /// `Actor::MAX_NATIVE_JOBS` (set per `ChildSpec`); `None` keeps the actor's /// constant. +#[cfg(feature = "async")] #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub(crate) struct Bounds { pub(crate) max_outstanding: Option, pub(crate) max_native_jobs: Option, } +#[cfg(feature = "async")] impl Bounds { /// The effective `(max_outstanding, max_native_jobs)` for an actor whose /// constants are `outstanding` and `native`. diff --git a/concurrency/tests/deferred_api.rs b/concurrency/tests/deferred_api.rs index dcdfb287..f0f0cddd 100644 --- a/concurrency/tests/deferred_api.rs +++ b/concurrency/tests/deferred_api.rs @@ -93,7 +93,7 @@ mod tasks_mode { if entries.len() >= n { return entries; } - tokio::time::sleep(Duration::from_millis(2)).await; + spawned_rt::tasks::sleep(Duration::from_millis(2)).await; } panic!("log never reached {n} entries: {:?}", log.snapshot()); } @@ -104,7 +104,7 @@ mod tasks_mode { assert_eq!(::MAX_NATIVE_JOBS, Some(2)); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[spawned_macros::sim_test] async fn macro_bounds_apply_and_permits_keep_the_parts() { let (actor, _log) = start(); let kinds = in_actor(&actor, |_, ctx| { @@ -132,6 +132,7 @@ mod tasks_mode { actor.join().await; } + #[cfg(not(spawned_sim))] // native jobs are not simulated #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn blocking_jobs_run_on_the_tokio_blocking_pool() { let (actor, log) = start(); @@ -162,6 +163,7 @@ mod tasks_mode { actor.join().await; } + #[cfg(not(spawned_sim))] // native jobs are not simulated #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn emit_at_arrives_late_but_before_the_terminal_message() { let (actor, log) = start(); @@ -185,7 +187,31 @@ mod tasks_mode { actor.join().await; } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[spawned_macros::sim_test] + async fn async_emit_at_holds_the_terminal_message() { + let (actor, log) = start(); + in_actor(&actor, |_, ctx| { + ctx.pipe_with_ctx( + JobOptions::default(), + |job| async move { + let at = Instant::now() + Duration::from_millis(50); + job.emit_at(at, Note("late".into())).await?; + job.emit(Note("now".into())).await + }, + |o| describe("job", o), + ) + .unwrap(); + }) + .await; + let entries = wait_for(&log, 3).await; + assert_eq!(entries, ["now", "late", "job:ok:Ok(())"]); + let outstanding = in_actor(&actor, |_, ctx| ctx.deferred_outstanding()).await; + assert_eq!(outstanding, 0); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] async fn a_cancelled_job_fences_its_scheduled_emits() { let (actor, log) = start(); in_actor(&actor, |_, ctx| { @@ -215,7 +241,7 @@ mod tasks_mode { .unwrap(); }) .await; - tokio::time::sleep(Duration::from_millis(150)).await; + spawned_rt::tasks::sleep(Duration::from_millis(150)).await; assert_eq!(log.snapshot(), ["new:ok:1"]); let outstanding = in_actor(&actor, |_, ctx| ctx.deferred_outstanding()).await; assert_eq!(outstanding, 0); @@ -223,6 +249,7 @@ mod tasks_mode { actor.join().await; } + #[cfg(not(spawned_sim))] // native jobs are not simulated #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn job_deadlines_are_hard() { let (actor, log) = start(); @@ -241,13 +268,13 @@ mod tasks_mode { }) .await; assert_eq!(wait_for(&log, 1).await, ["slow:deadline"]); - tokio::time::sleep(Duration::from_millis(200)).await; + spawned_rt::tasks::sleep(Duration::from_millis(200)).await; assert_eq!(log.snapshot(), ["slow:deadline"]); actor.context().stop(); actor.join().await; } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[spawned_macros::sim_test] async fn send_handler_impls_answer_requests_with_unit() { let (actor, log) = start(); assert!(matches!(actor.request(Poke(false)).await, Ok(()))); @@ -260,7 +287,7 @@ mod tasks_mode { assert!(actor.request(Poke(false)).await.is_err()); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[spawned_macros::sim_test] async fn child_spec_overrides_the_actor_bounds() { let system = ActorSystem::new().unwrap(); let group = system.async_group("bounds", 1).unwrap(); @@ -295,6 +322,7 @@ mod tasks_mode { } } +#[cfg(not(spawned_sim))] mod threads_mode { use super::*; use spawned_concurrency::threads::{ diff --git a/concurrency/tests/job_core_loom.rs b/concurrency/tests/job_core_loom.rs index 8d82170f..46036b48 100644 --- a/concurrency/tests/job_core_loom.rs +++ b/concurrency/tests/job_core_loom.rs @@ -431,12 +431,13 @@ fn scheduled_emit_vs_finish_and_cancel() { if let Some(canceller) = canceller { canceller.join().unwrap(); } - let mut terminals = 0; + let (mut terminals, mut emits) = (0, 0); for msg in mailbox.take() { match msg { Msg::Terminal(_) => terminals += 1, Msg::Emit(..) => { assert_eq!(terminals, 0, "the delayed emit followed the terminal"); + emits += 1; job.core.return_emit_credit(); } } @@ -445,6 +446,7 @@ fn scheduled_emit_vs_finish_and_cancel() { assert_eq!(job.core.scheduled(), 0); if !cancel { assert_eq!(terminals, 1, "the held terminal was lost"); + assert_eq!(emits, 1, "the delayed emit was dropped without a cancel"); } if terminals == 0 { assert_eq!(job.core.notification(), Notification::Suppressed); diff --git a/concurrency/tests/job_trace.rs b/concurrency/tests/job_trace.rs index 14b45f0c..e049c29c 100644 --- a/concurrency/tests/job_trace.rs +++ b/concurrency/tests/job_trace.rs @@ -74,9 +74,9 @@ impl Handler for Owner { fn done(name: &'static str) -> impl FnOnce(JobOutcome) -> Done { move |outcome| { - Done(match outcome { - JobOutcome::Ok(v) => format!("{name}:ok:{v:?}"), - JobOutcome::Err(e) => format!("{name}:{e}"), + Done(match outcome.into_result() { + Ok(v) => format!("{name}:ok:{v:?}"), + Err(e) => format!("{name}:{e}"), }) } } diff --git a/concurrency/tests/ui/actor_unknown_param.stderr b/concurrency/tests/ui/actor_unknown_param.stderr index 0f396996..3c889ae0 100644 --- a/concurrency/tests/ui/actor_unknown_param.stderr +++ b/concurrency/tests/ui/actor_unknown_param.stderr @@ -1,4 +1,4 @@ -error: unknown parameter, expected `protocol` +error: unknown parameter, expected `protocol`, `max_outstanding` or `max_native_jobs` --> tests/ui/actor_unknown_param.rs:11:9 | 11 | #[actor(protocols = CounterProtocol)] diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 4120bed4..5308fb3d 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -50,6 +50,44 @@ the deadline is fixed when the request is sent. GUIDE's Metrics section) and the `deferred_reply_latency` histogram. Code that builds a `Snapshot` with a struct literal must add them. +**Jobs (API refinements).** + +- `JobOutcome`, `JobError` and `PipeErrorKind` are `#[non_exhaustive]`: add + a wildcard arm, or match `outcome.into_result()` with an `Err(_)` arm. +- `tasks` mode: `pipe_blocking_with` / `pipe_reply_blocking_with` run on + Tokio's blocking pool. Remove a `pipe_with(.., async { spawn_blocking(f) + .await })` workaround (and its `resume_unwind` of the join error): + `pipe_blocking_with(opts, move |_job| f(), done)` reports a panic as + `JobError::Panicked`. Raise the native budget with + `#[actor(max_native_jobs = N)]` (or `ChildSpec::max_native_jobs(N)` for a + supervised child) instead of avoiding native jobs. A Tokio runtime being + dropped now waits for blocking jobs still running, as for any + `spawn_blocking`. +- To keep "exactly one result" when a job is refused, reserve first instead + of cloning ahead: `let Ok(permit) = ctx.reserve_blocking_job() else { + /* report with the values you still own */ }; + permit.pipe_blocking_with(opts, work, done);` (`reserve_job` for async + jobs). Spawning from a permit cannot fail. +- A job that re-scheduled its messages with `send_after` (unfenced) can use + `job.emit_at(at, msg)` / `emit_at_blocking`: fenced with the job, counted + against its emit credits, and the terminal message waits for it. +- `JobOptions::default().deadline_after(d)` replaces + `JobOptions { deadline: Some(spawned_rt::time::Instant::now() + d), .. }`; + use `spawned_concurrency::Instant` instead of `spawned_rt::time::Instant`. + Job deadlines are hard (later emits fail, the result is dropped); for a + soft deadline, check the time in the work (see the GUIDE). + +**Send-only manual handlers.** A manual `impl Handler` whose `Result` is +`()` no longer needs the `_reply: PendingReply<'_, ()>` parameter: put +`#[send_handler]` (from `spawned_concurrency`) on the impl and write +`handle(&mut self, msg, ctx)`. Without it, a handler that drops its `()` +reply gives a `request` caller `ReplyDropped`; with it, the caller gets +`Ok(())`. + +**Tests.** Replace test-only "run this closure in the actor" messages and +`cfg(test)` bypasses with `spawned_test::tasks::in_actor(&actor, |state, +ctx| ..)` (`threads::in_actor` in threads mode). + **Lifecycle.** At death the registry closes (open deferred tokens answer `ActorStopped`), then the mailbox is disposed, then actor state is dropped: a state destructor that sends to its own actor now fails with `ActorStopped` diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index 7dc9c290..c5f1a4d0 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -465,7 +465,8 @@ error) and still owns the token. ## 8. Bounds **Credits per actor instance: `MAX_OUTSTANDING`** (an `Actor` constant, -default 1024, overridable per `ChildSpec`). This is a count limit, not a +default 1024; `#[actor(max_outstanding = N)]`, overridable per `ChildSpec` +with `ChildSpec::max_outstanding`). This is a count limit, not a memory budget; byte credits are rank 18. - An in-place reply needs no credit. @@ -491,7 +492,8 @@ matter which job finishes first: the request holds `max(running jobs, live token)` credits. **Native execution budget: `MAX_NATIVE_JOBS`** (an `Actor` constant, -overridable per `ChildSpec`; provisional default +`#[actor(max_native_jobs = N)]`, overridable per `ChildSpec` with +`ChildSpec::max_native_jobs`; provisional default `min(4, available_parallelism)`). It covers `threads`-mode jobs and `pipe_blocking*`. @@ -522,6 +524,10 @@ ctx.pipe_blocking_with(opts, |job| .., done) -> Result Result> ctx.pipe_reply_with(opts, token, work, done) -> Result> ctx.pipe_reply_blocking_with(opts, token, |job| .., done) -> Result> +// Reserve-then-spawn: the admission checks (owner running, credit, native +// slot) before `work`/`done` exist; spawning from the permit cannot fail. +ctx.reserve_job() -> Result, PipeError> // .pipe/.pipe_with/.pipe_with_ctx +ctx.reserve_blocking_job() -> Result, PipeError> // .pipe_blocking_with pub enum PipeErrorKind { Overloaded, NativeBudget, SpawnFailed(io::Error), OwnerStopping } pub struct PipeError { pub kind: PipeErrorKind } @@ -537,7 +543,12 @@ impl JobCtx { pub async fn cancelled(&self); pub async fn emit(&self, msg: M) -> Result<(), JobGone> where A: Handler; pub fn emit_blocking(&self, msg: M) -> Result<(), JobGone> where A: Handler; + // Delayed emit: credit taken now, step 2 at `at` (see the emit protocol). + pub async fn emit_at(&self, at: Instant, msg: M) -> Result<(), JobGone> where A: Handler; + pub fn emit_at_blocking(&self, at: Instant, msg: M) -> Result<(), JobGone> where A: Handler; } +// `JobOutcome`, `JobError` and `PipeErrorKind` are #[non_exhaustive]; +// `JobOptions::deadline_after(d)` sets `deadline` to now + d. ``` **Failure disposition.** On every error no job is spawned, and any credit @@ -589,6 +600,17 @@ handle, the key map and every message it produces. It has three parts: which return `JobGone`. A blocked emitter therefore never holds up the stop-time join. +**Delayed emits (`emit_at`).** Step 1 runs at the call. A scheduled emit is +then registered under the job's lock (refused, `JobGone`, if the job is no +longer accepting emits) and serviced by the owner's deadline heap; at its +instant, step 2 runs as for `emit`. So it is fenced and credit-accounted like +any emit. To keep the terminal message last, the work's result is **held** +while scheduled emits remain: the last one to fire enqueues it. Cancellation +drops the scheduled emits and a held result; a job deadline takes the +notification as usual (deadlines are hard), and later scheduled emits are +then dropped at their instant. Not modelled in `DeferredJobs.tla`; Loom +model `scheduled_emit_vs_finish_and_cancel`. + **Fencing at dispatch.** Every job message carries the job's `JobState` and the owner's `ActorId`. The run loop drops it before the handler runs if: @@ -612,11 +634,14 @@ panic, so no consumed `FnOnce` has to be retried. **Executors.** -- **`tasks` mode:** async jobs on the owner's runtime and blocking jobs on the - native budget. +- **`tasks` mode:** async jobs on the owner's runtime; blocking jobs on + Tokio's blocking pool (`spawn_blocking` on the managed actor's group + runtime, or the current one), under the native budget. Outside a Tokio + runtime, and under the simulator, an owned thread. - **`threads` mode:** one owned native thread per admitted job under the - native budget. Spawning is fallible (`SpawnFailed`). The shared pool comes - in rank 11b. + native budget. Spawning is fallible (`SpawnFailed`); the thread is spawned + at reservation and parked until its job starts, so a permit's spawn is not. + A managed compute pool comes in rank 11b. - **Unmanaged actors** own their job set like managed ones; managed actors also register theirs with the scope. - **Jobs must not wait on their own owner.** Once the owner is exiting it no diff --git a/examples/capabilities/src/bin/deferred_replies.rs b/examples/capabilities/src/bin/deferred_replies.rs index 47d6b52e..f7cf80aa 100644 --- a/examples/capabilities/src/bin/deferred_replies.rs +++ b/examples/capabilities/src/bin/deferred_replies.rs @@ -2,7 +2,8 @@ //! //! 1. a router delegates a slow request with `pipe_reply` and keeps serving //! its mailbox meanwhile; -//! 2. CPU work offloaded with `pipe_blocking_with` and a job deadline; +//! 2. CPU work offloaded with a reserved `pipe_blocking_with` (Tokio's +//! blocking pool) and a job deadline; //! 3. the ethrex `outgoing_request` pattern: request ids in a bounded //! `PendingReplies`, answered when the "peer" responds, late responses //! ignored; @@ -122,22 +123,22 @@ impl Message for Verdicts { impl Verifier { #[send_handler] async fn verify(&mut self, msg: Verify, ctx: &Context) { - let opts = JobOptions { - deadline: Some(spawned_rt::time::Instant::now() + Duration::from_millis(100)), - ..JobOptions::default() + // Reserve first: a refusal still has `msg` to report, so every + // object gets exactly one verdict. + let permit = match ctx.reserve_blocking_job() { + Ok(permit) => permit, + Err(e) => return self.verdicts.push(format!("{}: rejected ({e})", msg.id)), }; let (id, spin) = (msg.id, msg.spin); - let started = ctx.pipe_blocking_with( - opts, + // On Tokio's blocking pool, under the actor's native-job budget. + permit.pipe_blocking_with( + JobOptions::default().deadline_after(Duration::from_millis(100)), move |_job| { std::thread::sleep(spin); // stands in for a CPU-bound check true }, move |res| Verdict { id, res }, ); - if let Err(e) = started { - self.verdicts.push(format!("{id}: rejected ({e})")); - } } #[send_handler] From a102947ab73dcbfb3ba035f2836e33e30fe7c0a8 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 03:34:07 -0300 Subject: [PATCH 092/208] test(exit): Loom model of the link table and monitor fence, simulator orderings Share the link table protocol (src/link/table.rs) and the observer's monitor table (src/monitor/fence.rs) with a Loom model through injected sync types, as the reply slot does; behaviour is unchanged. The model (tests/link_fence_loom.rs, 9 tests, unbounded) checks one notice per edge, none after unlink/demonitor or the receiver's death, and exactly one notice (real reason or NoProc) for a link or monitor racing the death point. tests/exit_signals_sim.rs explores kill vs stopped(), a late signal vs D, link/monitor vs D, the unlink/demonitor fences, supervisor shutdown vs a trapping child's queue, and S1's deferred exit vs the tree commit with PCT and bounded DFS. Two load flakes: the system kill at grace now lands before stopped() is released in stuck_stopped_hook_triggers_cleanup_deadline_before_terminal_publication, and the stopped_skipped log test rebuilds the tracing interest cache. --- .github/workflows/verification.yml | 4 +- concurrency/src/link.rs | 188 ++------ concurrency/src/link/table.rs | 226 ++++++++++ concurrency/src/monitor.rs | 18 +- concurrency/src/monitor/fence.rs | 101 +++++ concurrency/src/tasks/actor.rs | 87 ++-- concurrency/src/threads/actor.rs | 82 ++-- concurrency/tests/exit_signals_sim.rs | 603 ++++++++++++++++++++++++++ concurrency/tests/link_fence_loom.rs | 515 ++++++++++++++++++++++ concurrency/tests/otp_exit_signals.rs | 4 + concurrency/tests/supervision.rs | 3 + 11 files changed, 1558 insertions(+), 273 deletions(-) create mode 100644 concurrency/src/link/table.rs create mode 100644 concurrency/src/monitor/fence.rs create mode 100644 concurrency/tests/exit_signals_sim.rs create mode 100644 concurrency/tests/link_fence_loom.rs diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml index df77e335..ded68f5b 100644 --- a/.github/workflows/verification.yml +++ b/.github/workflows/verification.yml @@ -100,10 +100,10 @@ jobs: env: RUSTFLAGS: --cfg spawned_scheduler_loom run: cargo test --locked -p spawned-concurrency --no-default-features --test mailbox_admission_loom - - name: Model-check the reply slot's packed-word protocol + - name: Model-check the reply slot, job core, link table and monitor fence env: RUSTFLAGS: --cfg spawned_scheduler_loom - run: cargo test --locked --release -p spawned-concurrency --no-default-features --test reply_slot_loom --test registry_turn_loom --test job_core_loom + run: cargo test --locked --release -p spawned-concurrency --no-default-features --test reply_slot_loom --test registry_turn_loom --test job_core_loom --test link_fence_loom # Data races in the runtime and its dependencies. The crates forbid # unsafe code, so ThreadSanitizer is the sanitizer that matters here. diff --git a/concurrency/src/link.rs b/concurrency/src/link.rs index 517fefa0..9712f2eb 100644 --- a/concurrency/src/link.rs +++ b/concurrency/src/link.rs @@ -3,7 +3,7 @@ use crate::error::{ActorError, ExitReason}; use crate::lifecycle_trace as trace; use std::panic::{catch_unwind, AssertUnwindSafe}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Mutex, Weak}; +use std::sync::{Arc, Mutex}; // --------------------------------------------------------------------------- // Exit message @@ -307,41 +307,11 @@ fn apply_signal( } } -/// An entry in an actor's link table, representing one linked peer. -pub(crate) struct LinkEntry { - /// The peer's unique actor ID. - pub peer_id: ActorId, - /// Edge generation: both records of one link share it; a late record has - /// its own. Delivery applies only to a record with this generation. - pub gen: u64, - /// The peer, to deliver this actor's death notice to it. - pub peer: Arc, - /// Reference to the peer's link table, for receiver-side delivery. - pub peer_links: Weak>, - /// A pending late `NoProc` notice (the peer was past D at `link`). - pub late: bool, -} - -/// An actor's exit table: its link records and its death point. -#[derive(Default)] -pub(crate) struct LinkState { - entries: Vec, - // Set at the death point D under the same mutex as registration. - exited: Option, -} -pub(crate) type LinkTable = Arc>; - -impl LinkState { - /// Past the death point D. - pub(crate) fn is_dead(&self) -> bool { - self.exited.is_some() - } -} - -/// Create a new empty link table. -pub(crate) fn new_link_table() -> LinkTable { - Arc::new(Mutex::new(LinkState::default())) +mod sync { + pub(crate) use std::sync::{Arc, Mutex, MutexGuard, Weak}; } +mod table; +pub(crate) use table::{new_link_table, LinkNotices, LinkOutcome, LinkTable}; /// Create a new trap_exit flag (default: false). pub(crate) fn new_trap_exit_flag() -> TrapExitFlag { @@ -350,35 +320,6 @@ pub(crate) fn new_trap_exit_flag() -> TrapExitFlag { static NEXT_EDGE: AtomicU64 = AtomicU64::new(1); -/// Mutate both ends in one transaction. Callbacks and removed entries are -/// handled after releasing both locks. Pointer order is stable while Arcs live. -fn with_pair( - own: &LinkTable, - peer: &LinkTable, - f: impl FnOnce(&mut LinkState, &mut LinkState) -> R, -) -> R { - if Arc::as_ptr(own) < Arc::as_ptr(peer) { - let mut a = own.lock().unwrap_or_else(|p| p.into_inner()); - let mut b = peer.lock().unwrap_or_else(|p| p.into_inner()); - f(&mut a, &mut b) - } else { - let mut b = peer.lock().unwrap_or_else(|p| p.into_inner()); - let mut a = own.lock().unwrap_or_else(|p| p.into_inner()); - f(&mut a, &mut b) - } -} - -/// The outcome of a link attempt. -#[derive(Debug, PartialEq, Eq)] -pub(crate) enum LinkOutcome { - Linked, - /// Nothing to do: self link, an existing record, or the caller's exit - /// began (inert, as `monitor`). - Unchanged, - /// The peer was past D. - NoProc, -} - /// Link `own` and `peer` (OTP `link/1`). An existing record (live or /// pending) makes it a no-op. Past the peer's D, `try_only` returns /// `NoProc` without a record; otherwise a pending late record is created and @@ -415,38 +356,7 @@ fn link_in_section( } fail_point!("link.before_install"); let gen = NEXT_EDGE.fetch_add(1, Ordering::Relaxed); - let outcome = with_pair(own_links, peer_links, |a, b| { - if a.exited.is_some() || a.entries.iter().any(|e| e.peer_id == peer.id) { - return LinkOutcome::Unchanged; - } - if b.exited.is_some() { - if !try_only { - a.entries.push(LinkEntry { - peer_id: peer.id, - gen, - peer: peer.clone(), - peer_links: Arc::downgrade(peer_links), - late: true, - }); - } - return LinkOutcome::NoProc; - } - a.entries.push(LinkEntry { - peer_id: peer.id, - gen, - peer: peer.clone(), - peer_links: Arc::downgrade(peer_links), - late: false, - }); - b.entries.push(LinkEntry { - peer_id: own.id, - gen, - peer: own.clone(), - peer_links: Arc::downgrade(own_links), - late: false, - }); - LinkOutcome::Linked - }); + let outcome = table::install(own, own_links, peer, peer_links, gen, try_only); if outcome != LinkOutcome::Unchanged { trace::life_emit(own.id.raw(), || { trace::LifeEvent::new("Link", own.id.raw()) @@ -464,14 +374,6 @@ fn link_in_section( outcome } -fn remove(table: &mut LinkState, id: ActorId) -> Option { - table - .entries - .iter() - .position(|e| e.peer_id == id) - .map(|i| table.entries.swap_remove(i)) -} - /// Unlink removes both records, live or pending: once it returns, no notice /// of that edge takes effect (E6). An `Exit` already queued stays. pub(crate) fn unregister_link( @@ -484,7 +386,7 @@ pub(crate) fn unregister_link( return; } let removed = trace::life_section(own_id.raw(), || { - let removed = with_pair(own, peer, |a, b| (remove(a, peer_id), remove(b, own_id))); + let removed = table::remove_pair(own_id, own, peer_id, peer); if removed.0.is_some() || removed.1.is_some() { trace::life_emit(own_id.raw(), || { trace::LifeEvent::new("Unlink", own_id.raw()).s(peer_id.raw()) @@ -517,59 +419,33 @@ fn deliver_notice_in_section( gen: u64, reason: ExitReason, ) { - let mut table = receiver_links.lock().unwrap_or_else(|p| p.into_inner()); - let Some(i) = table - .entries - .iter() - .position(|e| e.peer_id == from && e.gen == gen) - else { - return; - }; - let record = table.entries.swap_remove(i); - trace::life_emit(receiver.id.raw(), || { - trace::LifeEvent::new("Deliver", receiver.id.raw()) - .s(from.raw()) - .x(trace::life_reason(&reason)) - .k("link") - .o(if record.late { "late" } else { "death" }) - }); - // One failing internal callback must not prevent other peers from being - // notified, or panic again during actor-future destruction. - if let Err(payload) = catch_unwind(AssertUnwindSafe(|| { - deliver_signal(receiver, from, reason, Origin::Link, false) - })) { - crate::mailbox::dispose_panic(payload); - crate::actor_cleanup::log_safely(|| { - tracing::error!("Panic while delivering a linked exit"); + let record = table::take_record(receiver_links, from, gen, |record| { + trace::life_emit(receiver.id.raw(), || { + trace::LifeEvent::new("Deliver", receiver.id.raw()) + .s(from.raw()) + .x(trace::life_reason(&reason)) + .k("link") + .o(if record.late { "late" } else { "death" }) }); - } - drop(table); + // One failing internal callback must not prevent other peers from + // being notified, or panic again during actor-future destruction. + if let Err(payload) = catch_unwind(AssertUnwindSafe(|| { + deliver_signal(receiver, from, reason, Origin::Link, false) + })) { + crate::mailbox::dispose_panic(payload); + crate::actor_cleanup::log_safely(|| { + tracing::error!("Panic while delivering a linked exit"); + }); + } + }); drop(record); } -/// Death notices taken at D, delivered at publication. -#[derive(Default)] -pub(crate) struct LinkNotices { - from: Option, - reason: Option, - entries: Vec, -} - impl LinkNotices { /// Deliver every notice (publication). Each is applied only if the /// receiver still holds the record (E6 fence). pub(crate) fn deliver(self) { - let (Some(from), Some(reason)) = (self.from, self.reason) else { - return; - }; - for entry in self.entries { - if entry.late { - continue; - } - if let Some(peer_links) = entry.peer_links.upgrade() { - deliver_notice(&peer_links, &entry.peer, from, entry.gen, reason.clone()); - } - } + self.deliver_with(deliver_notice); } } @@ -590,17 +466,7 @@ pub(crate) fn close_links_at_death( reason: &ExitReason, at_death: impl FnOnce(&ExitReason), ) -> LinkNotices { - let mut state = own.lock().unwrap_or_else(|p| p.into_inner()); - if state.exited.is_some() { - return LinkNotices::default(); - } - at_death(reason); - state.exited = Some(reason.clone()); - LinkNotices { - from: Some(own_id), - reason: Some(reason.clone()), - entries: std::mem::take(&mut state.entries), - } + table::close(own_id, own, reason, at_death) } /// Close and deliver at once (tests and paths with nothing in between). diff --git a/concurrency/src/link/table.rs b/concurrency/src/link/table.rs new file mode 100644 index 00000000..4f6c7b99 --- /dev/null +++ b/concurrency/src/link/table.rs @@ -0,0 +1,226 @@ +//! The link table protocol (docs/design/otp-exit-signals.md E5, E6): edge +//! generations, both records installed in one two-lock transaction, +//! receiver-side fenced delivery and the close at the death point D that +//! takes the notices. +//! +//! Shared with the Loom model (`tests/link_fence_loom.rs`), which supplies +//! `ActorId`, `ExitReason`, `SignalTarget` and the synchronization types +//! through `super`. The protocol never runs user code under a table lock +//! except through the `apply` callback of [`take_record`], which production +//! uses to apply the signal to the receiver (it takes only the receiver's +//! exit-request lock, never a table lock). + +use super::sync::{Arc, Mutex, MutexGuard, Weak}; +use super::{ActorId, ExitReason, SignalTarget}; + +/// An entry in an actor's link table, representing one linked peer. +pub(crate) struct LinkEntry { + /// The peer's unique actor ID. + pub peer_id: ActorId, + /// Edge generation: both records of one link share it; a late record has + /// its own. Delivery applies only to a record with this generation. + pub gen: u64, + /// The peer, to deliver this actor's death notice to it. + pub peer: Arc, + /// Reference to the peer's link table, for receiver-side delivery. + pub peer_links: Weak>, + /// A pending late `NoProc` notice (the peer was past D at `link`). + pub late: bool, +} + +/// An actor's exit table: its link records and its death point. +#[derive(Default)] +pub(crate) struct LinkState { + pub(super) entries: Vec, + // Set at the death point D under the same mutex as registration. + pub(super) exited: Option, +} +pub(crate) type LinkTable = Arc>; + +impl LinkState { + /// Past the death point D. + pub(crate) fn is_dead(&self) -> bool { + self.exited.is_some() + } +} + +/// Create a new empty link table. +pub(crate) fn new_link_table() -> LinkTable { + Arc::new(Mutex::new(LinkState::default())) +} + +pub(super) fn lock(table: &Mutex) -> MutexGuard<'_, LinkState> { + table.lock().unwrap_or_else(|p| p.into_inner()) +} + +/// Mutate both ends in one transaction. Callbacks and removed entries are +/// handled after releasing both locks. Pointer order is stable while Arcs live. +fn with_pair( + own: &LinkTable, + peer: &LinkTable, + f: impl FnOnce(&mut LinkState, &mut LinkState) -> R, +) -> R { + if Arc::as_ptr(own) < Arc::as_ptr(peer) { + let mut a = lock(own); + let mut b = lock(peer); + f(&mut a, &mut b) + } else { + let mut b = lock(peer); + let mut a = lock(own); + f(&mut a, &mut b) + } +} + +/// The outcome of a link attempt. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum LinkOutcome { + Linked, + /// Nothing to do: self link, an existing record, or the caller's exit + /// began (inert, as `monitor`). + Unchanged, + /// The peer was past D. + NoProc, +} + +/// Install the records of edge `gen` between `own` and `peer`. An existing +/// record (live or pending) or a closed own table makes it a no-op. Past the +/// peer's D, `try_only` installs nothing; otherwise a pending late record is +/// installed on `own`, whose `NoProc` notice the caller delivers through +/// [`take_record`] (so a concurrent `unlink` still fences it). +pub(crate) fn install( + own: &Arc, + own_links: &LinkTable, + peer: &Arc, + peer_links: &LinkTable, + gen: u64, + try_only: bool, +) -> LinkOutcome { + with_pair(own_links, peer_links, |a, b| { + if a.exited.is_some() || a.entries.iter().any(|e| e.peer_id == peer.id) { + return LinkOutcome::Unchanged; + } + if b.exited.is_some() { + if !try_only { + a.entries.push(LinkEntry { + peer_id: peer.id, + gen, + peer: peer.clone(), + peer_links: Arc::downgrade(peer_links), + late: true, + }); + } + return LinkOutcome::NoProc; + } + a.entries.push(LinkEntry { + peer_id: peer.id, + gen, + peer: peer.clone(), + peer_links: Arc::downgrade(peer_links), + late: false, + }); + b.entries.push(LinkEntry { + peer_id: own.id, + gen, + peer: own.clone(), + peer_links: Arc::downgrade(own_links), + late: false, + }); + LinkOutcome::Linked + }) +} + +fn remove(table: &mut LinkState, id: ActorId) -> Option { + table + .entries + .iter() + .position(|e| e.peer_id == id) + .map(|i| table.entries.swap_remove(i)) +} + +/// Remove both records of the edge, live or pending, in one transaction: +/// once this returns no notice of that edge takes effect (E6). The caller +/// drops the removed records after every lock is released. +pub(crate) fn remove_pair( + own_id: ActorId, + own: &LinkTable, + peer_id: ActorId, + peer: &LinkTable, +) -> (Option, Option) { + with_pair(own, peer, |a, b| (remove(a, peer_id), remove(b, own_id))) +} + +/// Receiver-side delivery: under the receiver's table lock, apply the notice +/// only if the record with that peer and generation is still present, and +/// remove it. `unlink` and the receiver's own close at D remove records +/// under the same lock (the fence). Returns the removed record, which the +/// caller drops after the lock is released. +pub(crate) fn take_record( + receiver_links: &LinkTable, + from: ActorId, + gen: u64, + apply: impl FnOnce(&LinkEntry), +) -> Option { + let mut table = lock(receiver_links); + let i = table + .entries + .iter() + .position(|e| e.peer_id == from && e.gen == gen)?; + let record = table.entries.swap_remove(i); + apply(&record); + drop(table); + Some(record) +} + +/// Death notices taken at D, delivered at publication. +#[derive(Default)] +pub(crate) struct LinkNotices { + from: Option, + reason: Option, + entries: Vec, +} + +impl LinkNotices { + /// Deliver every notice (publication) through `notice(receiver table, + /// receiver, from, gen, reason)`, which applies it only if the receiver + /// still holds the record (E6 fence). Late records carry no notice of + /// this actor's death: their `NoProc` was this actor's own. + pub(crate) fn deliver_with( + self, + mut notice: impl FnMut(&LinkTable, &SignalTarget, ActorId, u64, ExitReason), + ) { + let (Some(from), Some(reason)) = (self.from, self.reason) else { + return; + }; + for entry in self.entries { + if entry.late { + continue; + } + if let Some(peer_links) = entry.peer_links.upgrade() { + notice(&peer_links, &entry.peer, from, entry.gen, reason.clone()); + } + } + } +} + +/// The exit-table part of the death point D: set the final reason and take +/// one notice per link, taking only this actor's own lock. `at_death` runs +/// inside the same critical section (it must not run user code). Terminal +/// closure is idempotent: a second close takes nothing. +pub(crate) fn close( + own_id: ActorId, + own: &LinkTable, + reason: &ExitReason, + at_death: impl FnOnce(&ExitReason), +) -> LinkNotices { + let mut state = lock(own); + if state.exited.is_some() { + return LinkNotices::default(); + } + at_death(reason); + state.exited = Some(reason.clone()); + LinkNotices { + from: Some(own_id), + reason: Some(reason.clone()), + entries: std::mem::take(&mut state.entries), + } +} diff --git a/concurrency/src/monitor.rs b/concurrency/src/monitor.rs index fdb04db7..1e0a8662 100644 --- a/concurrency/src/monitor.rs +++ b/concurrency/src/monitor.rs @@ -55,18 +55,20 @@ impl Message for Down { type Result = (); } +mod sync { + pub(crate) use std::sync::{Mutex, MutexGuard}; +} +pub(crate) mod fence; + /// Each monitor owns a cancellation token that wakes its waiter immediately. -pub(crate) type MonitorTable = std::sync::Arc< - std::sync::Mutex< - Option>, - >, ->; +pub(crate) type MonitorTable = + std::sync::Arc>; /// Release outgoing monitors on every actor exit path, including unwinding. pub(crate) struct MonitorCleanup(pub MonitorTable); impl Drop for MonitorCleanup { fn drop(&mut self) { - let monitors = self.0.lock().unwrap_or_else(|p| p.into_inner()).take(); + let monitors = fence::close(&self.0); if let Some(monitors) = monitors { // HashMap order is per-process random; under simulation, wake // waiters in monitor-creation order so the schedule is seeded. @@ -84,9 +86,7 @@ impl Drop for MonitorCleanup { pub(crate) struct MonitorRegistration(pub MonitorTable, pub MonitorRef); impl Drop for MonitorRegistration { fn drop(&mut self) { - if let Some(entries) = self.0.lock().unwrap_or_else(|p| p.into_inner()).as_mut() { - entries.remove(&self.1); - } + drop(fence::remove(&self.0, &self.1, || {})); } } diff --git a/concurrency/src/monitor/fence.rs b/concurrency/src/monitor/fence.rs new file mode 100644 index 00000000..9b837033 --- /dev/null +++ b/concurrency/src/monitor/fence.rs @@ -0,0 +1,101 @@ +//! The observer-side monitor table protocol (docs/design/otp-exit-signals.md +//! E4, E6): registration linearised against the target's death point, the +//! `Down` enqueued in the same critical section that removes the entry, the +//! `demonitor` fence and the observer's own close. +//! +//! Shared with the Loom model (`tests/link_fence_loom.rs`), which supplies +//! the synchronization types through `super::sync`. The callbacks run under +//! the observer's table lock; production uses them only to read the target's +//! death point, record a trace event and enqueue a `Down` (which never takes +//! a monitor table lock). + +use super::sync::{Mutex, MutexGuard}; +use std::collections::HashMap; +use std::hash::Hash; + +/// An observer's monitors: `None` once the observer's exit closed it. +pub(crate) type Entries = Mutex>>; + +fn lock(table: &Entries) -> MutexGuard<'_, Option>> { + table.lock().unwrap_or_else(|p| p.into_inner()) +} + +/// The outcome of a `monitor` call. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Registration { + /// The observer is exiting or closed: no entry, no notice. + Inert, + /// The target was past D: the `NoProc` notice was enqueued, no entry. + Late, + /// The entry is installed; its waiter delivers the `Down` at publication. + Active, +} + +/// `monitor`: under the observer's lock, inert once `observer_exiting` or the +/// table is closed; past the target's D (`target_dead`, which reads the +/// target's exit table) enqueue the late notice (`late`); otherwise install +/// the entry and run `active`. +pub(crate) fn register( + table: &Entries, + key: K, + value: V, + observer_exiting: impl FnOnce() -> bool, + target_dead: impl FnOnce() -> bool, + late: impl FnOnce(), + active: impl FnOnce(), +) -> Registration { + let mut guard = lock(table); + if observer_exiting() { + return Registration::Inert; + } + let Some(entries) = guard.as_mut() else { + return Registration::Inert; + }; + if target_dead() { + late(); + return Registration::Late; + } + entries.insert(key, value); + active(); + Registration::Active +} + +/// The waiter, after the target's exit is published: remove the entry and +/// enqueue the `Down` (`deliver`) in one critical section, so once +/// [`remove`] (a `demonitor`) returns no `Down` of this monitor is enqueued. +/// Returns whether it was delivered. +pub(crate) fn deliver( + table: &Entries, + key: &K, + deliver: impl FnOnce(), +) -> bool { + let mut guard = lock(table); + let removed = guard.as_mut().and_then(|entries| entries.remove(key)); + if removed.is_some() { + deliver(); + } + drop(guard); + removed.is_some() +} + +/// `demonitor` (and a waiter that is dropped unpolled): remove the entry +/// under the lock, running `removed` there when it was present. The caller +/// wakes the waiter with the returned value after the lock is released. +pub(crate) fn remove( + table: &Entries, + key: &K, + removed: impl FnOnce(), +) -> Option { + let mut guard = lock(table); + let value = guard.as_mut().and_then(|entries| entries.remove(key)); + if value.is_some() { + removed(); + } + value +} + +/// The observer's exit: close the table and take every entry, whose waiters +/// the caller wakes after the lock is released. +pub(crate) fn close(table: &Entries) -> Option> { + lock(table).take() +} diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 8624f5a4..6a0f24e8 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -655,47 +655,42 @@ impl Context { let observer = self.shared.id.raw(); let registered = crate::lifecycle_trace::life_section(observer, || { - let mut monitors = self - .shared - .monitors - .lock() - .unwrap_or_else(|p| p.into_inner()); - // The token check makes registration inert once shutdown starts. - // The closed table below serializes registration with exit cleanup, - // even on paths that unwind before cancelling the owner token. - if self.shared.cancellation_token.is_cancelled() { - return false; - } - let Some(monitors) = monitors.as_mut() else { - return false; - }; let event = |x| { crate::lifecycle_trace::LifeEvent::new("Monitor", observer) .s(target.id().raw()) .m(monitor_ref.raw()) .x(x) }; - // Linearised against the target's death point (E4): past it the - // only notice is NoProc, delivered under the observer's lock. - if target.is_past_death() { - crate::lifecycle_trace::life_emit(observer, || event("late")); - crate::lifecycle_trace::life_emit(observer, || { - crate::lifecycle_trace::LifeEvent::new("Deliver", observer) - .s(target.id().raw()) - .m(monitor_ref.raw()) - .x("NoProc") - .k("down") - .o("late") - }); - let _ = self.send(Down { - monitor_ref, - reason: ExitReason::NoProc, - }); - return false; - } - monitors.insert(monitor_ref, active.clone()); - crate::lifecycle_trace::life_emit(observer, || event("active")); - true + crate::monitor::fence::register( + &self.shared.monitors, + monitor_ref, + active.clone(), + // The token check makes registration inert once shutdown + // starts. The closed table serializes registration with exit + // cleanup, even on paths that unwind before cancelling the + // owner token. + || self.shared.cancellation_token.is_cancelled(), + // Linearised against the target's death point (E4): past it + // the only notice is NoProc, delivered under the observer's + // lock. + || target.is_past_death(), + || { + crate::lifecycle_trace::life_emit(observer, || event("late")); + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Deliver", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x("NoProc") + .k("down") + .o("late") + }); + let _ = self.send(Down { + monitor_ref, + reason: ExitReason::NoProc, + }); + }, + || crate::lifecycle_trace::life_emit(observer, || event("active")), + ) == crate::monitor::fence::Registration::Active }); if !registered { return monitor_ref; @@ -721,12 +716,7 @@ impl Context { // enqueued in one critical section, so once `demonitor` returns // no Down of this monitor is enqueued. crate::lifecycle_trace::life_section(observer, || { - let mut entries = monitors.lock().unwrap_or_else(|p| p.into_inner()); - if entries - .as_mut() - .and_then(|entries| entries.remove(&monitor_ref)) - .is_some() - { + crate::monitor::fence::deliver(&monitors, &monitor_ref, || { crate::lifecycle_trace::life_emit(observer, || { crate::lifecycle_trace::LifeEvent::new("Deliver", observer) .s(target.id().raw()) @@ -739,8 +729,7 @@ impl Context { monitor_ref, reason, }); - } - drop(entries); + }); }); }); @@ -754,20 +743,12 @@ impl Context { pub fn demonitor(&self, monitor_ref: MonitorRef) { let observer = self.shared.id.raw(); let active = crate::lifecycle_trace::life_section(observer, || { - let active = self - .shared - .monitors - .lock() - .unwrap_or_else(|p| p.into_inner()) - .as_mut() - .and_then(|entries| entries.remove(&monitor_ref)); - if active.is_some() { + crate::monitor::fence::remove(&self.shared.monitors, &monitor_ref, || { crate::lifecycle_trace::life_emit(observer, || { crate::lifecycle_trace::LifeEvent::new("Demonitor", observer) .m(monitor_ref.raw()) }); - } - active + }) }); // Waking the waiter must happen after releasing the table lock. if let Some(active) = active { diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 35a27902..f787a440 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -554,43 +554,42 @@ impl Context { let observer = self.id.raw(); let registered = crate::lifecycle_trace::life_section(observer, || { - let mut monitors = self.monitors.lock().unwrap_or_else(|p| p.into_inner()); - // The token check makes registration inert once shutdown starts. - // The closed table below serializes registration with exit cleanup, - // even on paths that unwind before cancelling the owner token. - if self.cancellation_token.is_cancelled() { - return false; - } - let Some(monitors) = monitors.as_mut() else { - return false; - }; let event = |x| { crate::lifecycle_trace::LifeEvent::new("Monitor", observer) .s(target.id().raw()) .m(monitor_ref.raw()) .x(x) }; - // Linearised against the target's death point (E4): past it the - // only notice is NoProc, delivered under the observer's lock. - if target.is_past_death() { - crate::lifecycle_trace::life_emit(observer, || event("late")); - crate::lifecycle_trace::life_emit(observer, || { - crate::lifecycle_trace::LifeEvent::new("Deliver", observer) - .s(target.id().raw()) - .m(monitor_ref.raw()) - .x("NoProc") - .k("down") - .o("late") - }); - let _ = self.send(Down { - monitor_ref, - reason: ExitReason::NoProc, - }); - return false; - } - monitors.insert(monitor_ref, active.clone()); - crate::lifecycle_trace::life_emit(observer, || event("active")); - true + crate::monitor::fence::register( + &self.monitors, + monitor_ref, + active.clone(), + // The token check makes registration inert once shutdown + // starts. The closed table serializes registration with exit + // cleanup, even on paths that unwind before cancelling the + // owner token. + || self.cancellation_token.is_cancelled(), + // Linearised against the target's death point (E4): past it + // the only notice is NoProc, delivered under the observer's + // lock. + || target.is_past_death(), + || { + crate::lifecycle_trace::life_emit(observer, || event("late")); + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Deliver", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x("NoProc") + .k("down") + .o("late") + }); + let _ = self.send(Down { + monitor_ref, + reason: ExitReason::NoProc, + }); + }, + || crate::lifecycle_trace::life_emit(observer, || event("active")), + ) == crate::monitor::fence::Registration::Active }); if !registered { return monitor_ref; @@ -619,12 +618,7 @@ impl Context { // enqueued in one critical section, so once `demonitor` returns // no Down of this monitor is enqueued. crate::lifecycle_trace::life_section(observer, || { - let mut entries = monitors.lock().unwrap_or_else(|p| p.into_inner()); - if entries - .as_mut() - .and_then(|entries| entries.remove(&monitor_ref)) - .is_some() - { + crate::monitor::fence::deliver(&monitors, &monitor_ref, || { crate::lifecycle_trace::life_emit(observer, || { crate::lifecycle_trace::LifeEvent::new("Deliver", observer) .s(target.id().raw()) @@ -637,8 +631,7 @@ impl Context { monitor_ref, reason, }); - } - drop(entries); + }); }); }); @@ -652,19 +645,12 @@ impl Context { pub fn demonitor(&self, monitor_ref: MonitorRef) { let observer = self.id.raw(); let active = crate::lifecycle_trace::life_section(observer, || { - let active = self - .monitors - .lock() - .unwrap_or_else(|p| p.into_inner()) - .as_mut() - .and_then(|entries| entries.remove(&monitor_ref)); - if active.is_some() { + crate::monitor::fence::remove(&self.monitors, &monitor_ref, || { crate::lifecycle_trace::life_emit(observer, || { crate::lifecycle_trace::LifeEvent::new("Demonitor", observer) .m(monitor_ref.raw()) }); - } - active + }) }); // Waking the waiter must happen after releasing the table lock. if let Some(active) = active { diff --git a/concurrency/tests/exit_signals_sim.rs b/concurrency/tests/exit_signals_sim.rs new file mode 100644 index 00000000..587a50c6 --- /dev/null +++ b/concurrency/tests/exit_signals_sim.rs @@ -0,0 +1,603 @@ +//! Simulator orderings for OTP exit signals (docs/design/otp-exit-signals.md), +//! explored with PCT and bounded DFS over real actors: a kill racing +//! `stopped()`, a late signal racing the death point D, a link or monitor +//! racing D (real notice or `NoProc`), the unlink/demonitor fences against +//! delivery, a supervisor's shutdown (the parent's exit) against a trapping +//! child's queued messages, and S1's deferral of a ready child's exit racing +//! the tree's commit. Run with: +//! +//! ```sh +//! RUSTFLAGS="--cfg spawned_sim --cfg tokio_unstable" cargo test -p spawned-concurrency --test exit_signals_sim +//! ``` +#![cfg(all(spawned_sim, feature = "async"))] + +use spawned_concurrency::{ + message::Message, + reply::PendingReply, + supervision::{ChildSpec, SupervisorConfig, SupervisorStrategy}, + system::ActorSystem, + tasks::{Actor, ActorRef, ActorStart, Context, Handler}, + Down, Exit, ExitReason, MonitorRef, +}; +use spawned_rt::tasks::{ + sim::explore::{explore, Strategy}, + sleep, spawn, +}; +use std::{ + collections::BTreeMap, + future::Future, + sync::{Arc, Mutex}, + time::Duration, +}; + +type Log = Arc>>; + +fn push(log: &Log, entry: impl Into) { + log.lock().unwrap().push(entry.into()); +} + +fn entries(log: &Log) -> Vec { + log.lock().unwrap().clone() +} + +fn position(log: &[String], entry: &str) -> Option { + log.iter().position(|e| e == entry) +} + +/// Outcome counts per body, printed by the driver: every branch the +/// assertions distinguish must be reached. +static OUTCOMES: Mutex> = Mutex::new(BTreeMap::new()); + +fn outcome(name: impl Into) { + *OUTCOMES.lock().unwrap().entry(name.into()).or_default() += 1; +} + +fn ms(n: u64) -> Duration { + Duration::from_millis(n) +} + +/// A logging actor: `stopped()` takes `slow_stop` virtual milliseconds. +struct Node { + log: Log, + trap: bool, + slow_stop: u64, +} + +impl Node { + fn new(log: &Log, trap: bool) -> Self { + Self { + log: log.clone(), + trap, + slow_stop: 0, + } + } +} + +impl Actor for Node { + async fn started(&mut self, ctx: &Context) { + if self.trap { + ctx.trap_exit(true); + } + } + async fn stopped(&mut self, ctx: &Context) { + push(&self.log, format!("stopped:begin:{}", ctx.exit_pending())); + for _ in 0..self.slow_stop { + sleep(ms(1)).await; + } + push(&self.log, "stopped:end"); + } + async fn exit_received(&mut self, exit: Exit, _: &Context) { + push(&self.log, format!("exit:{:?}", exit.reason)); + } +} + +impl Handler for Node { + async fn handle(&mut self, down: Down, reply: PendingReply<'_, ()>, _: &Context) { + push(&self.log, format!("down:{:?}", down.reason)); + reply.reply(()); + } +} + +type Action = Box) + Send>; +struct Do(Action); +impl Message for Do { + type Result = (); +} +impl Handler for Node { + async fn handle(&mut self, f: Do, reply: PendingReply<'_, ()>, ctx: &Context) { + (f.0)(ctx); + reply.reply(()); + } +} + +/// Logged when handled; records whether an exit is pending. +struct Probe(u32); +impl Message for Probe { + type Result = (); +} +impl Handler for Node { + async fn handle(&mut self, p: Probe, reply: PendingReply<'_, ()>, ctx: &Context) { + push(&self.log, format!("probe:{}:{}", p.0, ctx.exit_pending())); + reply.reply(()); + } +} + +async fn on(actor: &ActorRef, f: impl FnOnce(&Context) + Send + 'static) { + actor.request(Do(Box::new(f))).await.unwrap(); +} + +/// A round trip, so everything queued before it was handled. +async fn settle(actor: &ActorRef) { + on(actor, |_| {}).await; +} + +async fn stop(actor: &ActorRef) { + let handle = actor.child_handle(); + handle.stop(); + handle.wait_exit_async().await; +} + +/// A `Down` is enqueued by the monitor's waiter after the target's exit is +/// published (unlike a link notice, not before `wait_exit` returns): wait +/// for it, bounded, then settle once more so a second one would show. +async fn downs(observer: &ActorRef, log: &Log) -> Vec { + for _ in 0..100 { + settle(observer).await; + if !notices(&entries(log), "down:").is_empty() { + break; + } + sleep(ms(1)).await; + } + settle(observer).await; + notices(&entries(log), "down:") +} + +fn notices(log: &[String], kind: &str) -> Vec { + log.iter() + .filter(|e| e.starts_with(kind)) + .cloned() + .collect() +} + +// --------------------------------------------------------------------------- +// Bodies +// --------------------------------------------------------------------------- + +/// E1: a kill racing an administrative stop's slow `stopped()`. Before +/// `stopped()` begins it is skipped and the actor ends `Killed`; during it, +/// it finishes and the actor ends `Killed`; after D it is ignored. A monitor +/// sees the same final reason. +async fn kill_vs_stopped(seed: u64) { + let (log, watched) = (Log::default(), Log::default()); + let mut node = Node::new(&log, false); + node.slow_stop = 2; + let target = node.start(); + let watcher = Node::new(&watched, false).start(); + let handle = target.child_handle(); + let h = handle.clone(); + on(&watcher, move |ctx| { + ctx.monitor(&h); + }) + .await; + let (stopper, killer) = (handle.clone(), handle.clone()); + let stop_at = seed % 3; + let kill_at = (seed / 3) % 5; + let stopping = spawn(async move { + sleep(ms(stop_at)).await; + stopper.stop_with_reason(ExitReason::Shutdown); + }); + let l = log.clone(); + let killing = spawn(async move { + sleep(ms(kill_at)).await; + push(&l, "kill"); + killer.kill(); + }); + let reason = handle.wait_exit_async().await; + stopping.await.unwrap(); + killing.await.unwrap(); + let log = entries(&log); + let kill = position(&log, "kill").unwrap(); + let begin = log.iter().position(|e| e.starts_with("stopped:begin")); + let end = position(&log, "stopped:end"); + assert_eq!( + begin.is_some(), + end.is_some(), + "a begun stopped() finishes: {log:?}" + ); + assert!( + begin.is_none_or(|b| b < kill), + "stopped() began after the kill: {log:?}" + ); + outcome(match (&reason, begin) { + (ExitReason::Killed, None) => "kill: before stopped()", + (ExitReason::Killed, Some(_)) => "kill: during stopped()", + _ => "kill: after D", + }); + match reason { + ExitReason::Killed => {} + ExitReason::Shutdown => { + assert!( + end.is_some_and(|e| e < kill), + "the kill came before D but did not win: {log:?}" + ); + } + other => panic!("unexpected reason {other:?}: {log:?}"), + } + assert_eq!( + downs(&watcher, &watched).await, + vec![format!("down:{reason:?}")] + ); + stop(&watcher).await; +} + +/// Decision 8: an untrapped signal racing D after a `Normal` stop replaces +/// the reason until D (the running `stopped()` still finishes); after D it +/// is ignored. A trapping link and a monitor see the same final reason. +async fn late_signal_vs_death(seed: u64) { + let (log, observed) = (Log::default(), Log::default()); + let mut node = Node::new(&log, false); + node.slow_stop = 2; + let target = node.start(); + let observer = Node::new(&observed, true).start(); + let handle = target.child_handle(); + let h = handle.clone(); + on(&observer, move |ctx| { + ctx.link(&h); + ctx.monitor(&h); + }) + .await; + handle.stop(); + let (signaller, l) = (handle.clone(), log.clone()); + let at = seed % 5; + let signalling = spawn(async move { + sleep(ms(at)).await; + push(&l, "signal"); + signaller.exit(ExitReason::Shutdown); + }); + let reason = handle.wait_exit_async().await; + signalling.await.unwrap(); + let log = entries(&log); + let signal = position(&log, "signal").unwrap(); + let end = position(&log, "stopped:end").expect("a self-exit runs stopped()"); + outcome(format!("late signal: {reason:?}")); + match reason { + ExitReason::Shutdown => {} + ExitReason::Normal => assert!(end < signal, "a signal before D was lost: {log:?}"), + other => panic!("unexpected reason {other:?}: {log:?}"), + } + let downs = downs(&observer, &observed).await; + assert_eq!( + notices(&entries(&observed), "exit:"), + vec![format!("exit:{reason:?}")] + ); + assert_eq!(downs, vec![format!("down:{reason:?}")]); + stop(&observer).await; +} + +/// E4/E5: `link` + `monitor` (even seeds) or `try_link` (odd seeds) racing +/// the target's D. Each registration gets exactly one notice, the real +/// reason before D or `NoProc` after it; registrations made in one turn +/// agree; `try_link` after D gets `Err(NoProc)` and no notice. +async fn link_vs_death(seed: u64) { + let observed = Log::default(); + let target = Node::new(&Log::default(), false).start(); + let observer = Node::new(&observed, true).start(); + let handle = target.child_handle(); + let stopper = handle.clone(); + let stop_at = seed % 3; + let stopping = spawn(async move { + sleep(ms(stop_at)).await; + stopper.stop_with_reason(ExitReason::Shutdown); + }); + sleep(ms((seed / 3) % 3)).await; + let use_try = seed % 2 == 1; + let result = Arc::new(Mutex::new(None)); + let (h, r) = (handle.clone(), result.clone()); + on(&observer, move |ctx| { + if use_try { + *r.lock().unwrap() = Some(ctx.try_link(&h).is_ok()); + } else { + ctx.link(&h); + ctx.monitor(&h); + } + }) + .await; + stopping.await.unwrap(); + assert_eq!(handle.wait_exit_async().await, ExitReason::Shutdown); + let downs = if use_try { + settle(&observer).await; + Vec::new() + } else { + downs(&observer, &observed).await + }; + let observed = entries(&observed); + let exits = notices(&observed, "exit:"); + outcome(format!( + "link: try={use_try} {:?}", + exits.first().map(String::as_str) + )); + if use_try { + let linked = result.lock().unwrap().unwrap(); + let expected = if linked { + vec!["exit:Shutdown".to_string()] + } else { + vec![] + }; + assert_eq!(exits, expected, "try_link linked={linked}"); + } else { + assert_eq!(exits.len(), 1, "{observed:?}"); + assert!(["exit:Shutdown", "exit:NoProc"].contains(&exits[0].as_str())); + let down = exits[0].replace("exit:", "down:"); + assert_eq!(downs, vec![down], "link and monitor in one turn disagree"); + } + stop(&observer).await; +} + +/// E6: `unlink` + `demonitor` racing the target's death and delivery. A +/// marker sent to itself right after the fence turn is behind any notice +/// enqueued before it; no `Exit` or `Down` of that link or monitor is handled +/// after the marker, and at most one of each before it. +async fn unlink_demonitor_fence(seed: u64) { + struct Marker; + impl Message for Marker { + type Result = (); + } + impl Handler for Node { + async fn handle(&mut self, _: Marker, reply: PendingReply<'_, ()>, _: &Context) { + push(&self.log, "marker"); + reply.reply(()); + } + } + let observed = Log::default(); + let target = Node::new(&Log::default(), false).start(); + let observer = Node::new(&observed, true).start(); + let handle = target.child_handle(); + let monitor: Arc>> = Arc::default(); + let (h, m) = (handle.clone(), monitor.clone()); + on(&observer, move |ctx| { + ctx.link(&h); + *m.lock().unwrap() = Some(ctx.monitor(&h)); + }) + .await; + let stopper = handle.clone(); + let stop_at = seed % 3; + let stopping = spawn(async move { + sleep(ms(stop_at)).await; + stopper.stop_with_reason(ExitReason::Panic("gone".into())); + }); + sleep(ms((seed / 3) % 3)).await; + let (h, m) = (handle.clone(), monitor.clone()); + on(&observer, move |ctx| { + ctx.unlink(&h); + ctx.demonitor(m.lock().unwrap().unwrap()); + ctx.send(Marker).unwrap(); + }) + .await; + stopping.await.unwrap(); + handle.wait_exit_async().await; + settle(&observer).await; + let observed = entries(&observed); + let marker = position(&observed, "marker").unwrap(); + assert!( + observed[marker + 1..] + .iter() + .all(|e| !e.starts_with("exit:") && !e.starts_with("down:")), + "a notice after unlink/demonitor returned: {observed:?}" + ); + let (exits, downs) = (notices(&observed, "exit:"), notices(&observed, "down:")); + assert!(exits.len() <= 1 && downs.len() <= 1, "{observed:?}"); + outcome(format!("fence: exit={} down={}", exits.len(), downs.len())); + stop(&observer).await; +} + +/// E3: a supervisor's shutdown (the parent's exit signal) racing a trapping +/// child's queued messages. Messages queued before the parent exit are +/// handled first and in order, `exit_pending` turns true once and stays +/// true, `exit_received` never sees the parent exit, `stopped()` runs and +/// sees it pending, and the child ends `Shutdown`. +async fn supervisor_shutdown_vs_queue(seed: u64) { + let log = Log::default(); + let system = ActorSystem::new().unwrap(); + let group = system.async_group("sim-e3", 1).unwrap(); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let l = log.clone(); + let child = tree + .add( + ChildSpec::new("trapping", &group, move |_| { + let log = l.clone(); + async move { Ok::<_, String>(Node::new(&log, true)) } + }) + .shutdown_timeout(Duration::from_secs(5)), + ) + .unwrap(); + let supervisor = tree.start().await.unwrap(); + let actor = child.current().unwrap(); + let handle = actor.child_handle(); + let gap = seed % 2; + let sending = spawn(async move { + for i in 0..4 { + let _ = actor.send(Probe(i)); + sleep(ms(gap)).await; + } + }); + sleep(ms((seed / 2) % 4)).await; + assert!(supervisor.shutdown(Duration::from_secs(10)).await.complete); + sending.await.unwrap(); + assert_eq!(handle.wait_exit_async().await, ExitReason::Shutdown); + let log = entries(&log); + assert!( + notices(&log, "exit:").is_empty(), + "exit_received saw the parent exit: {log:?}" + ); + assert_eq!( + log.last().map(String::as_str), + Some("stopped:end"), + "{log:?}" + ); + assert!(log.contains(&"stopped:begin:true".to_string()), "{log:?}"); + let probes: Vec<(u32, bool)> = log + .iter() + .filter_map(|e| e.strip_prefix("probe:")) + .map(|p| { + let (i, pending) = p.split_once(':').unwrap(); + (i.parse().unwrap(), pending == "true") + }) + .collect(); + assert!( + probes.windows(2).all(|w| w[0].0 < w[1].0), + "out of order: {log:?}" + ); + assert!( + probes.windows(2).all(|w| w[0].1 <= w[1].1), + "exit_pending went back to false: {log:?}" + ); + outcome(format!( + "e3: handled={} pending={}", + probes.len(), + probes.iter().filter(|p| p.1).count() + )); + assert!(system.shutdown(Duration::from_secs(10)).await.complete); +} + +/// S1: a ready child killed while the tree's initial start is still running +/// (a slow sibling). Its exit is deferred until the tree commits ready, then +/// handled under OneForOne: `start()` succeeds, the child restarts once +/// and only after the sibling's `started()`, and the sibling is untouched. +async fn deferred_exit_vs_commit(seed: u64) { + let log = Log::default(); + let system = ActorSystem::new().unwrap(); + let group = system.async_group("sim-s1", 2).unwrap(); + let config = SupervisorConfig { + strategy: SupervisorStrategy::OneForOne, + ..SupervisorConfig::default() + }; + let mut tree = system.supervisor(config).unwrap(); + let (l, starts) = (log.clone(), Arc::new(Mutex::new(0u32))); + let first = tree + .add(ChildSpec::new("first", &group, move |_| { + let mut n = starts.lock().unwrap(); + *n += 1; + push(&l, format!("first:start:{n}")); + let log = Log::default(); + async move { Ok::<_, String>(Node::new(&log, false)) } + })) + .unwrap(); + let (l, delay) = (log.clone(), 1 + (seed / 4) % 4); + let second = tree + .add(ChildSpec::new("second", &group, move |_| { + let l = l.clone(); + async move { + sleep(ms(delay)).await; + push(&l, "second:ready"); + Ok::<_, String>(Node::new(&Log::default(), false)) + } + })) + .unwrap(); + let kill_at = seed % 4; + let (f, l) = (first.clone(), log.clone()); + let killing = spawn(async move { + let actor = f.wait_ready().await.unwrap(); + sleep(ms(kill_at)).await; + push(&l, "kill"); + actor.child_handle().kill(); + }); + let supervisor = tree + .start() + .await + .expect("a ready child's exit failed start()"); + killing.await.unwrap(); + first.wait_for_generation(1).await.unwrap(); + let log = entries(&log); + let restart = position(&log, "first:start:2").unwrap(); + let ready = position(&log, "second:ready").unwrap(); + let kill = position(&log, "kill").unwrap(); + outcome(if kill < ready { + "s1: exit during startup (deferred)" + } else { + "s1: exit after the commit" + }); + assert!( + ready < restart, + "restarted before the tree committed: {log:?}" + ); + assert!(position(&log, "first:start:3").is_none(), "{log:?}"); + assert_eq!(second.snapshot().generation, 1); + assert!(supervisor.shutdown(Duration::from_secs(10)).await.complete); + assert!(system.shutdown(Duration::from_secs(10)).await.complete); +} + +// --------------------------------------------------------------------------- +// Driver +// --------------------------------------------------------------------------- + +/// PCT draws a new seed per run; DFS keeps one input seed per search, so it +/// is repeated over the first `DFS_SEEDS` seeds. +const PCT: (&str, usize) = ("pct:d=3", 1000); +const DFS: (&str, usize) = ("dfs:bound=3,depth=300", 300); +const DFS_SEEDS: u64 = 12; + +fn check(name: &str, body: F) +where + F: Fn(u64) -> Fut, + Fut: Future, +{ + OUTCOMES.lock().unwrap().clear(); + let report = explore(&Strategy::parse(PCT.0).unwrap(), 0, PCT.1, &body); + assert!( + report.found.is_none(), + "{name} under {}: {:?}", + PCT.0, + report.found + ); + println!( + "{name} {}: {} runs, {} schedules", + PCT.0, report.runs, report.schedules + ); + let (mut runs, mut exhausted) = (0, 0); + for seed in 0..DFS_SEEDS { + let report = explore(&Strategy::parse(DFS.0).unwrap(), seed, DFS.1, &body); + assert!( + report.found.is_none(), + "{name} under {} seed {seed}: {:?}", + DFS.0, + report.found + ); + runs += report.runs; + exhausted += usize::from(report.complete()); + } + println!( + "{name} {}: {runs} runs over {DFS_SEEDS} seeds, {exhausted} searches exhausted", + DFS.0 + ); + println!("{name} outcomes: {:?}", OUTCOMES.lock().unwrap()); +} + +#[test] +fn kill_races_stopped() { + check("kill vs stopped", kill_vs_stopped); +} + +#[test] +fn late_signal_races_the_death_point() { + check("late signal vs D", late_signal_vs_death); +} + +#[test] +fn link_and_monitor_race_the_death_point() { + check("link vs D", link_vs_death); +} + +#[test] +fn unlink_and_demonitor_fence_delivery() { + check("unlink/demonitor fence", unlink_demonitor_fence); +} + +#[test] +fn supervisor_shutdown_races_a_trapping_childs_queue() { + check("supervisor shutdown vs queue", supervisor_shutdown_vs_queue); +} + +#[test] +fn ready_child_exit_races_the_tree_commit() { + check("deferred exit vs commit", deferred_exit_vs_commit); +} diff --git a/concurrency/tests/link_fence_loom.rs b/concurrency/tests/link_fence_loom.rs new file mode 100644 index 00000000..a80075bf --- /dev/null +++ b/concurrency/tests/link_fence_loom.rs @@ -0,0 +1,515 @@ +//! Loom model of the link table and the monitor fence +//! (docs/design/otp-exit-signals.md E4-E6). The production protocols +//! (`src/link/table.rs`, `src/monitor/fence.rs`) are compiled here with +//! Loom's `Mutex`; `ActorId`, `ExitReason` and `SignalTarget` are stand-ins. +//! Applying a notice (production: `deliver_signal` on the receiver's exit +//! request, under the receiver's table lock) is modelled as appending to the +//! receiver's log under its own lock. Table `Arc`/`Weak` are `std` (only the +//! locks are explored; reference counts are not part of the protocol). +//! +//! Properties: +//! - at most one notice per edge generation, and none after the receiver's +//! `unlink` returned or after the receiver's own death point; +//! - a `link` racing the peer's death point gets exactly one notice: the +//! real reason (linked first) or a late `NoProc` (linked after D), never +//! both and never none; `try_link` after D gets none; +//! - at most one `Down` per monitor, none after `demonitor` returned or after +//! the observer's own close; a monitor racing the target's D gets exactly +//! one `Down`: the real reason or `NoProc`. +//! +//! Run: `RUSTFLAGS="--cfg spawned_scheduler_loom" cargo test --release +//! -p spawned-concurrency --no-default-features --test link_fence_loom`. +#![cfg(spawned_scheduler_loom)] + +mod sync { + pub(crate) use loom::sync::{Mutex, MutexGuard}; + pub(crate) use std::sync::{Arc, Weak}; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct ActorId(u64); + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ExitReason { + Panic, + Shutdown, + NoProc, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct Notice { + from: ActorId, + gen: u64, + reason: ExitReason, +} + +/// The receiver side of a notice: its applied notices and fenced edges. +pub(crate) struct SignalTarget { + id: ActorId, + log: loom::sync::Mutex>, + /// Generations whose `unlink` (or the receiver's death) has returned. + fenced: loom::sync::Mutex>, +} + +#[allow(dead_code)] +#[path = "../src/link/table.rs"] +mod table; + +#[allow(dead_code)] +#[path = "../src/monitor/fence.rs"] +mod fence; + +use loom::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicUsize, Ordering as StdOrdering}; +use std::sync::Arc; +use table::{LinkOutcome, LinkTable}; + +fn model(name: &'static str, f: impl Fn() + Sync + Send + 'static) { + let runs = Arc::new(AtomicUsize::new(0)); + let counted = runs.clone(); + // Unbounded: the protocols are lock-based and small enough to explore + // every interleaving. + let builder = loom::model::Builder::new(); + builder.check(move || { + counted.fetch_add(1, StdOrdering::Relaxed); + f() + }); + eprintln!("{name}: {} executions", runs.load(StdOrdering::Relaxed)); +} + +struct Actor { + target: Arc, + links: LinkTable, +} + +impl Actor { + fn new(id: u64) -> Self { + Self { + target: Arc::new(SignalTarget { + id: ActorId(id), + log: loom::sync::Mutex::new(Vec::new()), + fenced: loom::sync::Mutex::new(Vec::new()), + }), + links: table::new_link_table(), + } + } + fn id(&self) -> ActorId { + self.target.id + } + fn log(&self) -> Vec { + self.target.log.lock().unwrap().clone() + } + fn fence(&self, gen: u64) { + self.target.fenced.lock().unwrap().push(gen); + } + fn records(&self) -> usize { + table::lock(&self.links).entries.len() + } +} + +/// `deliver_notice`: the receiver-side fenced step. +fn notice(links: &LinkTable, receiver: &SignalTarget, from: ActorId, gen: u64, reason: ExitReason) { + drop(table::take_record(links, from, gen, |record| { + assert_eq!(record.late, reason == ExitReason::NoProc); + assert!( + !receiver.fenced.lock().unwrap().contains(&gen), + "notice of edge {gen} took effect after unlink or death returned" + ); + let mut log = receiver.log.lock().unwrap(); + assert!( + log.iter().all(|n| n.gen != gen), + "a second notice of edge {gen}" + ); + log.push(Notice { from, gen, reason }); + })); +} + +/// `link_in_section` for a running caller. +fn link(own: &Actor, peer: &Actor, gen: u64, try_only: bool) -> LinkOutcome { + let outcome = table::install( + &own.target, + &own.links, + &peer.target, + &peer.links, + gen, + try_only, + ); + if outcome == LinkOutcome::NoProc && !try_only { + notice(&own.links, &own.target, peer.id(), gen, ExitReason::NoProc); + } + outcome +} + +/// `unregister_link`, then the edge is fenced on both sides. +fn unlink(own: &Actor, peer: &Actor, gen: u64) { + drop(table::remove_pair( + own.id(), + &own.links, + peer.id(), + &peer.links, + )); + own.fence(gen); +} + +/// The death point D, then publication (the notices taken at D). +fn die(actor: &Actor, reason: ExitReason, fenced: &[u64]) { + let notices = table::close(actor.id(), &actor.links, &reason, |_| {}); + // Past D nothing takes effect on this actor: its records are gone. + for gen in fenced { + actor.fence(*gen); + } + notices.deliver_with(notice); +} + +fn spawn(f: impl FnOnce() -> T + Send + 'static) -> loom::thread::JoinHandle { + loom::thread::spawn(f) +} + +/// `unlink` racing the peer's death point and publication: the notice +/// takes effect only before `unlink` returned, at most once. +#[test] +fn unlink_fences_a_death_notice() { + model("unlink_fences_a_death_notice", || { + let (a, b) = (Arc::new(Actor::new(1)), Arc::new(Actor::new(2))); + assert_eq!(link(&b, &a, 1, false), LinkOutcome::Linked); + let dying = { + let a = a.clone(); + spawn(move || die(&a, ExitReason::Panic, &[])) + }; + unlink(&b, &a, 1); + dying.join().unwrap(); + assert!(b.log().len() <= 1); + assert!(a.log().is_empty()); + assert_eq!(a.records() + b.records(), 0); + }); +} + +/// `link` racing the peer's death point: exactly one notice, the real +/// reason or a late `NoProc`; `try_link` after D gets none and no record. +#[test] +fn link_racing_death_gets_exactly_one_notice() { + for try_only in [false, true] { + model("link_racing_death_gets_exactly_one_notice", move || { + let (a, b) = (Arc::new(Actor::new(1)), Arc::new(Actor::new(2))); + let dying = { + let a = a.clone(); + spawn(move || die(&a, ExitReason::Shutdown, &[])) + }; + let outcome = link(&b, &a, 1, try_only); + dying.join().unwrap(); + let log = b.log(); + match outcome { + LinkOutcome::Linked => assert_eq!( + log, + vec![Notice { + from: a.id(), + gen: 1, + reason: ExitReason::Shutdown + }] + ), + LinkOutcome::NoProc if try_only => assert!(log.is_empty()), + LinkOutcome::NoProc => assert_eq!( + log, + vec![Notice { + from: a.id(), + gen: 1, + reason: ExitReason::NoProc + }] + ), + LinkOutcome::Unchanged => panic!("a first link is never unchanged"), + } + assert_eq!(a.records() + b.records(), 0); + }); + } +} + +/// Unlink then relink while the old edge's notice is in flight: the old +/// generation never takes effect after the unlink; the new one gets exactly +/// one notice (the real reason, or `NoProc` past D). +#[test] +fn relink_after_unlink_ignores_the_stale_generation() { + model("relink_after_unlink_ignores_the_stale_generation", || { + let (a, b) = (Arc::new(Actor::new(1)), Arc::new(Actor::new(2))); + assert_eq!(link(&b, &a, 1, false), LinkOutcome::Linked); + let dying = { + let a = a.clone(); + spawn(move || die(&a, ExitReason::Panic, &[])) + }; + unlink(&b, &a, 1); + let outcome = link(&b, &a, 2, false); + dying.join().unwrap(); + let log = b.log(); + assert!(log.iter().filter(|n| n.gen == 1).count() <= 1); + let fresh: Vec<_> = log.iter().filter(|n| n.gen == 2).collect(); + assert_eq!(fresh.len(), 1, "{outcome:?}: {log:?}"); + let expected = match outcome { + LinkOutcome::Linked => ExitReason::Panic, + LinkOutcome::NoProc => ExitReason::NoProc, + LinkOutcome::Unchanged => panic!("the relink found a stale record"), + }; + assert_eq!(fresh[0].reason, expected); + assert_eq!(a.records() + b.records(), 0); + }); +} + +/// Three parties: `a` dies while `b` (linked) unlinks and `c` links to it. +/// `b` gets its notice only before its unlink returned; `c` gets exactly +/// one notice (the real reason or a late `NoProc`). +#[test] +fn death_racing_unlink_and_a_new_link() { + model("death_racing_unlink_and_a_new_link", || { + let (a, b, c) = ( + Arc::new(Actor::new(1)), + Arc::new(Actor::new(2)), + Arc::new(Actor::new(3)), + ); + assert_eq!(link(&b, &a, 1, false), LinkOutcome::Linked); + let dying = { + let a = a.clone(); + spawn(move || die(&a, ExitReason::Panic, &[])) + }; + let linking = { + let (a, c) = (a.clone(), c.clone()); + spawn(move || link(&c, &a, 2, false)) + }; + unlink(&b, &a, 1); + dying.join().unwrap(); + let outcome = linking.join().unwrap(); + assert!(b.log().len() <= 1); + let expected = match outcome { + LinkOutcome::Linked => ExitReason::Panic, + LinkOutcome::NoProc => ExitReason::NoProc, + LinkOutcome::Unchanged => panic!("a first link is never unchanged"), + }; + assert_eq!( + c.log(), + vec![Notice { + from: a.id(), + gen: 2, + reason: expected + }] + ); + assert_eq!(a.records() + b.records() + c.records(), 0); + }); +} + +/// Both ends die at once: each notice takes effect at most once, and never +/// on a receiver past its own death point (its close took the record). +#[test] +fn mutual_death_delivers_at_most_one_notice() { + model("mutual_death_delivers_at_most_one_notice", || { + let (a, b) = (Arc::new(Actor::new(1)), Arc::new(Actor::new(2))); + assert_eq!(link(&a, &b, 1, false), LinkOutcome::Linked); + let dying = { + let a = a.clone(); + spawn(move || die(&a, ExitReason::Panic, &[1])) + }; + die(&b, ExitReason::Shutdown, &[1]); + dying.join().unwrap(); + assert!(a.log().len() + b.log().len() <= 1); + assert_eq!(a.records() + b.records(), 0); + }); +} + +/// Concurrent `link` from both ends: one edge, symmetric records. +#[test] +fn concurrent_links_install_one_edge() { + model("concurrent_links_install_one_edge", || { + let (a, b) = (Arc::new(Actor::new(1)), Arc::new(Actor::new(2))); + let other = { + let (a, b) = (a.clone(), b.clone()); + spawn(move || link(&b, &a, 2, false)) + }; + let mine = link(&a, &b, 1, false); + let theirs = other.join().unwrap(); + let outcomes = [mine, theirs]; + assert_eq!( + outcomes + .iter() + .filter(|o| **o == LinkOutcome::Linked) + .count(), + 1 + ); + assert!(outcomes.contains(&LinkOutcome::Unchanged)); + assert_eq!((a.records(), b.records()), (1, 1)); + // Release the records (no cycle is left behind). + die(&a, ExitReason::Panic, &[]); + die(&b, ExitReason::Panic, &[]); + }); +} + +// --------------------------------------------------------------------------- +// Monitors +// --------------------------------------------------------------------------- + +type Monitors = fence::Entries; + +/// A target: its exit table (D) and its publication flag. +struct Target { + actor: Actor, + published: AtomicBool, +} + +impl Target { + fn new() -> Arc { + Arc::new(Self { + actor: Actor::new(9), + published: AtomicBool::new(false), + }) + } + fn is_past_death(&self) -> bool { + table::lock(&self.actor.links).is_dead() + } + fn die(&self) { + drop(table::close( + self.actor.id(), + &self.actor.links, + &ExitReason::Panic, + |_| {}, + )); + self.published.store(true, Ordering::Release); + } +} + +/// The observer: its monitors, mailbox of `Down`s and fence flag. +struct Observer { + monitors: Monitors, + downs: loom::sync::Mutex>, + fenced: AtomicBool, + cancelled: AtomicBool, +} + +impl Observer { + fn new() -> Arc { + Arc::new(Self { + monitors: loom::sync::Mutex::new(Some(Default::default())), + downs: loom::sync::Mutex::new(Vec::new()), + fenced: AtomicBool::new(false), + cancelled: AtomicBool::new(false), + }) + } + fn enqueue(&self, reason: ExitReason) { + assert!( + !self.fenced.load(Ordering::Acquire), + "a Down was enqueued after demonitor or the observer's close returned" + ); + let mut downs = self.downs.lock().unwrap(); + assert!(downs.is_empty(), "a second Down"); + downs.push(reason); + } + fn downs(&self) -> Vec { + self.downs.lock().unwrap().clone() + } + fn register(&self, target: &Target) -> fence::Registration { + fence::register( + &self.monitors, + 1, + (), + || false, + || target.is_past_death(), + || self.enqueue(ExitReason::NoProc), + || {}, + ) + } + /// The waiter: woken by publication or by its token. + fn waiter(self: &Arc, target: &Arc) -> loom::thread::JoinHandle { + let (observer, target) = (self.clone(), target.clone()); + spawn(move || loop { + if observer.cancelled.load(Ordering::Acquire) { + return false; + } + if target.published.load(Ordering::Acquire) { + return fence::deliver(&observer.monitors, &1, || { + observer.enqueue(ExitReason::Panic) + }); + } + loom::thread::yield_now(); + }) + } +} + +/// `demonitor` racing the target's death and the waiter: once it returns no +/// `Down` is enqueued; if it found no entry, the `Down` was delivered. +#[test] +fn demonitor_fences_the_down() { + model("demonitor_fences_the_down", || { + let (observer, target) = (Observer::new(), Target::new()); + assert_eq!(observer.register(&target), fence::Registration::Active); + let waiter = observer.waiter(&target); + let dying = { + let target = target.clone(); + spawn(move || target.die()) + }; + let removed = fence::remove(&observer.monitors, &1, || {}); + observer.fenced.store(true, Ordering::Release); + if removed.is_some() { + observer.cancelled.store(true, Ordering::Release); + } + dying.join().unwrap(); + let delivered = waiter.join().unwrap(); + assert_eq!(delivered, removed.is_none()); + let expected = if removed.is_some() { + vec![] + } else { + vec![ExitReason::Panic] + }; + assert_eq!(observer.downs(), expected); + }); +} + +/// `monitor` racing the target's death point: exactly one `Down`, the real +/// reason (registered before D) or `NoProc` (after D). +#[test] +fn monitor_racing_death_gets_exactly_one_down() { + model("monitor_racing_death_gets_exactly_one_down", || { + let (observer, target) = (Observer::new(), Target::new()); + let dying = { + let target = target.clone(); + spawn(move || target.die()) + }; + // Seen published before the call: the registration is after D. + let dead_before = target.published.load(Ordering::Acquire); + let registration = observer.register(&target); + if dead_before { + assert_eq!(registration, fence::Registration::Late, "monitor after D"); + } + let delivered = match registration { + fence::Registration::Active => observer.waiter(&target).join().unwrap(), + fence::Registration::Late => false, + fence::Registration::Inert => panic!("a running observer is never inert"), + }; + dying.join().unwrap(); + let expected = match registration { + fence::Registration::Active => { + assert!(delivered); + ExitReason::Panic + } + _ => ExitReason::NoProc, + }; + assert_eq!(observer.downs(), vec![expected]); + }); +} + +/// The observer's own exit racing the delivery: at most one `Down`, none +/// after the close returned; a monitor after the close is inert. +#[test] +fn observer_close_fences_the_down() { + model("observer_close_fences_the_down", || { + let (observer, target) = (Observer::new(), Target::new()); + assert_eq!(observer.register(&target), fence::Registration::Active); + let waiter = observer.waiter(&target); + let dying = { + let target = target.clone(); + spawn(move || target.die()) + }; + let taken = fence::close(&observer.monitors); + observer.fenced.store(true, Ordering::Release); + if taken.is_some_and(|entries| !entries.is_empty()) { + observer.cancelled.store(true, Ordering::Release); + } + assert_eq!(observer.register(&target), fence::Registration::Inert); + dying.join().unwrap(); + let delivered = waiter.join().unwrap(); + assert_eq!(observer.downs().len(), usize::from(delivered)); + }); +} diff --git a/concurrency/tests/otp_exit_signals.rs b/concurrency/tests/otp_exit_signals.rs index b9b392ec..da6dbcac 100644 --- a/concurrency/tests/otp_exit_signals.rs +++ b/concurrency/tests/otp_exit_signals.rs @@ -820,6 +820,10 @@ mod tasks_mode { let seen = Seen::default(); let killed = Node::new(&seen, true).start(); settle(&killed).await; + // Another test thread may have been registering the + // `stopped_skipped` callsite while `set_default` rebuilt the + // interest cache, leaving it cached as "never". + tracing::callsite::rebuild_interest_cache(); killed.child_handle().kill(); assert_eq!(ended(&killed.child_handle()).await, ExitReason::Killed); let signalled = Node::new(&seen, false).start(); diff --git a/concurrency/tests/supervision.rs b/concurrency/tests/supervision.rs index 7d02a14c..63110887 100644 --- a/concurrency/tests/supervision.rs +++ b/concurrency/tests/supervision.rs @@ -437,6 +437,9 @@ async fn stuck_stopped_hook_triggers_cleanup_deadline_before_terminal_publicatio let report = system.shutdown(Duration::from_millis(5)).await; assert!(!report.complete); assert!(report.unfinished_actors.contains(&old.id())); + // Let the system's kill at grace land before releasing stopped(): the + // report can return before the kill reaches the actor under load. + spawned_rt::tasks::sleep(Duration::from_millis(100)).await; release.send(()).unwrap(); let report = report.wait(LIMIT).await; assert!(report.complete); From 7a709cd6e952b4e0316a6f0e5943740c9e691d6c Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 03:34:48 -0300 Subject: [PATCH 093/208] docs(exit): GUIDE exit-signal section, exit_signals example, semantics rows GUIDE: "Links, monitors and exit signals" (exit/kill rules, trapping, parent exits, NoProc and try_link, unlink/demonitor fences, runtime drop) replaces the v0.7 wording; supervisor startup follows S1/S2; the faults::exit row describes the exact exit signal. New capability example exit_signals (run in CI). semantics.md L3-L6, X1, X3-X6, V1, V3, V7, V9 and the OTP differences table point at the Loom model, simulator tests and Lifecycle.tla invariants. TESTING and CHANGELOG describe the new suites. --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 14 +- docs/GUIDE.md | 215 +++++++++++---- docs/TESTING.md | 45 ++- docs/design/semantics.md | 256 ++++++++++++------ examples/README.md | 1 + examples/capabilities/README.md | 2 + examples/capabilities/src/bin/exit_signals.rs | 246 +++++++++++++++++ 8 files changed, 633 insertions(+), 148 deletions(-) create mode 100644 examples/capabilities/src/bin/exit_signals.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 538fab8c..8d9c6e56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: run: cargo test --locked --verbose - name: Run capability examples run: | - for example in execution_groups execution_profiles supervision_recovery supervision_tree dedicated_thread runtime_isolation byte_budget_pipeline timeout_outcomes graceful_drain protocol_timeouts streams_and_recipients registrations_and_fakes native_lifecycle coordinated_shutdown test_kit; do + for example in execution_groups execution_profiles supervision_recovery supervision_tree dedicated_thread runtime_isolation byte_budget_pipeline timeout_outcomes graceful_drain protocol_timeouts streams_and_recipients registrations_and_fakes native_lifecycle coordinated_shutdown test_kit exit_signals; do timeout 60s cargo run --locked -p spawned-capability-examples --bin "$example" done timeout 60s cargo run --locked -p spawned-capability-examples --features metrics --bin framework_metrics diff --git a/CHANGELOG.md b/CHANGELOG.md index fdfdb7c0..7a66bd7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,7 +75,19 @@ `Context::exit_pending()` (both modes) tells a trapping actor that its exit began or a parent exit is queued behind the current message, so it can refuse late work (advisory, read-only, no per-message cost). - Remaining: Loom and simulator coverage, GUIDE and examples. + Concurrency checks: the link table (`src/link/table.rs`) and the monitor + fence (`src/monitor/fence.rs`) are shared with a Loom model + (`tests/link_fence_loom.rs`, 9 tests explored exhaustively: one notice per + edge, none after `unlink`/`demonitor` or the receiver's death, a link or + monitor racing the death point gets the real reason or `NoProc`, never both + nor none); `tests/exit_signals_sim.rs` explores six orderings on the + simulator with PCT and bounded DFS (kill vs `stopped()`, late signal vs the + death point, link/monitor vs the death point, unlink/demonitor fences, + supervisor shutdown vs a trapping child's queue and `exit_pending`, a ready + child's exit vs the tree commit). GUIDE "Links, monitors and exit signals", + the `exit_signals` capability example, and semantics L3-L6, X1-X6, V1, V3, + V9 and the OTP differences table. + Remaining: the heavy TLC configurations and Apalache after the canaries. - **Breaking (0.8), in progress:** deferred replies and pipe-to-self (ROADMAP rank 11), implementing diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 24f3c2cb..cbda6dea 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -24,7 +24,7 @@ or model that checks each one, are collected in the - [Message Trait](#message-trait) - [Error Handling](#error-handling) - [spawned-rt](#spawned-rt) -- [Links and trapped exits](#links-and-trapped-exits) +- [Links, monitors and exit signals](#links-monitors-and-exit-signals) (v0.8) - [Execution groups and supervision](#execution-groups-and-supervision) (v0.7) - [Metrics](#metrics) - [Testing your actors](#testing-your-actors) @@ -67,7 +67,7 @@ All three phases are wrapped in `catch_unwind`: - **Panic in `started()`** — the actor stops immediately. No messages are processed, `stopped()` is not called. - **Panic in a handler** — the current message is lost. The actor stops and `stopped()` is called. -- **Panic in `stopped()`** — the panic is logged and `join()` still returns, but the exit reason becomes `Panic("panic in stopped(): ...")` unless it was already abnormal. +- **Panic in `stopped()`** — the panic is logged and `join()` still returns, but the exit reason becomes `Panic("panic in stopped(): ...")` unless the actor was killed (a kill's `Killed` stands). In all cases, `join()` will eventually return and subsequent `send()`/`request()` calls return `Err(NoProc)`. @@ -129,9 +129,12 @@ State that must be released on every exit, including kills and runtime drops, be | `ctx.actor_ref()` | Get an `ActorRef` from the context | | `ctx.id()` | This actor's `ActorId` | | `ctx.monitor(&child_handle)` | Receive a `Down` message when the target exits; returns a `MonitorRef` | -| `ctx.demonitor(monitor_ref)` | Cancel a monitor | -| `ctx.link(&child_handle)` / `ctx.unlink(&child_handle)` | Bidirectional exit propagation (see [Links and trapped exits](#links-and-trapped-exits)) | -| `ctx.trap_exit(bool)` | Receive linked exits as messages instead of being cancelled | +| `ctx.demonitor(monitor_ref)` | Cancel a monitor (a fence: no later `Down`) | +| `ctx.link(&child_handle)` / `ctx.unlink(&child_handle)` | Bidirectional exit propagation (see [Links, monitors and exit signals](#links-monitors-and-exit-signals)) | +| `ctx.try_link(&child_handle)` | Link, or `Err(NoProc)` if the target is already gone | +| `ctx.exit(&child_handle, reason)` | Send an exit signal from this actor (OTP `exit/2`) | +| `ctx.trap_exit(bool)` | Receive exit signals and link notices as `Exit` messages instead of exiting | +| `ctx.exit_pending()` | Whether this actor's exit has begun or a parent exit is queued | | `ctx.spawn_helper(future)` / `ctx.spawn_blocking_helper(f)` | Actor-owned helper tasks, cancelled and joined before cleanup completes (v0.7) | | `ctx.register_name(name)` | Register this actor under a name, released with its generation (v0.7) | | `ctx.execution_group()` | The `ExecutionGroup` a managed actor runs on (v0.7) | @@ -734,55 +737,144 @@ Wraps standard library primitives: See [Execution Modes and Backends](#execution-modes-and-backends) for choosing between `tasks` and `threads`. + -## Links and trapped exits +## Links, monitors and exit signals -Available since v0.6.0. +Exit signals follow Erlang/OTP (v0.8, [design](design/otp-exit-signals.md)). +Everything in this section works in both modes and across modes; use the +`threads` imports and synchronous hooks in synchronous mode. The +[exit_signals example](../examples/capabilities/src/bin/exit_signals.rs) runs +each rule below. -Links are bidirectional failure notifications. Use `ctx.link(&child_handle)` and -`ctx.unlink(&child_handle)` for existing actors, or `child.start_linked(&parent_ctx)` -to perform registration before the child starts. Child handles allow links across -execution modes. Normal peer exits are ignored by non-trapping actors; abnormal -exits request cooperative cancellation and propagate their cause. +### Exit signals -Call `ctx.trap_exit(true)` to receive linked exits through `Actor::exit_received` -instead. The actor macro supports this lifecycle hook: +`ChildHandle::exit(reason)` sends an exit signal from outside any actor +(`from` is `ActorId::EXTERNAL`); `ctx.exit(&target, reason)` sends it from +the calling actor; `ChildHandle::kill()` is `exit(ExitReason::Kill)`. The +target applies OTP's `exit_signal/2` rules, with the trap flag read when the +signal arrives: + +| Signal | Non-trapping target | Trapping target | +|---|---|---| +| `Kill` | ends `Killed` | ends `Killed` (untrappable) | +| `Normal` | ignored | `Exit { from, reason: Normal }` queued | +| any other reason | ends with that reason after its current handler, without `stopped()`; its queue is discarded | `Exit { from, reason }` queued | +| from the target's parent | as above | a parent exit is queued (see [Parent exits](#parent-exits)) | + +A kill supersedes any other reason until the actor's death point: an actor +already stopping (even inside `stopped()`) ends `Killed`, and a panic after +the kill does not replace it. `stopped()` is skipped if it has not begun; a +`stopped()` that is running finishes, since Rust cannot interrupt it. A +non-kill signal to a non-trapping actor that is already exiting replaces its +reason, unless it was killed; a signal after the death point does nothing. +Signals that arrive while `started()` runs take effect when it returns, with +the trap flag read then, so `ctx.trap_exit(true)` first thing in `started()` +covers them. + +`stop()`, `shutdown()` and `stop_with_reason(r)` (on a `ChildHandle` or the +`Context`) are not exit signals. They are administrative stops, as +`gen_server:stop`: untrappable, `stopped()` runs, and the first stop's reason +wins over later stops. `stop_with_reason(ExitReason::Kill)` is such a stop +and ends `Kill`; only a `Kill` exit signal (`kill()`, `exit(Kill)`) +produces `Killed`. + +### Links and trapped exits + +Links are bidirectional. Use `ctx.link(&child_handle)` and +`ctx.unlink(&child_handle)` for existing actors, or +`child.start_linked(&parent_ctx)`, which links before the child's `started()` +and makes the caller its parent. Repeated links and links to oneself are +no-ops. When a linked actor dies, each peer receives a link notice with its +final reason, under the same rules as an exit signal except that a `Kill` +notice is trappable: a non-trapping peer ignores `Normal` and otherwise ends +with the same reason, without `stopped()`, and propagates it to its own links. + +Call `ctx.trap_exit(true)` to receive signals and notices through +`Actor::exit_received` instead: ```rust use spawned_concurrency::{actor, Exit}; use spawned_concurrency::tasks::{Actor, Context}; -struct Supervisor; +struct Watcher; #[actor] -impl Supervisor { +impl Watcher { #[started] async fn setup(&mut self, ctx: &Context) { ctx.trap_exit(true); } #[exit_received] - async fn child_exit(&mut self, exit: Exit, _ctx: &Context) { + async fn peer_exit(&mut self, exit: Exit, _ctx: &Context) { tracing::warn!(actor = %exit.from, reason = ?exit.reason, "Linked actor exited"); } } ``` -Use synchronous methods and the `threads` imports in synchronous mode. The hook -is optional and defaults to doing nothing. Trapped exits use the ordinary actor -mailbox; they do not interrupt a running handler. A direct -`stop_with_reason(ExitReason::Kill)` bypasses trapping and requests cooperative -cancellation. Propagated Kill notices are trappable, so they cannot bypass -another actor's failure boundary. The notice keeps the existing Kill enum value. A normal stop already committed is -not relabelled by a later peer failure; callback failures are still reported. - -A link to a closed peer creates no retained relationship and applies that peer's -exit-signal rules for the attempt. Repeated live links are no-ops. Unlink prevents -future propagation through the relationship but does not retract an exit already -in flight. Links are a supervision primitive. Managed OneForOne/OneForAll/RestForOne recovery is described in -[execution groups and supervision](#execution-groups-and-supervision). -Transparent reference rebinding remains separate work. The [review and regression evidence](../legacy/docs/reviews/pr-166-links.md) is archived. +The hook is optional and defaults to doing nothing. `Exit`, `Down` and +parent exits are ordinary mailbox messages: they queue behind messages already +there and never interrupt a running handler. A death notice follows every +message the dying actor sent earlier to the same receiver. A link notice is +queued before the dying actor's `wait_exit()` returns; a `Down` is queued by +the monitor shortly after. + +A link or monitor made through a `Context` whose actor is already exiting (for +example from a helper that outlives it) is inert: no record and no notice. + +### Parent exits + +An actor's parent is fixed when it starts: the actor passed to +`start_linked`, the supervisor of a supervised child, or the `ActorSystem` of +a standalone `start_on` actor; a plain `.start()` actor has none. A signal +from the parent to a **trapping** actor is queued as a parent exit: when +dispatched, the actor exits with that reason (`Normal` included) and +`stopped()` runs; `exit_received` never sees it. A non-trapping actor treats +it as any exit signal. + +Supervisor and system shutdown are the parent's `Shutdown` signal. A +non-trapping child ends `Shutdown` without `stopped()`. A trapping child +handles the messages queued before the shutdown, then runs `stopped()` +([`exit_pending()`](#when-stopped-does-not-run) reports the queued shutdown). +A child that has not reached its death point `shutdown_timeout` after its +signal is killed, and reported stuck after a second interval; a system kills +the actors left when its `grace` expires. + +### NoProc + +An actor is gone from its death point on, which comes right after `stopped()` +returns (or is skipped). After it: + +- `ctx.link(&h)` delivers one `Exit { from: h, reason: NoProc }` to a trapping + caller; a non-trapping caller receives that exit signal and ends `NoProc`; +- `ctx.try_link(&h)` returns `Err(NoProc)` and keeps no link; +- `ctx.monitor(&h)` delivers one `Down { reason: NoProc }`; +- `send`/`request` return `Err(ActorError::NoProc)`. + +Before it, including while the target runs `stopped()`, links and monitors +register and fire with the final reason. `ChildHandle::exit_reason()` and +`wait_exit()` give the real reason once it is published, shortly after the +death point. + +### Unlink and demonitor are fences + +Once `ctx.unlink(&h)` returns, that link has no further effect on the caller; +once `ctx.demonitor(r)` returns, no `Down` of that monitor is queued. An +`Exit` or `Down` already in the mailbox stays. Linking again creates a new +link that gets its own notice. + +### Runtime drop + +An actor whose runtime drops it (runtime shutdown, abort, never polled) ends +`Killed` in both modes, unless a reason other than `Normal` was already +requested. `stopped()` does not run; links receive `Killed` and monitors +`Down { reason: Killed }`. + +Callers see the same final reason: see [Error Handling](#error-handling) +(`ActorStopped(reason)`, `NoProc`). Supervisor startup rules are in +[Construct fresh children after failure](#construct-fresh-children-after-failure). --- @@ -950,8 +1042,8 @@ let report = supervisor.shutdown(Duration::from_secs(5)).await; ``` A factory may return an error or panic. Registered construction helpers are -cancelled and joined before retry, including when the factory never returns an -actor. Existing `started()` and `stopped()` hooks keep their signatures. Readiness +cancelled and joined before a replacement is retried, including when the +factory never returns an actor. Existing `started()` and `stopped()` hooks keep their signatures. Readiness commits after `started()` returns and admission remains open; no ordinary handler runs before that barrier. Enqueued startup messages are discarded if startup fails. Calling `ready()` later reports that historical barrier, not current liveness. @@ -963,11 +1055,23 @@ Drop, not only by `stopped()`. Legacy unmanaged `.start()` and synchronous `threads` actors retain their behavior of running both hooks after an early cooperative stop, provided startup itself succeeds. Failed readiness waits for the actual startup outcome, so a non-returning hook -can keep it pending; use a timeout. A startup deadline is a failed attempt and can retry under Permanent/Transient -policy after cleanup, whether it expired in the factory or in `started()`. -A self-requested normal stop before readiness is still a -startup failure: a Transient/Temporary spec fails the tree rather than claiming -a ready service; a Permanent spec can retry within its budget. +can keep it pending; use a timeout. + +The initial start follows OTP's `start_link` (v0.8): + +- A child's **first start** that fails for any reason (factory error or panic, + panic or stop in `started()`, a missed `startup_timeout`, placement or group + closure) fails `start()` whatever the restart mode; it is not retried. Retry + in the application if a dependency may be briefly unavailable at boot. +- The started children roll back in reverse order, and `start()` returns its + error only after every child is terminal (or `Failed` once its cleanup is + stuck): `SupervisorError::FailedToStartChild { child, cause, rollback_failures }`, + or `StartupCleanupStuck`. +- A ready child that exits while later siblings are still starting is + recorded; its exit is handled under the strategy once the whole tree is + ready, as OTP handles a child's `'EXIT'` after `init` returns. +- A replacement that fails before readiness is a failed restart attempt and + is retried under the restart policy and budget. A supervisor accepts only groups owned by its ActorSystem. ActorRefs can still communicate between independent systems. Children start in declaration order @@ -997,7 +1101,7 @@ Supervision emits structured `tracing` events: `restart_scheduled` and at WARN. Fields include `child`, `generation`, `reason` and an available error. Enable an application tracing subscriber to collect them; Spawned installs none. -| Restart mode while supervisor is running | Normal / Shutdown | Panic / Kill | +| Restart mode while supervisor is running | Normal / Shutdown | Panic / Kill / Killed / NoProc | |---|---|---| | Permanent (default) | Restart | Restart | | Transient | Stay stopped | Restart | @@ -1010,14 +1114,13 @@ before cooperative cancellation while preserving a previously committed cause; actual handler or cleanup panics remain failures. Existing link propagation still applies and can affect siblings explicitly linked by the application. Receiver-owned cleanup discards queued envelopes individually during completion, -cancellation and unwinding in both modes. An otherwise completed Normal exit -becomes `Panic("queued message destructor: ...")` when a queued destructor fails. -Committed Shutdown, Kill and linked/handler panics keep their primary reason; -`child_handle.mailbox_cleanup_failure()` records the secondary destructor failure -independently. In particular, canceled managed startup remains Shutdown and does -not become restartable because of discarded messages. Runtime-abort fallback -Kill is retained. Every accepted queued reply sender is released before actor -completion, even while ActorRefs remain alive. +cancellation and unwinding in both modes. Queued messages and the actor state +are dropped after the death point, so a panicking destructor does not change +the exit reason; it is recorded in `child_handle.cleanup_failure()`, and a +Transient child whose reason is `Normal` or `Shutdown` restarts when its +cleanup failed. An actor dropped by its runtime ends `Killed`. Every accepted +queued request is answered before actor completion, even while ActorRefs +remain alive. Async sends retain their channel fast path; receiver cleanup closes Tokio's channel and receives through EOF without cooperative-budget deferral. Native @@ -1030,9 +1133,10 @@ user value cannot be recovered by the envelope boundary. `ctx.stop_with_reason(ExitReason::Shutdown)` requests a cooperative shutdown; `ExitReason::Kill` requests an abnormal exit without panicking. Both modes finish -the current handler and run `stopped()`. The first committed reason wins, while -an actual cleanup panic can still make the exit abnormal. A transient supervisor -restarts Kill exits but leaves Shutdown exits stopped. `ctx.stop()` keeps Normal. +the current handler and run `stopped()`. The first stop's reason wins over later +stops; a panic, a kill or an untrapped exit signal before the death point still +replaces it. A transient supervisor restarts Kill exits but leaves Shutdown +exits stopped. `ctx.stop()` keeps Normal. The default shared budget permits five retry attempts per rolling 30-second window. Initial construction does not consume a retry; every subsequent attempt @@ -1070,11 +1174,10 @@ notifications are coalesced into a bounded per-child pending set, with one recov cohort at a time. A late old-generation failure is absorbed by its assigned round. A new upstream failure interrupts reconstruction and starts a fresh cleanup round, so dependent factories cannot hold recovery behind a dead upstream. -Initial startup remains serial, retrying a failed construction attempt before -starting dependents. For OneForAll/RestForOne, if an already-ready dependency -exits restartably before the whole tree commits readiness, startup fails and rolls -back the tree. It does not replace that dependency alone underneath initializing -children. A parent supervisor can retry that whole subtree from its factory. +Initial startup is serial and does not retry a failed first start. If an +already-ready dependency exits before the whole tree commits readiness, its +exit is deferred until then and handled under the strategy, so it is not +replaced alone underneath initializing children. The rolling restart budget charges every subsequent factory invocation, including collateral reconstruction and failed constructors. For a large OneForAll tree, @@ -1431,7 +1534,7 @@ capture it when called, since `#[track_caller]` does not reach into an | Fault | How | |---|---| -| Stop any actor with a chosen reason | `faults::exit(&handle, reason)`: `Normal` calls `stop()` and `Shutdown` calls `shutdown()`; both stop the actor even if it traps exits (unlike Erlang's `exit(Pid, normal)`, which a non-trapping process ignores). `Panic(msg)` arrives from a crashed linked peer and can be trapped. `Kill` uses a direct `ChildHandle::stop_with_reason(Kill)`, bypassing the target's trapping flag in both modes; its linked peers can trap the resulting notice. Because the panic travels through a link, the actor exits with `Panic("panic in handler: {msg}")`, and the default panic hook prints `msg`. An already-exited target is left alone. Never blocks the caller, even on a `current_thread` runtime. | +| Send an exit signal with a chosen reason | `faults::exit(&handle, reason)` is `ChildHandle::exit(reason)` (see [Exit signals](#exit-signals)): `Normal` is ignored by a non-trapping target; `Kill` ends the target `Killed`, trapping or not; any other reason (`Panic(msg)` included) ends a non-trapping target with exactly that reason, without `stopped()`, and reaches a trapping one as an `Exit`. For an untrappable stop that runs `stopped()`, use `ChildHandle::stop`/`stop_with_reason`. An already-exited target is left alone. Never blocks the caller, even on a `current_thread` runtime. | | Crash a supervised child and get its replacement | `tasks::restart_child(&child, reason, deadline)`: the returned instance always replaces the one it targeted, even if the child was already restarting; `deadline` bounds the whole call | | Slow dependency / forced request timeout | `Mock::delay::(d)` with a caller timeout shorter than `d` (`with_request_timeout`, `#[protocol(timeout = ..)]`) | | Lost message | `Mock::discard::()` (fire-and-forget messages: a request returning `Response<()>` still gets `()`; to lose a request, `delay` it past the caller's timeout) | diff --git a/docs/TESTING.md b/docs/TESTING.md index 28b6e10f..5a3e3f8d 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -415,6 +415,31 @@ RUSTFLAGS="--cfg spawned_sim --cfg tokio_unstable" CARGO_TARGET_DIR=target/sim \ cargo test -p spawned-concurrency --test deferred_sim --test deferred_replies --test deferred_jobs ``` +`exit_signals_sim` explores the exit-signal orderings with PCT (`d=3`, 1,000 +runs) and bounded DFS (`bound=3`, `depth=300`, up to 300 runs for each of +input seeds 0-11): a kill against an administrative stop's slow `stopped()` +(skipped before it begins, finishes during it, ignored after the death point; +a monitor sees the same reason), a late untrapped signal against the death +point (replaces the reason before it; link and monitor agree), a `link` + +`monitor` or `try_link` against the target's death (one notice each, the real +reason or `NoProc`, the same for both registrations of one turn), `unlink` + +`demonitor` against delivery (nothing after a marker sent right after the +fence), a supervisor's shutdown against a trapping child's queued messages +(handled in order, `exit_pending` monotone, `stopped()` runs, no +`exit_received`), and a ready child's kill against the tree's initial commit +(start succeeds, one restart, after the slow sibling is ready). Each test +counts the outcomes its assertions distinguish and every one is reached. The +DFS searches are exhausted for every seed of the four single-actor workloads +and for 10 of 12 seeds of the supervisor shutdown one; the startup workload +reaches its 300-run budget for every seed. Planted +bugs (a late signal ignored, a kill that does not skip `stopped()`, a monitor +that ignores the death point) are found in the first run. + +```sh +RUSTFLAGS="--cfg spawned_sim --cfg tokio_unstable" CARGO_TARGET_DIR=target/sim \ + cargo test -p spawned-concurrency --test exit_signals_sim -- --nocapture +``` + ### Exploration strategies (`sim::explore`) The random walk can be swapped for a systematic strategy with @@ -531,7 +556,8 @@ models that inject Loom's synchronization types (`super::sync`): ```sh RUSTFLAGS="--cfg spawned_scheduler_loom" cargo test --release -p spawned-concurrency \ - --no-default-features --test reply_slot_loom --test registry_turn_loom --test job_core_loom + --no-default-features --test reply_slot_loom --test registry_turn_loom --test job_core_loom \ + --test link_fence_loom ``` `reply_slot_loom` covers the packed slot word (reply, expiry, shortening, @@ -544,6 +570,23 @@ final destruction after a `Weak` upgrade (modelled `Arc`/`Weak`; the destruction takes the job set's locks, so running it under one is a Loom deadlock). Each model was checked against a planted bug. +`link_fence_loom` shares the link table (`src/link/table.rs`: edge +generations, the two-lock install, receiver-side fenced delivery, the close at +the death point) and the observer's monitor table (`src/monitor/fence.rs`) +with production in the same way; `ActorId`, `ExitReason` and the receiver are +stand-ins, and applying a notice is an append under the receiver's lock. Its +nine tests are explored without a preemption bound (5 to 382 executions +each): at most one notice per edge generation, none after the receiver's +`unlink` returned or after its own death point, exactly one notice (real +reason or `NoProc`) for a `link` racing the peer's death and none for a +`try_link` after it, a stale generation ignored after unlink and relink, one +edge from concurrent links; at most one `Down`, none after `demonitor` or the +observer's close returned, exactly one (real reason or `NoProc`) for a +`monitor` racing the target's death. Seven planted bugs (no generation check, +unlink keeping the receiver's record, close keeping records, install ignoring +the peer's death, `Down` enqueued outside the lock, registration ignoring the +death point, a close that keeps the table open) each fail at least one test. + Run sustained actor creation, traffic, monitoring, and shutdown: ```sh diff --git a/docs/design/semantics.md b/docs/design/semantics.md index 98a04c4a..cebc6daa 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -186,45 +186,64 @@ Checked by: [`startup_admission`][startup_admission], [`execution_groups::failed_startup_discards_queued_payloads_and_reply_channels_without_dispatch`][execution_groups]. **L3. Stop is cooperative.** `ctx.stop()`, `ctx.stop_with_reason(r)`, -`ChildHandle::stop()`, `stop_with_reason(r)` and `shutdown()` request a stop: -the running handler finishes, `stopped()` runs, then the actor exits. Nothing -preempts a running handler or blocking code. The first committed reason wins; a -later stop or linked failure does not relabel it, but an actual panic in -cleanup still makes the exit abnormal. -Checked by: [`context_exit`][context_exit] (both modes), -`src/link_regression_tests.rs::late_link_failure_cannot_relabel_an_already_requested_normal_stop`. - -**L4. Exit reasons.** Every actor publishes exactly one `ExitReason`: +`ChildHandle::stop()`, `stop_with_reason(r)` and `shutdown()` are self-exits +or administrative stops: they ignore `trap_exit`, the running handler +finishes, `stopped()` runs, then the actor exits. Nothing preempts a running +handler or blocking code, a kill included. The first stop's reason wins over +later stops; until the death point D a kill makes it `Killed`, and a panic or +an untrapped exit signal or link notice replaces it unless a kill was received +(OTP: a crash or signal during `terminate` replaces the reason). After D the +reason is final. +Checked by: `Lifecycle!KilledFromKill`, `Lifecycle!LateSignalWins` (TLC); +[`context_exit`][context_exit] (both modes), `src/link.rs::tests::reason_precedence_until_death`, +`src/link_regression_tests.rs::late_link_failure_relabels_an_already_requested_normal_stop`, +[`otp_exit_signals`][otp_exit_signals]`::tasks_mode::{late_signal_replaces_the_requested_reason, kill_during_stopped_supersedes_and_panic_does_not_replace_it}`; +[`exit_signals_sim`][exit_signals_sim]`::{kill_races_stopped, late_signal_races_the_death_point}`. + +**L4. Exit reasons.** Every actor publishes exactly one `ExitReason` +(`#[non_exhaustive]`): | Reason | Arises from | |---|---| -| `Normal` | `ctx.stop()`, `ChildHandle::stop()` | -| `Shutdown` | `ChildHandle::shutdown()`, `stop_with_reason(Shutdown)`, supervisor or system shutdown, a linked peer's `Shutdown` (non-trapping) | -| `Panic(msg)` | a panic in `started()` (`"panic in started(): .."`), a handler, `exit_received` or `Down` handler (`"panic in handler: .."`), `stopped()` (`"panic in stopped(): .."`, only if not already abnormal); a linked peer's `Panic`; a destructor panic on an otherwise `Normal` exit (`"actor state destructor: .."`, `"queued message destructor: .."`) | -| `Kill` | `stop_with_reason(Kill)`, a linked peer's `Kill`; tasks only: the actor future dropped without completing (runtime shutdown, abort, never polled), unless a non-`Normal` cause was already committed | - -`is_abnormal()` is true for `Panic` and `Kill`. +| `Normal` | `ctx.stop()`, `ChildHandle::stop()`, a parent's `Normal` exit to a trapping child | +| `Shutdown` | `ChildHandle::shutdown()`, `stop_with_reason(Shutdown)`, supervisor or system shutdown, an exit signal or link notice with `Shutdown` (non-trapping) | +| `Panic(msg)` | a panic in `started()` (`"panic in started(): .."`), a handler, `exit_received` or `Down` handler (`"panic in handler: .."`), `stopped()` (`"panic in stopped(): .."`); an exit signal or link notice with `Panic` | +| `Kill` | `stop_with_reason(Kill)` (a stop, not a kill), a `Kill` link notice (non-trapping) | +| `Killed` | `ChildHandle::kill()`, `exit(Kill)`, a supervisor's or system's kill escalation, a `Killed` link notice; the runtime dropping the actor unless a non-`Normal` reason was requested | +| `NoProc` | a `link` to an actor past its death point (non-trapping caller), a `NoProc` signal or notice | + +`is_abnormal()` is true for `Panic`, `Kill`, `Killed` and `NoProc`. +Destructor panics after the death point do not change the reason (L6). Checked by: [`exit_signal_matrix`][exit_signal_matrix], [`context_exit`][context_exit], -[`panic_safety`][panic_safety], `src/link_regression_tests.rs::never_polled_actor_propagates_runtime_shutdown_to_peer`, -`src/actor_cleanup.rs::destructor_failure_only_escalates_normal_even_without_a_committed_cause`, -[`exit_reason`][ex_exit_reason] and [`native_lifecycle`][ex_native_lifecycle] examples. +[`otp_exit_signals`][otp_exit_signals], [`panic_safety`][panic_safety], +`src/link_regression_tests.rs::never_polled_actor_propagates_runtime_shutdown_to_peer`, +[`exit_reason`][ex_exit_reason], [`native_lifecycle`][ex_native_lifecycle] and +[`exit_signals`][ex_exit_signals] examples. **L5. Panics in hooks.** A panic in `started()` stops the actor without calling `stopped()` and without handling any message. A panic in a handler loses that message, then `stopped()` runs. A panic in `stopped()` is caught; `join()` -returns and the reason becomes `Panic` unless it was already abnormal. +returns and the reason becomes `Panic`, replacing an earlier requested reason +unless a kill was received (then `Killed` stands). `stopped()` does not run +after a kill, an untrapped exit signal or link notice, or a runtime drop +(see X3, X4); one that already began finishes. Checked by: `src/{tasks,threads}/actor.rs::panic_in_started_stops_actor`, `panic_in_handler_stops_actor`, `panic_in_stopped_still_completes`, -`panic_during_early_stop_is_reported`; [`panic_safety`][panic_safety]. - -**L6. Destructor panics are secondary.** A panicking `Drop` of the actor state -or of a queued message turns only an otherwise `Normal` exit into `Panic`; a -committed `Shutdown`, `Kill` or panic keeps its reason and the queued failure -is kept in `ChildHandle::mailbox_cleanup_failure()`. A panic payload whose own -destructor panics is forgotten (`dispose_panic`) to stop the cascade. +`panic_during_early_stop_is_reported`; [`panic_safety`][panic_safety]; +`Lifecycle!SkipHook` (TLC); +[`otp_exit_signals`][otp_exit_signals]`::tasks_mode::{kill_during_stopped_supersedes_and_panic_does_not_replace_it, stopped_skipped_log::skipped_stopped_is_logged_at_debug}`; +[`exit_signals_sim::kill_races_stopped`][exit_signals_sim]; lifecycle traces +`kill_during_stopped`, `untrapped_signal_skips_stopped` ([`lifecycle_trace`][lifecycle_trace]). + +**L6. Destructor panics are secondary.** The actor state and queued +messages are dropped after the death point, so a panicking `Drop` does not +change the exit reason; it is recorded in `ChildHandle::cleanup_failure()` +(`mailbox_cleanup_failure()` is a deprecated alias), and a Transient child +restarts on it. A panic payload whose own destructor panics is forgotten +(`dispose_panic`) to stop the cascade. Checked by: [`mailbox_drop_panic`][mailbox_drop_panic] (5 placements x 1/2/256 destructors), [`panic_safety`][panic_safety], [`native_mailbox_drop`][native_mailbox_drop], -`queued_drop_panic`/`queued_drop_preserves_shutdown` lifecycle traces. +`queued_drop_panic`/`queued_drop_shutdown_restarts` lifecycle traces. **L7. Exit publication is last.** The exit reason is published after `stopped()`, after queued messages and reply senders are released, after links @@ -239,12 +258,16 @@ Checked by: [`shutdown_publication`][shutdown_publication], **X1. Down exactly once.** `ctx.monitor(&handle)` (requires `Handler`) yields exactly one `Down { monitor_ref, reason }` per monitor that is not -cancelled, carrying the target's published reason. An already-exited target -produces `Down` immediately. Each call returns a distinct `MonitorRef`; the +cancelled: the target's final reason if registered before its death point D, +`NoProc` (at once) if after it. Each call returns a distinct `MonitorRef`; the target is unaffected. -Checked by: [`actor_scenarios::demonitor_before_exit_suppresses_notification_and_races_do_not_duplicate`][actor_scenarios], +Checked by: `Lifecycle!DownAtMostOnce`, `Lifecycle!NoProcIffLate`, `Lifecycle!DownDelivered` (TLC); +[`link_fence_loom`][link_fence_loom]`::monitor_racing_death_gets_exactly_one_down`; +[`exit_signals_sim::link_and_monitor_race_the_death_point`][exit_signals_sim]; +[`actor_scenarios::demonitor_before_exit_suppresses_notification_and_races_do_not_duplicate`][actor_scenarios], [`exit_signal_matrix`][exit_signal_matrix], `src/*/actor.rs::monitor_*`, `multiple_monitors_each_get_own_ref_and_down`, +[`otp_exit_signals::tasks_mode::registrations_after_death_report_noproc`][otp_exit_signals], [`cancellation_safety::monitor_and_link_setup_requests_are_cancel_safe`][cancellation_safety]. **X2. Demonitor and late registration.** `demonitor` before the target exits @@ -260,45 +283,76 @@ Checked by: `src/*/actor.rs::demonitor_before_target_dies_suppresses_down`, [`otp_exit_signals::tasks_mode::link_from_a_dead_actor_is_inert`][otp_exit_signals], `closed_monitor_table_rejects_late_registration`, [`leaks::monitor_waiters_released_on_demonitor_owner_exit_and_target_exit`][leaks]. -**X3. Link propagation.** When a linked actor exits: +**X3. Exit signals and link propagation.** An exit signal +(`ChildHandle::exit`, `ctx.exit`, `kill`) or a linked actor's death notice is +applied at the receiver by OTP's `exit_signal/2` rules, with the trap flag +read when it arrives: -| Peer exits with | Non-trapping peer | Trapping peer (`ctx.trap_exit(true)`) | +| Signal or notice | Non-trapping receiver | Trapping receiver (`ctx.trap_exit(true)`) | |---|---|---| +| `Kill` signal (`kill()`, `exit(Kill)`) | ends `Killed` | ends `Killed` | | `Normal` | nothing | `exit_received(Exit { from, reason: Normal })` | -| `Shutdown`, `Panic`, `Kill` | stopped cooperatively, exits with the same reason (unless it already committed one); propagates further | `exit_received(Exit { from, reason })` | - -Propagation is transitive through non-trapping actors. Links work across modes. -Checked by: `src/tasks/actor.rs::link_propagates_panic_to_non_trapping_peer`, +| any other reason (a `Kill` notice included) | ends with that reason after its current handler, without `stopped()`, queue discarded; propagates further | `exit_received(Exit { from, reason })` | +| from the receiver's parent (`start_linked` caller, supervisor, system) | as above | a parent exit is queued: when dispatched the actor exits with that reason through `stopped()`; `exit_received` never sees it | + +Propagation is transitive through non-trapping actors. Links work across +modes. Signals arriving during `started()` take effect when it returns, with +the trap flag read then. +Checked by: `Lifecycle!TrapRespected`, `Lifecycle!NormalSignalIgnored`, `Lifecycle!KilledFromKill`, +`Lifecycle!SkipHook`, `Lifecycle!LinkedPeersStop`, `Lifecycle!ShutdownEndsChild` (TLC); +[`otp_exit_signals`][otp_exit_signals]`::tasks_mode::{exit_signals_follow_exit_signal_2, kill_is_untrappable_and_propagates_killed, ctx_exit_sends_from_the_calling_actor, parent_exit_ends_a_trapping_child_normally, signals_during_started_are_resolved_with_the_flag_set_in_started}`, +`threads_mode::kill_exit_and_noproc_in_threads_mode`; +`src/link.rs::tests::signal_rules_follow_exit_signal_2`; +`src/tasks/actor.rs::link_propagates_panic_to_non_trapping_peer`, `link_delivers_exit_to_trapping_peer`, `link_normal_exit_not_propagated_to_non_trapping_peer`, `chain_propagation_through_non_trapping_middle` (and threads equivalents), [`exit_signal_matrix`][exit_signal_matrix], [`linked_exit_macro`][linked_exit_macro], -`src/link_regression_tests.rs::linked_runtime_abort_cancels_an_idle_threads_actor` (cross-mode). - -**X4. Direct stops are untrappable, propagated Kill is trappable.** Every -direct stop (`stop`, `shutdown`, `stop_with_reason(any)`) ignores `trap_exit`. -A `Kill` arriving over a link is trappable, so one kill cannot take down a -graph of trapping observers. This follows Erlang's `kill`/`killed` split, except the -propagated value is still `Kill` (section 11). -Checked by: [`kill_links`][kill_links] (both modes). - -**X5. Link registration.** Linking to an already-exited peer delivers that -peer's actual reason once, under X3, and retains no link (Erlang reports -`noproc` instead). Repeating a live link and linking to oneself are no-ops. -`unlink` stops future propagation but does not retract a signal in flight. -`start_linked` registers the link before the child's `started()`. A link -installed while the peer is between draining its links and publishing its exit -still receives the exit. See section 11 for the planned `NoProc`. -Checked by: `src/tasks/actor.rs::link_to_already_dead_actor_delivers_signal`, -`link_to_already_dead_delivers_exactly_once_to_trapping_peer`, `duplicate_link_is_idempotent`, -`unlink_prevents_signal_delivery`; `src/link_regression_tests.rs::linking_in_post_drain_pre_completion_window_delivers_exit`, -`start_linked_registers_before_async_started`; `src/native_link_regression_tests.rs::start_linked_registers_before_threads_started`. - -**X6. Runtime teardown still notifies.** A tasks actor whose future is dropped -by its runtime, polled or never polled, still signals its links and monitors -with `Kill`, and the notice is trappable. -Checked by: `src/link_regression_tests.rs::never_polled_actor_propagates_runtime_shutdown_to_peer`, +`src/link_regression_tests.rs::linked_runtime_abort_cancels_an_idle_threads_actor` (cross-mode); +lifecycle traces ([`lifecycle_trace`][lifecycle_trace]); [`exit_signals`][ex_exit_signals] example. + +**X4. Kills are untrappable; administrative stops are not signals.** Only a +`Kill` *signal* is untrappable and produces `Killed`. A `Kill` or `Killed` +arriving as a link notice is trappable, so one kill cannot take down a graph +of trapping observers (OTP's `kill`/`killed` split). `stop`, `shutdown` and +`stop_with_reason(any)` are administrative stops (as `gen_server:stop`): +untrappable, `stopped()` runs, and `stop_with_reason(Kill)` ends `Kill`. +Checked by: [`kill_links`][kill_links] (both modes), +[`otp_exit_signals`][otp_exit_signals]`::tasks_mode::{kill_is_untrappable_and_propagates_killed, administrative_kill_reason_runs_stopped_and_ends_kill}`; +`Lifecycle!KilledFromKill` (TLC). + +**X5. Link registration, NoProc and fences.** A link records both ends in +one transaction. Linking to an actor past its death point D keeps no +relationship and delivers one `NoProc` notice: a trapping caller gets +`Exit { reason: NoProc }`, a non-trapping caller ends `NoProc`; +`ctx.try_link` returns `Err(NoProc)` and delivers nothing. A link racing D +gets exactly one notice, the real reason or `NoProc`. Repeating a link +(including one whose notice is pending) and linking to oneself are no-ops. +`unlink` and `demonitor` are fences: once they return, no notice of that edge +takes effect; one already queued stays. Each edge has a generation, so a stale +notice never applies to a later link. `start_linked` registers the link +before the child's `started()`. +Checked by: `Lifecycle!NoProcIffLate`, `Lifecycle!NoEdgeToDead`, `Lifecycle!CancelFenced` (TLC, `MCLifecycleFence`); +[`link_fence_loom`][link_fence_loom] (the production link table and monitor +fence under Loom: `unlink_fences_a_death_notice`, `link_racing_death_gets_exactly_one_notice`, +`relink_after_unlink_ignores_the_stale_generation`, `death_racing_unlink_and_a_new_link`, +`mutual_death_delivers_at_most_one_notice`, `concurrent_links_install_one_edge`, +`demonitor_fences_the_down`, `observer_close_fences_the_down`); +[`exit_signals_sim`][exit_signals_sim]`::{link_and_monitor_race_the_death_point, unlink_and_demonitor_fence_delivery}`; +`src/link.rs::tests::{closure_is_terminal_and_delivers_each_edge_once, try_link_to_a_dead_actor_keeps_no_record, unlink_fences_a_notice_taken_at_death}`; +[`otp_exit_signals::tasks_mode::registrations_after_death_report_noproc`][otp_exit_signals]; +`src/tasks/actor.rs::duplicate_link_is_idempotent`, `unlink_prevents_signal_delivery`; +`src/native_link_regression_tests.rs::start_linked_registers_before_threads_started`; +lifecycle traces `late_registrations_get_noproc`, `unlink_fences_a_notice_taken_at_death`, +`demonitor_fences_a_down`. + +**X6. Runtime teardown still notifies.** An actor whose future or thread is +dropped by its runtime, polled or never polled, ends `Killed` in both modes +(unless a non-`Normal` reason was requested), without `stopped()`, and still +signals its links (trappable) and monitors. +Checked by: `Lifecycle!RuntimeDropReason` (TLC); +`src/link_regression_tests.rs::never_polled_actor_propagates_runtime_shutdown_to_peer`, `polled_actor_propagates_runtime_shutdown_to_peer`, `never_polled_actor_notifies_monitor_on_runtime_shutdown`, -`linked_runtime_abort_is_trappable`. +`linked_runtime_abort_is_trappable`; lifecycle trace `runtime_drop_ends_killed`. **X7. Ordering.** Trapped `Exit` and `Down` are ordinary mailbox messages: they queue behind messages already in the mailbox and never interrupt a running @@ -314,7 +368,7 @@ Checked by: [`semantics_gaps`][semantics_gaps]`::{native,tasks}::exit_and_down_f **V1. Restart table.** While the supervisor is open: -| Mode | `Normal` / `Shutdown` exit | `Panic` / `Kill` exit | +| Mode | `Normal` / `Shutdown` exit | `Panic` / `Kill` / `Killed` / `NoProc` exit, or a cleanup failure | |---|---|---| | Permanent (default) | restart | restart | | Transient | stay stopped | restart | @@ -344,11 +398,19 @@ Checked by: `Cohorts!StrategySelection`, `Cohorts!WholeCohortBarrier`, **V3. At most one live generation.** A slot never has two live generations: a replacement starts only after the previous actor, its registered helpers and -its owned cleanup actually finished. If cleanup misses its `shutdown_timeout`, -the slot is stuck, the supervisor fails and nothing replaces it. +its owned cleanup actually finished. When the supervisor stops a child +(shutdown, a OneForAll/RestForOne round, a startup rollback), a child that has +not reached its death point `shutdown_timeout` after its signal is killed +(`Killed`, design E3); if it misses a second `shutdown_timeout` the slot is +stuck. A child that exits on its own is stuck if its cleanup misses +`shutdown_timeout` from its exit (no kill: it would relabel its reason). A +stuck slot fails the supervisor and nothing replaces it. Checked by: `Supervision!AtMostOneGeneration`, `Cohorts!AtMostOneGeneration`, `NestedSupervision!AtMostOneGeneration`; [`supervision::cleanup_deadline_escalates_without_overlapping_a_live_generation`][supervision], -`repeated_crash_producer_and_shutdown_races_never_overlap_generations`. +`repeated_crash_producer_and_shutdown_races_never_overlap_generations`, +`stuck_stopped_hook_triggers_cleanup_deadline_before_terminal_publication`; +[`otp_exit_signals::tasks_mode::supervisor_shutdown_is_the_parents_exit_signal`][otp_exit_signals] +(the kill at `shutdown_timeout`). **V4. Generations and publication.** Each construction attempt increments the slot's generation. `SupervisedChild::current()` returns only the latest live, @@ -400,6 +462,7 @@ Factories run on the child's group. Cancelling the start future before readiness rolls the tree back. Checked by: `Supervision!ReadyMeansAllStarted`, `Supervision!RestartFollowsPolicy` (no restart before the tree is ready), `Cohorts!FirstStartOnce`; [`supervision::failed_partial_startup_rolls_back_without_starting_later_children`][supervision], `otp_exit_signals::first_start_failure_aborts_start_whatever_the_restart_type`, `ready_child_exit_during_startup_is_processed_afterwards`, `start_returns_its_error_after_rollback`, +[`exit_signals_sim::ready_child_exit_races_the_tree_commit`][exit_signals_sim], `transient_startup_timeout_retries_after_actual_cleanup`, `normal_pre_ready_exit_records_real_cause_and_fails_transient_startup`, `transient_replacement_exiting_before_readiness_is_retried`, [`supervision_trace::trace_replacement_exiting_before_readiness_is_retried`][supervision_trace], `failed_factories_cancel_owned_helpers_before_retry_and_catch_panics`, @@ -416,9 +479,15 @@ Checked by: `NestedSupervision!NoEarlyParentJoin`, `NestedSupervision!DoneMeansD [`nested_supervision`][nested_supervision], [`tree_trace`][tree_trace]. **V9. Supervisor shutdown.** Closing a supervisor never reopens it and starts -or publishes nothing afterwards. Ready children get their stop turn in reverse -declaration order, each bounded by its `shutdown_timeout` (default 5 s); at the -report deadline the remaining siblings are signalled. "Done" means no live +or publishes nothing afterwards. Ready children get their turn in reverse +declaration order. Each receives its parent's `Shutdown` exit signal (E3): a +non-trapping child ends `Shutdown` without `stopped()`; a trapping child +handles the messages queued before it (`exit_pending()` is `true` there), then +runs `stopped()` and ends `Shutdown`, never seeing it in `exit_received`. Each +turn is bounded by the child's `shutdown_timeout` (default 5 s), then the +child is killed (V3); at the report deadline the remaining siblings are +signalled. `ActorSystem::shutdown(grace)` signals standalone actors the same +way and kills those left at `grace`. "Done" means no live generation and every controller finished. Supervisors observe children through internal completion state, not their mailboxes, so a message backlog does not delay observation. @@ -426,7 +495,10 @@ Checked by: `Supervision!NoStartAfterClose`, `Supervision!ReverseShutdown`, `Supervision!NeverReopens`, `Supervision!DoneMeansQuiescent`, `Cohorts!NoStartAfterClose`, `NestedSupervision!NoStartAfterClose`, `Supervision!ClosedLeadsToDone` (TLC-only, assumes cleanup ends); [`supervision::shutdown_during_backoff_prevents_new_factory_invocation`][supervision], -`supervisor_report_deadline_signals_remaining_siblings`, [`failpoints`][failpoints]. +`supervisor_report_deadline_signals_remaining_siblings`, [`failpoints`][failpoints]; +`Lifecycle!ShutdownEndsChild` (TLC); +[`otp_exit_signals`][otp_exit_signals]`::tasks_mode::{supervisor_shutdown_is_the_parents_exit_signal, exit_pending::queued_work_sees_the_supervisors_pending_shutdown}`; +[`exit_signals_sim::supervisor_shutdown_races_a_trapping_childs_queue`][exit_signals_sim]. ## 7. Systems and execution groups @@ -564,9 +636,9 @@ stopped after `Shutdown`), - **Erlang/OTP equivalence.** See section 11 for each current difference. - **Liveness under schedules nobody explored.** Tests run real schedulers; models abstract time and handler bodies (see [TESTING](../TESTING.md#how-spawned-is-verified)). - The base actor lifecycle (links, monitors, requests around shutdown) has no - TLA+ model of the v0.7 behaviour; `Lifecycle.tla` models the proposed v0.8 - contract ([next model](../../verification/tla/README.md#next-model)). + The base actor lifecycle (signals, links, monitors, requests around the + death point) is modelled by `Lifecycle.tla` (TLC only), and recorded + lifecycle traces are checked against it. ## 11. Differences from Erlang/OTP @@ -575,18 +647,25 @@ where Rust types add strictness, or where Spawned adds bounds Erlang lacks. Already the same as OTP: per-sender FIFO (D1), at-most-once delivery (D2), `gen_server:call`'s 5 s default timeout (R3), a stop dropping the mailbox like a -process exit (D4), linked exits propagating the same reason and `Normal` being -ignored by non-trapping peers (X3), and the Permanent/Transient/Temporary table (V1). +process exit (D4), exit signals and link notices following `exit_signal/2` +(X3), the `kill`/`killed` split (X4), `noproc` for late links, monitors and +calls and the unlink/demonitor fences (X1, X5, R1), a parent's exit ending a +trapping child through `stopped()` (X3, V9), `stopped()` skipped on kills and +untrapped signals (L5), and the Permanent/Transient/Temporary table (V1). The +remaining differences of the exit-signal design are listed in +[otp-exit-signals.md](otp-exit-signals.md#differences-from-otp-that-remain). | Today | OTP | Status | |---|---|---| -| Every direct stop (`stop`, `shutdown`, `stop_with_reason(any)`) ignores `trap_exit` (X4) | `exit(Pid, normal)` is ignored by a non-trapping process; every reason except `kill` is trappable | Planned v0.8, rank 5e: a `Normal` exit signal is ignored by a non-trapping actor, all reasons except `Kill` (including `Shutdown`) are trappable, only `Kill` is untrappable | -| Supervisor shutdown cancels a trapping child directly | `shutdown` reaches a trapping child as an `'EXIT'` message and it runs normal termination | Planned v0.8, rank 5e | -| A kill propagated over a link stays `Kill` (X4) | It arrives as `killed` | Planned v0.8, rank 5e: a distinct trappable `Killed` | -| Linking or monitoring an already-exited actor delivers its real exit reason (X1, X5) | `noproc` | Planned v0.8, rank 5e: `NoProc` | +| v0.7 had no exit signals: every outside stop ignored `trap_exit` | `exit(Pid, normal)` is ignored by a non-trapping process; every reason except `kill` is trappable | Aligned in v0.8 (rank 5e, #240): `exit`/`kill` follow `exit_signal/2` (X3) | +| `stop`, `shutdown`, `stop_with_reason` return at once and stop after the current handler (X4) | `gen_server:stop` waits for the backlog | Bound: stop latency is one handler; they stay administrative stops, distinct from exit signals | +| v0.7 supervisor shutdown cancelled a trapping child directly | `shutdown` reaches a trapping child as an `'EXIT'` message and it runs normal termination | Aligned in v0.8 (rank 5e, #240): a parent exit (V9) | +| v0.7 propagated a kill over a link as `Kill` | It arrives as `killed` | Aligned in v0.8 (rank 5e, #240): `Killed` (X4) | +| v0.7 linking or monitoring an already-exited actor delivered its real exit reason | `noproc` | Aligned in v0.8 (rank 5e, #240): `NoProc` (X1, X5) | +| A kill or late signal does not interrupt a running `stopped()` (L3) | The process dies at once | Rust-forced: the reason still becomes `Killed` or the signal's reason | | The restart budget counts every child factory call, collateral ones included (V5) | Intensity counts recovery rounds | Planned v0.8, rank 5f | | Initial startup retried a Permanent or Transient child whose factory failed, panicked or timed out (V7), within the budget | `start_link` aborts on any child start error | Aligned in v0.8 (rank 5f, #240): any first-start failure fails `start()` | -| An actor dropped by its runtime exits `Kill` in tasks; the threads fallback is `Panic("unexpected framework panic")` (L4) | No counterpart | Planned v0.8, rank 5g: one reason in both modes | +| v0.7 reported a runtime-dropped actor as `Kill` (tasks) or `Panic("unexpected framework panic")` (threads) | No counterpart | v0.8 (rank 5g, #240): `Killed` in both modes (X6) | | Stop is cooperative: a running handler or blocking call is never interrupted (L3) | `kill` ends a process at once | Rust-forced: Rust cannot preempt or kill a thread | | Panics are caught only with unwinding (P2) | Process crashes are always isolated | Rust-forced: `panic = "abort"` and a panicking panic hook end the process | | No per-actor memory accounting | `max_heap_size` | Rust-forced (no per-actor heap); opt-in size hook proposed, rank 18a | @@ -596,16 +675,11 @@ ignored by non-trapping peers (X3), and the Permanent/Transient/Temporary table ## 12. Planned changes to these semantics -- v0.8, rank 5e, exit signals as in OTP: trappable - `Shutdown`, `Normal` ignored by non-trapping actors, `Killed` for propagated - kills, `NoProc` for links and monitors to exited actors, normal termination - for a trapping child on supervisor shutdown (changes X1, X3, X4, X5, L3, L4). +- v0.8, ranks 5e, 5f (startup) and 5g, exit signals, startup and runtime + drop as in OTP ([otp-exit-signals.md](otp-exit-signals.md)): implemented on + the exit-signals branch (#240, on top of rank 11's #239) and described in + L3-L6, X1-X6, V1, V3, V7 and V9 above. - v0.8, rank 5f: the restart budget counts recovery rounds (changes V5). -- v0.8, rank 5g: one exit reason for runtime-dropped actors in both modes (changes L4, X6). -- Design for 5e, the 5f startup rule and 5g, accepted (not implemented): - [otp-exit-signals.md](otp-exit-signals.md). Reason-bearing `ActorStopped` - and `NoProc` for sends and requests are implemented on the exit-signals - branch (#240, on top of rank 11's #239). - Rank 11, deferred typed replies, forwarding and pipe-to-self: a reply may leave the handler's turn; adds exactly-one completion on panic and timeout. @@ -626,6 +700,9 @@ See the [ranked backlog](../../ROADMAP.md#ranked-backlog). [context_exit]: ../../concurrency/tests/context_exit.rs [cross_backend]: ../../concurrency/tests/cross_backend.rs [otp_exit_signals]: ../../concurrency/tests/otp_exit_signals.rs +[exit_signals_sim]: ../../concurrency/tests/exit_signals_sim.rs +[link_fence_loom]: ../../concurrency/tests/link_fence_loom.rs +[lifecycle_trace]: ../../concurrency/tests/lifecycle_trace.rs [delivery_simulation]: ../../concurrency/tests/delivery_simulation.rs [execution_groups]: ../../concurrency/tests/execution_groups.rs [exit_signal_matrix]: ../../concurrency/tests/exit_signal_matrix.rs @@ -653,6 +730,7 @@ See the [ranked backlog](../../ROADMAP.md#ranked-backlog). [supervisor_policies]: ../../concurrency/tests/supervisor_policies.rs [tree_trace]: ../../concurrency/tests/tree_trace.rs [ex_exit_reason]: ../../examples/exit_reason +[ex_exit_signals]: ../../examples/capabilities/src/bin/exit_signals.rs [ex_native_lifecycle]: ../../examples/capabilities/src/bin/native_lifecycle.rs [ex_panic_abort]: ../../concurrency/examples/panic_abort.rs [ex_protocol_timeouts]: ../../examples/capabilities/src/bin/protocol_timeouts.rs diff --git a/examples/README.md b/examples/README.md index 7734f841..1c0e437d 100644 --- a/examples/README.md +++ b/examples/README.md @@ -25,6 +25,7 @@ Usage of each feature is documented in the [guide](../docs/GUIDE.md). | Dedicated-thread actor with async helpers | [dedicated_thread](capabilities/src/bin/dedicated_thread.rs) | n/a | | Global registry (`registry::register`/`whereis`) | [service_discovery](service_discovery), [registrations and fakes](capabilities/src/bin/registrations_and_fakes.rs) | [registrations and fakes](capabilities/src/bin/registrations_and_fakes.rs), [thread_only](thread_only/README.md) | | Exit reasons, `join`, `wait_exit`, monitors, links, trapped exits | [exit_reason](exit_reason) | [native_lifecycle](capabilities/src/bin/native_lifecycle.rs) | +| Exit signals as in OTP (`exit`/`kill`, trapping, `NoProc`, `try_link`, parent exits, `exit_pending`), v0.8 | [exit_signals](capabilities/src/bin/exit_signals.rs) | gap (threads mode is tested in `otp_exit_signals::threads_mode`) | | Execution groups (`ActorSystem`, `start_on`, owned helpers), v0.7 | [execution_groups](capabilities/src/bin/execution_groups.rs) | n/a | | Supervision, v0.7 | [supervision_recovery](capabilities/src/bin/supervision_recovery.rs) | n/a | | Fixed native actor pools / parked Linux pinned shards, v0.7 | [execution_profiles](capabilities/src/bin/execution_profiles.rs) | n/a | diff --git a/examples/capabilities/README.md b/examples/capabilities/README.md index e7fce441..27196b32 100644 --- a/examples/capabilities/README.md +++ b/examples/capabilities/README.md @@ -25,6 +25,7 @@ tracking is opt-in; it adds per-turn clock/locking costs, not payload logging. | [streams_and_recipients](src/bin/streams_and_recipients.rs) | Share one message interface across actor types | Broadcast through typed recipients, reject stopped members, cancel an async source and release a blocking native iterator cooperatively. | | [registrations_and_fakes](src/bin/registrations_and_fakes.rs) | Test protocol clients and resolve named replacements | A fake implements the same protocol; managed names follow fresh generations; manual guards preserve newer registrations. | | [test_kit](src/bin/test_kit.rs) | Unit-test an actor without its real dependencies | A `spawned-test` mock answers a protocol and a probe records sends; a slow reply forces a timeout, a crashed dependency and a killed supervised child are injected, and the same mock runs in threads mode. | +| [exit_signals](src/bin/exit_signals.rs) | Stop, kill or observe actors with OTP's exit-signal rules | `exit(Normal)` ignored and `exit(Shutdown)` ending an actor without `stopped()`; trapped `Exit`; `kill()` ending a trapping actor `Killed`; late link/monitor giving `NoProc`, `try_link` returning `Err(NoProc)`; a supervisor's shutdown reaching a trapping child after its queued work, which sees `exit_pending()`. | | [native_lifecycle](src/bin/native_lifecycle.rs) | Handle failures from synchronous actors | Monitors, demonitoring, trapped exits and linked startup, with no ambient Tokio runtime. | | [coordinated_shutdown](src/bin/coordinated_shutdown.rs) | Migrate hand-wired client shutdown | Join producers, snapshot a live dependency, then join a two-group tree in reverse dependency order. | | [framework_metrics](src/bin/framework_metrics.rs) | Observe actor activity without modifying handlers | Configure sampled timing, issue requests and render process-wide Prometheus text. Metrics are explicitly enabled for this binary. | @@ -44,6 +45,7 @@ cargo run --locked -p spawned-capability-examples --bin streams_and_recipients cargo run --locked -p spawned-capability-examples --bin registrations_and_fakes cargo run --locked -p spawned-capability-examples --bin test_kit cargo run --locked -p spawned-capability-examples --bin native_lifecycle +cargo run --locked -p spawned-capability-examples --bin exit_signals cargo run --locked -p spawned-capability-examples --bin coordinated_shutdown cargo run --locked -p spawned-capability-examples --features metrics --bin framework_metrics ``` diff --git a/examples/capabilities/src/bin/exit_signals.rs b/examples/capabilities/src/bin/exit_signals.rs new file mode 100644 index 00000000..807673ab --- /dev/null +++ b/examples/capabilities/src/bin/exit_signals.rs @@ -0,0 +1,246 @@ +//! Exit signals as in OTP (docs/GUIDE.md, "Exit signals"): `exit` vs `kill`, +//! trapping, `NoProc` for registrations and sends after death, and +//! `exit_pending` for work queued before a supervisor's shutdown. +use spawned_concurrency::{ + message::Message, + reply::PendingReply, + supervision::{ChildSpec, SupervisorConfig}, + system::ActorSystem, + tasks::{Actor, ActorStart, Context, Handler}, + ActorError, ChildHandle, Down, Exit, ExitReason, NoProc, +}; +use std::{ + sync::{Arc, Mutex}, + time::Duration, +}; + +/// What an actor observed. +#[derive(Clone, Default)] +struct Seen(Arc>>); +impl Seen { + fn push(&self, entry: String) { + self.0.lock().unwrap().push(entry); + } + fn take(&self) -> Vec { + std::mem::take(&mut self.0.lock().unwrap()) + } +} + +struct Worker { + trap: bool, + seen: Seen, +} +impl Worker { + fn new(trap: bool, seen: &Seen) -> Self { + Self { + trap, + seen: seen.clone(), + } + } +} +impl Actor for Worker { + async fn started(&mut self, ctx: &Context) { + // Trap first thing: signals that arrive during started() are + // resolved with the flag as it is when started() returns. + ctx.trap_exit(self.trap); + } + async fn stopped(&mut self, ctx: &Context) { + self.seen + .push(format!("stopped (exit_pending: {})", ctx.exit_pending())); + } + async fn exit_received(&mut self, exit: Exit, _: &Context) { + self.seen.push(format!("Exit {:?}", exit.reason)); + } +} +impl Handler for Worker { + async fn handle(&mut self, down: Down, reply: PendingReply<'_, ()>, _: &Context) { + self.seen.push(format!("Down {:?}", down.reason)); + reply.reply(()); + } +} + +/// Link to and monitor a peer. +struct Watch(ChildHandle); +impl Message for Watch { + type Result = (); +} +impl Handler for Worker { + async fn handle(&mut self, msg: Watch, reply: PendingReply<'_, ()>, ctx: &Context) { + ctx.link(&msg.0); + ctx.monitor(&msg.0); + reply.reply(()); + } +} + +/// `try_link`: `Err(NoProc)` instead of a `NoProc` exit signal. +struct TryLink(ChildHandle); +impl Message for TryLink { + type Result = Result<(), NoProc>; +} +impl Handler for Worker { + async fn handle( + &mut self, + msg: TryLink, + reply: PendingReply<'_, Result<(), NoProc>>, + ctx: &Context, + ) { + reply.reply(ctx.try_link(&msg.0)); + } +} + +/// Work that refuses to start once the actor's exit is pending. +struct Work(u32); +impl Message for Work { + type Result = (); +} +impl Handler for Worker { + async fn handle(&mut self, msg: Work, reply: PendingReply<'_, ()>, ctx: &Context) { + let pending = ctx.exit_pending(); + self.seen + .push(format!("work {} (exit_pending: {pending})", msg.0)); + reply.reply(()); + } +} + +/// Busy until the actor's exit is pending (bounded), standing in for a long +/// handler that is running when the supervisor shuts down. +struct Busy; +impl Message for Busy { + type Result = (); +} +impl Handler for Worker { + async fn handle(&mut self, _: Busy, reply: PendingReply<'_, ()>, ctx: &Context) { + for _ in 0..1000 { + if ctx.exit_pending() { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + reply.reply(()); + } +} + +/// A round trip: everything queued before it was handled. +struct Sync; +impl Message for Sync { + type Result = (); +} +impl Handler for Worker { + async fn handle(&mut self, _: Sync, reply: PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } +} + +const LIMIT: Duration = Duration::from_secs(5); + +async fn ended(handle: &ChildHandle) -> ExitReason { + tokio::time::timeout(LIMIT, handle.wait_exit_async()) + .await + .expect("actor did not exit") +} + +#[tokio::main(flavor = "multi_thread", worker_threads = 2)] +async fn main() { + // 1. exit(reason) follows OTP's exit_signal/2: Normal is ignored by a + // non-trapping actor, any other reason ends it without stopped(). + let seen = Seen::default(); + let plain = Worker::new(false, &seen).start(); + plain.request(Sync).await.unwrap(); + plain.child_handle().exit(ExitReason::Normal); + plain.request(Sync).await.unwrap(); + assert!(plain.child_handle().is_alive(), "exit(Normal) is ignored"); + plain.child_handle().exit(ExitReason::Shutdown); + assert_eq!(ended(&plain.child_handle()).await, ExitReason::Shutdown); + assert!( + seen.take().is_empty(), + "an untrapped signal skips stopped()" + ); + println!("exit(Normal) ignored; exit(Shutdown) ended the actor without stopped()"); + + // 2. A trapping actor receives exit signals as `Exit` messages; only a + // kill is untrappable and ends it `Killed`, also without stopped(). + // An administrative stop (`stop`, `shutdown`) always runs stopped(). + let (observer_seen, target_seen) = (Seen::default(), Seen::default()); + let observer = Worker::new(true, &observer_seen).start(); + let target = Worker::new(true, &target_seen).start(); + observer + .request(Watch(target.child_handle())) + .await + .unwrap(); + target.child_handle().exit(ExitReason::Shutdown); + target.request(Sync).await.unwrap(); + assert_eq!(target_seen.take(), vec!["Exit Shutdown"]); + target.child_handle().kill(); + assert_eq!(ended(&target.child_handle()).await, ExitReason::Killed); + assert!(target_seen.take().is_empty(), "a kill skips stopped()"); + // The trapping observer gets the link notice and the monitor's Down. + tokio::time::sleep(Duration::from_millis(50)).await; + observer.request(Sync).await.unwrap(); + let mut notices = observer_seen.take(); + notices.sort(); + assert_eq!(notices, vec!["Down Killed", "Exit Killed"]); + println!("trapped Exit(Shutdown); kill() ended a trapping actor Killed; peers saw Killed"); + + // 3. Past its death point an actor is gone: link and monitor get NoProc + // (one notice each), try_link returns Err(NoProc), sends fail NoProc. + observer + .request(Watch(target.child_handle())) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + observer.request(Sync).await.unwrap(); + let mut notices = observer_seen.take(); + notices.sort(); + assert_eq!(notices, vec!["Down NoProc", "Exit NoProc"]); + let tried = observer.request(TryLink(target.child_handle())).await; + assert_eq!(tried.unwrap(), Err(NoProc)); + assert!(matches!(target.send(Sync), Err(ActorError::NoProc))); + // The real reason is still available from the handle. + assert_eq!( + target.child_handle().exit_reason(), + Some(ExitReason::Killed) + ); + // A non-trapping actor that links to a dead one exits NoProc itself. + let linker = Worker::new(false, &Seen::default()).start(); + linker.request(Watch(target.child_handle())).await.unwrap(); + assert_eq!(ended(&linker.child_handle()).await, ExitReason::NoProc); + observer.child_handle().stop(); + ended(&observer.child_handle()).await; + assert_eq!(observer_seen.take(), vec!["stopped (exit_pending: true)"]); + println!("late link/monitor gave NoProc, try_link Err(NoProc), send NoProc"); + + // 4. A supervisor's shutdown is its parent's exit signal. A trapping + // child handles the work queued before it, sees `exit_pending()`, then + // runs stopped(); a non-trapping child would end without stopped(). + let system = ActorSystem::new().unwrap(); + let group = system.async_group("exit-signals", 1).unwrap(); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let child_seen = Seen::default(); + let s = child_seen.clone(); + let child = tree + .add(ChildSpec::new("trapping", &group, move |_| { + let seen = s.clone(); + async move { Ok::<_, String>(Worker::new(true, &seen)) } + })) + .unwrap(); + let supervisor = tree.start().await.unwrap(); + let worker = child.current().unwrap(); + worker.request(Work(1)).await.unwrap(); + // Work queued behind a busy handler, then the shutdown: the parent exit + // queues behind that work. + worker.send(Busy).unwrap(); + worker.send(Work(2)).unwrap(); + let handle = worker.child_handle(); + assert!(supervisor.shutdown(LIMIT).await.complete); + assert_eq!(ended(&handle).await, ExitReason::Shutdown); + assert_eq!( + child_seen.take(), + vec![ + "work 1 (exit_pending: false)", + "work 2 (exit_pending: true)", + "stopped (exit_pending: true)", + ] + ); + assert!(system.shutdown(LIMIT).await.complete); + println!("supervisor shutdown: queued work saw exit_pending, then stopped() ran"); +} From c3f5a5cf570879737564ae25dbdb29282fbedce4 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 03:50:00 -0300 Subject: [PATCH 094/208] test(exit): per-thread outcome counts in exit_signals_sim The counts were shared by the six tests, which run in parallel. --- concurrency/tests/exit_signals_sim.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/concurrency/tests/exit_signals_sim.rs b/concurrency/tests/exit_signals_sim.rs index 587a50c6..811e2b6a 100644 --- a/concurrency/tests/exit_signals_sim.rs +++ b/concurrency/tests/exit_signals_sim.rs @@ -24,6 +24,7 @@ use spawned_rt::tasks::{ sleep, spawn, }; use std::{ + cell::RefCell, collections::BTreeMap, future::Future, sync::{Arc, Mutex}, @@ -44,12 +45,15 @@ fn position(log: &[String], entry: &str) -> Option { log.iter().position(|e| e == entry) } -/// Outcome counts per body, printed by the driver: every branch the -/// assertions distinguish must be reached. -static OUTCOMES: Mutex> = Mutex::new(BTreeMap::new()); +// Outcome counts per body, printed by the driver: every branch the +// assertions distinguish must be reached. Per test thread (the simulator +// runs a body on the thread that explores it). +thread_local! { + static OUTCOMES: RefCell> = const { RefCell::new(BTreeMap::new()) }; +} fn outcome(name: impl Into) { - *OUTCOMES.lock().unwrap().entry(name.into()).or_default() += 1; + OUTCOMES.with(|o| *o.borrow_mut().entry(name.into()).or_default() += 1); } fn ms(n: u64) -> Duration { @@ -541,7 +545,7 @@ where F: Fn(u64) -> Fut, Fut: Future, { - OUTCOMES.lock().unwrap().clear(); + OUTCOMES.with(|o| o.borrow_mut().clear()); let report = explore(&Strategy::parse(PCT.0).unwrap(), 0, PCT.1, &body); assert!( report.found.is_none(), @@ -569,7 +573,9 @@ where "{name} {}: {runs} runs over {DFS_SEEDS} seeds, {exhausted} searches exhausted", DFS.0 ); - println!("{name} outcomes: {:?}", OUTCOMES.lock().unwrap()); + let outcomes = OUTCOMES.with(|o| o.borrow().clone()); + assert!(!outcomes.is_empty(), "{name}: no outcome recorded"); + println!("{name} outcomes: {outcomes:?}"); } #[test] From 0fe28e0bdb9ec9c981d5077bb45e438ee9cd07e7 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 04:03:17 -0300 Subject: [PATCH 095/208] test(exit): compile otp_exit_signals out under spawned_sim It uses real Tokio runtimes and OS threads; exit_signals_sim covers its orderings under the simulator. --- concurrency/tests/otp_exit_signals.rs | 4 ++++ docs/TESTING.md | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/concurrency/tests/otp_exit_signals.rs b/concurrency/tests/otp_exit_signals.rs index da6dbcac..83630f10 100644 --- a/concurrency/tests/otp_exit_signals.rs +++ b/concurrency/tests/otp_exit_signals.rs @@ -3,6 +3,10 @@ //! design rule it pins; every one fails on v0.7 (the API or the outcome //! differs there). #![cfg(feature = "async")] +// Real Tokio runtimes and OS threads (plain `#[tokio::test]`, threads mode): +// not run under the simulator. `exit_signals_sim.rs` explores the same +// orderings under `spawned_sim`. +#![cfg(not(spawned_sim))] use spawned_concurrency::{message::Message, ActorId, Down, Exit, ExitReason, NoProc}; use std::{ sync::{ diff --git a/docs/TESTING.md b/docs/TESTING.md index 40b71d84..686c98ce 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -385,7 +385,8 @@ simulated at all): tests that need OS threads or a real Tokio runtime, in `cancellation_safety`, `cross_backend` (non-async placements and the fan-out and monitor workloads), `dedicated_runtime`, `delivery_simulation`, `execution_groups`, `execution_profiles`, `failure_windows`, `leaks`, -`mailbox_drop_panic`, `nested_supervision`, `panic_safety`, +`mailbox_drop_panic`, `nested_supervision`, `otp_exit_signals` (whole file; +`exit_signals_sim` explores its orderings), `panic_safety`, `process_failures`, `protocol_compile`, `request_histories`, `resource_cleanup`, `semantics_gaps` (its async module), `shutdown_publication`, `startup_admission`, `supervision_strategies` and `supervision`. `leaks` replaces its excluded From 600f98466e20b4e61f132fa9202ed8841dde4ccb Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 08:03:56 +0000 Subject: [PATCH 096/208] test(jobs): hold the replacing turn with an await, not a blocked worker replacement_fences_a_finished_keyed_job (deferred_jobs) and replaced_finished_keyed_job_is_fenced (job_trace) hung under load (about 1 run in 40 with nice -n 19): ReplaceWhen spun with std::thread::sleep on a Tokio worker until the test saw the old keyed job finish. When the test thread queued ReplaceWhen before the actor task returned from the poll that handled StartOld, the actor started spinning in that same poll, and the old job, spawned during StartOld into that worker LIFO slot (which other workers cannot steal), never ran. gdb on a hung run: one worker in ReplaceWhen, the other parked, the test polling is_running() forever. ReplaceWhen now awaits a oneshot, which keeps the turn busy (the terminal message still queues behind it and is fenced) without blocking the worker. 700/700 runs of each test pass, 300 of them pinned to one CPU. --- concurrency/tests/deferred_jobs.rs | 17 +++++++++-------- concurrency/tests/job_trace.rs | 16 ++++++++-------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/concurrency/tests/deferred_jobs.rs b/concurrency/tests/deferred_jobs.rs index 9495ffee..0a040c85 100644 --- a/concurrency/tests/deferred_jobs.rs +++ b/concurrency/tests/deferred_jobs.rs @@ -283,16 +283,17 @@ mod tasks_mode { reply.reply(()); } } - /// Busy until `go`, then replaces the key. - struct ReplaceWhen(Arc); + /// Holds its turn until `go` fires, then replaces the key. The turn + /// awaits instead of blocking the worker: a blocked worker strands the + /// old job, spawned into its non-stealable LIFO slot in the same poll, + /// so the old job never runs and the test hangs. + struct ReplaceWhen(oneshot::Receiver<()>); impl Message for ReplaceWhen { type Result = (); } impl Handler for Owner { async fn handle(&mut self, msg: ReplaceWhen, _: PendingReply<'_, ()>, ctx: &Context) { - while !msg.0.load(std::sync::atomic::Ordering::SeqCst) { - std::thread::sleep(Duration::from_millis(1)); - } + msg.0.await.unwrap(); ctx.pipe_keyed("r", async { 2 }, |o| describe("new", o)) .unwrap(); } @@ -307,15 +308,15 @@ mod tasks_mode { let (actor, log) = start(); let handle = Arc::new(Mutex::new(None)); actor.request(StartOld(handle.clone())).await.unwrap(); - let go = Arc::new(std::sync::atomic::AtomicBool::new(false)); - actor.send(ReplaceWhen(go.clone())).unwrap(); + let (go, held) = oneshot::channel::<()>(); + actor.send(ReplaceWhen(held)).unwrap(); // `ReplaceWhen` is queued before the old job can finish. let (old, open) = handle.lock().unwrap().take().unwrap(); open.send(()).unwrap(); while old.is_running() { sleep(Duration::from_millis(1)).await; } - go.store(true, std::sync::atomic::Ordering::SeqCst); + go.send(()).unwrap(); assert_eq!(entries(&actor, &log, 1).await, ["new:ok:2"]); sleep(Duration::from_millis(20)).await; assert!( diff --git a/concurrency/tests/job_trace.rs b/concurrency/tests/job_trace.rs index e049c29c..37cfd2e5 100644 --- a/concurrency/tests/job_trace.rs +++ b/concurrency/tests/job_trace.rs @@ -178,17 +178,17 @@ impl Handler for Owner { } } -/// Hold the turn until `go`, then replace key `k`: the old job finished +/// Hold the turn until `go` fires, then replace key `k`: the old job finished /// meanwhile, so its terminal message is queued behind this turn and fenced. -struct ReplaceWhen(Arc); +/// The turn awaits instead of blocking the worker: a blocked worker strands +/// the old job, spawned into its non-stealable LIFO slot in the same poll. +struct ReplaceWhen(oneshot::Receiver<()>); impl Message for ReplaceWhen { type Result = (); } impl Handler for Owner { async fn handle(&mut self, msg: ReplaceWhen, _: PendingReply<'_, ()>, ctx: &Context) { - while !msg.0.load(std::sync::atomic::Ordering::SeqCst) { - std::thread::sleep(Duration::from_millis(1)); - } + msg.0.await.unwrap(); ctx.pipe_keyed("k", async { 2u32 }, done("new")).unwrap(); } } @@ -277,14 +277,14 @@ async fn replaced_finished_keyed_job_is_fenced() { let old = Arc::default(); let (actor, log) = start_with(Arc::clone(&old)); actor.request(Run(Scenario::KeyedFenced)).await.unwrap(); - let go = Arc::new(std::sync::atomic::AtomicBool::new(false)); - actor.send(ReplaceWhen(go.clone())).unwrap(); + let (go, held) = oneshot::channel::<()>(); + actor.send(ReplaceWhen(held)).unwrap(); let (handle, open) = old.lock().unwrap().take().unwrap(); open.send(()).unwrap(); while handle.is_running() { sleep(Duration::from_millis(1)).await; } - go.store(true, std::sync::atomic::Ordering::SeqCst); + go.send(()).unwrap(); wait_for(&log, "new:ok:2").await; finish("keyed_fenced", actor, 16).await; assert!(!log.snapshot().iter().any(|e| e.starts_with("old"))); From e0f1785c703f27caabeb3f4a2929b090af265a74 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 08:13:24 +0000 Subject: [PATCH 097/208] test(sim): adapt the simulator suites to E3 and S1 Found by running the simulated suites (--cfg spawned_sim), which no CI job runs yet; all pass on feat/deferred-replies and fail here. - sim.rs shutdown_order_race_needs_preemption: every seed logged no stopped(). System and supervisor shutdown now send the parent exit signal, and a non-trapping actor ends without stopped() (E3, decision 2; MIGRATION "stopped() is skipped on shutdown unless the actor traps exits"). The shutdown-order nodes trap exits, as cross_backend does since 8beb7038. - sim.rs pr204_* and shrink_stays_within_its_run_budget, sim_explore pr204_stopped_outcome_stays_closed: the workload crashed c0 after readiness; under S1 that exit is deferred until the initial start completes, so the start no longer failed and no child stayed never-started (the tape tests found no failing seed). c0 now fails its first start (started() panics after k yields), which aborts the initial start under S1 and keeps the PR #204 check: never-started children end Failed, never Stopped (800-run sweep, DFS bound 2 exhausted). --- concurrency/tests/sim.rs | 45 ++++++++++++++++++++------------ concurrency/tests/sim_explore.rs | 36 ++++++++----------------- 2 files changed, 39 insertions(+), 42 deletions(-) diff --git a/concurrency/tests/sim.rs b/concurrency/tests/sim.rs index 2cce010d..3f35b946 100644 --- a/concurrency/tests/sim.rs +++ b/concurrency/tests/sim.rs @@ -59,25 +59,30 @@ impl Message for Crash { type Result = (); } -/// Records started/stopped. With `crash_after: Some(k)`, `started()` spawns a -/// task that yields `k` times and then sends the actor a panicking message, -/// so it dies a few scheduling steps after it publishes readiness. +/// Records started/stopped. With `fail_start_after: Some(k)`, `started()` +/// yields `k` times and then panics, so the child's first start fails and +/// the supervisor's initial start aborts (S1 in +/// docs/design/otp-exit-signals.md; a crash after readiness would instead be +/// processed after the initial start completes). With `trap`, the actor traps +/// exits, so the system's or supervisor's shutdown (the parent's exit signal) +/// still runs `stopped()` (E3). struct Node { id: u32, - crash_after: Option, + fail_start_after: Option, + trap: bool, log: Log, } impl Actor for Node { async fn started(&mut self, ctx: &Context) { push(&self.log, format!("started:{}", self.id)); - if let Some(yields) = self.crash_after { - let actor = ctx.actor_ref(); - drop(spawned_rt::tasks::spawn(async move { - for _ in 0..yields { - sim::yield_now().await; - } - let _ = actor.send(Crash); - })); + if self.trap { + ctx.trap_exit(true); + } + if let Some(yields) = self.fail_start_after { + for _ in 0..yields { + sim::yield_now().await; + } + panic!("injected first-start failure"); } } async fn stopped(&mut self, _: &Context) { @@ -132,7 +137,8 @@ async fn shutdown_order(log: Log) -> Vec { let group = system.async_group("sim-order", 2).unwrap(); let start = |id| Node { id, - crash_after: None, + fail_start_after: None, + trap: true, log: log.clone(), }; for id in 0..2 { @@ -147,7 +153,8 @@ async fn shutdown_order(log: Log) -> Vec { async move { Ok::<_, String>(Node { id: 100, - crash_after: None, + fail_start_after: None, + trap: true, log, }) } @@ -224,7 +231,8 @@ async fn initial_failure(log: Log, yields: u32) -> Placement { async move { Ok::<_, String>(Node { id: index, - crash_after: (index == 0).then_some(yields), + fail_start_after: (index == 0).then_some(yields), + trap: false, log, }) } @@ -366,7 +374,9 @@ fn thread_groups_are_rejected() { /// PR #204 finding 2, pinned: before the v0.7.1 lifecycle fixes (PR #209), /// seed 12 left the launched but never-started child `c1` Failed and seed 2 /// left it Stopped. With PR #209 a failed initial placement finishes every -/// never-started child as Failed, so both seeds must now agree. +/// never-started child as Failed, so both seeds must now agree. (Since S1 the +/// injected failure is `c0`'s first start, so the seeds select different +/// schedules than in v0.7; the expected states are unchanged.) #[test] fn pr204_never_started_child_state_pinned_seeds() { for (seed, expected) in [(12, "Failed"), (2, "Failed")] { @@ -424,7 +434,8 @@ async fn restart_window(out: Arc>>) { .add(ChildSpec::new("flappy", &group, |_| async { Ok::<_, String>(Node { id: 0, - crash_after: None, + fail_start_after: None, + trap: false, log: Log::default(), }) })) diff --git a/concurrency/tests/sim_explore.rs b/concurrency/tests/sim_explore.rs index 474ea564..7f47c2b4 100644 --- a/concurrency/tests/sim_explore.rs +++ b/concurrency/tests/sim_explore.rs @@ -205,38 +205,22 @@ async fn lost_update_three_workers() { // terminal state depends on which of two finishes runs last. // --------------------------------------------------------------------------- -struct Crash; -impl Message for Crash { - type Result = (); -} struct Node { - crash: bool, + fail_start: bool, } -/// As in `tests/sim.rs` with zero yields: `started()` spawns a task that sends -/// the actor a panicking message. +/// As in `tests/sim.rs` with zero yields: `started()` panics, so the first +/// child's first start fails and the initial start aborts (S1 in +/// docs/design/otp-exit-signals.md). impl Actor for Node { - async fn started(&mut self, ctx: &Context) { - if self.crash { - let actor = ctx.actor_ref(); - drop(spawned_rt::tasks::spawn(async move { - let _ = actor.send(Crash); - })); + async fn started(&mut self, _: &Context) { + if self.fail_start { + panic!("injected first-start failure"); } } } -impl Handler for Node { - async fn handle( - &mut self, - _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, - _: &Context, - ) { - panic!("injected crash after readiness"); - } -} /// States (`Debug` of the state, generation) of three OneForAll children -/// whose first child crashes right after `started()`. +/// whose first child fails its first start. async fn initial_failure() -> Vec<(String, u64)> { let system = ActorSystem::new().unwrap(); let group = system.async_group("explore-initial", 2).unwrap(); @@ -258,7 +242,9 @@ async fn initial_failure() -> Vec<(String, u64)> { .map(|index| { tree.add( ChildSpec::new(format!("c{index}"), &group, move |_| async move { - Ok::<_, String>(Node { crash: index == 0 }) + Ok::<_, String>(Node { + fail_start: index == 0, + }) }) .restart(RestartMode::Permanent), ) From 850fd785bc5dd241e833cb5bc98eda5ac7ef4282 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 08:13:24 +0000 Subject: [PATCH 098/208] test(shutdown): do not rely on stopped() starting before a zero-grace kill Two sim_tests failed under the simulator (seeds 0 and 1 of 8): cancellation_safety system_shutdown_is_cancel_safe ("report wait") and execution_groups concurrent_zero_deadline_reports_include_scopes_during_ shutdown_initialization. Both held a trapping actor in stopped() and shut the system down with zero grace. The parent exit is queued for a trapping actor (E3), and the kill at grace skips stopped() if it has not begun (E1, decision 2), so when the kill lands before the parent exit is dispatched nothing is stuck and the report completes. Both orders are allowed by the design; Tokio just rarely picks the second. - report wait: shut down with a 30 ms grace, so the probe enters stopped() before the kill (deterministic under virtual time). - zero-deadline reports: hold the actor in a handler instead; kills and parent exits wait for the current handler, so it is unfinished at any deadline. 512 seeds and 40 native runs of each pass. --- concurrency/tests/cancellation_safety.rs | 9 +++++--- concurrency/tests/execution_groups.rs | 28 ++++++++++++++++++------ 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/concurrency/tests/cancellation_safety.rs b/concurrency/tests/cancellation_safety.rs index f49b0e36..68c4e4ba 100644 --- a/concurrency/tests/cancellation_safety.rs +++ b/concurrency/tests/cancellation_safety.rs @@ -808,15 +808,18 @@ async fn system_shutdown_is_cancel_safe() { probe.stop_gate = Some(gate.clone()); let actor = fx.spawn(probe); actor.ready().await.unwrap(); - let report = fx.system.as_ref().unwrap().shutdown(Duration::ZERO).await; + // A short grace, not zero: the trapping probe must dispatch the + // parent's exit and enter stopped() before the kill at grace + // (E3). With zero grace the kill can land first and skip + // stopped() (E1, decision 2), leaving nothing stuck to wait for. + let report = fx.system.as_ref().unwrap().shutdown(PARK).await; assert!(!report.complete); let outcome = attempt(report.wait(LIMIT), n, PARK).await; assert!( !matches!(outcome, Attempt::Done(ref r) if r.complete), "{op}" ); - // Zero grace: the system kills at once after its ordered pass - // (E3); let the kill land while stopped() still runs (E1). + // The kill at grace landed while stopped() still runs (E1). spawned_rt::tasks::sleep(Duration::from_millis(50)).await; gate.notify_one(); assert_eq!( diff --git a/concurrency/tests/execution_groups.rs b/concurrency/tests/execution_groups.rs index a8e1d80d..edfd7566 100644 --- a/concurrency/tests/execution_groups.rs +++ b/concurrency/tests/execution_groups.rs @@ -556,16 +556,27 @@ async fn readiness_reports_actual_panic_after_startup_stop_instead_of_early_norm assert!(system.shutdown(LIMIT).await.complete); } +/// Held in a handler until released. A kill and the parent's exit both wait +/// for the current handler (E1, E3), so the actor stays unfinished at any +/// deadline. (A hold in a trapping actor's `stopped()` is skipped when the +/// zero-grace kill lands before the parent's exit is dispatched: E1, +/// decision 2.) struct SnapshotHeld { release: Option>, } -impl Actor for SnapshotHeld { - async fn started(&mut self, ctx: &Context) { - // v0.8: stopped() runs on the system's shutdown only for a - // trapping actor (E3, decision 2). - ctx.trap_exit(true); - } - async fn stopped(&mut self, _: &Context) { +impl Actor for SnapshotHeld {} +struct Hold(tokio::sync::oneshot::Sender<()>); +impl Message for Hold { + type Result = (); +} +impl Handler for SnapshotHeld { + async fn handle( + &mut self, + msg: Hold, + _: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &Context, + ) { + let _ = msg.0.send(()); let _ = self.release.take().unwrap().await; } } @@ -580,6 +591,9 @@ async fn concurrent_zero_deadline_reports_include_scopes_during_shutdown_initial .start_on(&group) .unwrap(); actor.ready().await.unwrap(); + let (entered, entered_rx) = tokio::sync::oneshot::channel(); + actor.send(Hold(entered)).unwrap(); + entered_rx.await.unwrap(); let mut reporters = Vec::new(); for _ in 0..64 { let owner = system.clone(); From 8a9cba7d34ad4767262eae53e33a0781065efe29 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 11:17:06 -0300 Subject: [PATCH 099/208] fix(jobs): discard a cancelled emit_at registration without self-deadlock JobState::schedule_emit rechecks cancellation after registering a delayed emit, because a cancel can drain the still-empty scheduled list between JobCore::schedule and the push. The recheck iterated over `mem::take(&mut *lock(&self.scheduled))` in the `for` expression, so the guard lived for the whole loop while `ScheduledEmit::discard` -> `settle` locked the same list again: a deadlock in both modes. The list is now taken in its own statement. The discarded call also returns `JobGone` instead of `Ok(())`: the emit will never be delivered. Test: reply::jobs::tests::cancel_between_schedule_and_registration_discards_without_deadlock (a test hook cancels the job between the two steps; it deadlocked before). --- concurrency/src/reply/jobs.rs | 50 ++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index a162e362..1a9635ad 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -392,6 +392,8 @@ impl JobState { track: impl FnOnce(u64, std::sync::Weak), ) -> Result<(), JobGone> { self.core.schedule()?; + #[cfg(test)] + tests::after_schedule(); let emit = Arc::new(ScheduledEmit { at, job: Arc::downgrade(self), @@ -400,11 +402,13 @@ impl JobState { lock(&self.scheduled).push(emit.clone()); if self.is_cancelled() { // Cancelled between the check and the registration: the cancel - // may have missed it. - for emit in std::mem::take(&mut *lock(&self.scheduled)) { + // may have missed it. Take the list in its own statement: the + // discard locks it again (`settle`), so its guard must be gone. + let scheduled = std::mem::take(&mut *lock(&self.scheduled)); + for emit in scheduled { emit.discard(self); } - return Ok(()); + return Err(JobGone); } if at <= super::now_ticks() { emit.expire(at); @@ -830,6 +834,46 @@ mod tests { use super::*; use crate::reply::registry::Registry; + std::thread_local! { + /// Runs once between `schedule_emit`'s `JobCore::schedule` and its + /// registration of the emit (a deterministic interleaving). + static AFTER_SCHEDULE: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; + } + + pub(super) fn after_schedule() { + if let Some(hook) = AFTER_SCHEDULE.with(|hook| hook.borrow_mut().take()) { + hook(); + } + } + + /// Review P1 (Codex 3): a cancel that drains the still-empty scheduled + /// list between `JobCore::schedule` and the registration leaves the + /// registration to discard its own emit. It used to iterate under the + /// list's guard while `settle` locked the list again: a self-deadlock. + #[test] + fn cancel_between_schedule_and_registration_discards_without_deadlock() { + let state = JobState::new(ActorId::next(), 2); + let cancelled = state.clone(); + let (sent_tx, sent_rx) = std::sync::mpsc::channel::<()>(); + let (done_tx, done_rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + AFTER_SCHEDULE.with(|hook| *hook.borrow_mut() = Some(Box::new(move || cancelled.cancel()))); + let send: SendScheduled = Box::new(move || { + let _ = sent_tx.send(()); + Ok(()) + }); + let result = state.schedule_emit(u64::MAX - 1, send, |_, _| {}); + let _ = done_tx.send((result, state.core.scheduled())); + }); + let (result, scheduled) = done_rx + .recv_timeout(std::time::Duration::from_secs(10)) + .expect("schedule_emit deadlocked discarding its own emit"); + assert_eq!(result, Err(JobGone)); + assert_eq!(scheduled, 0, "the discarded emit is no longer scheduled"); + assert!(sent_rx.try_recv().is_err(), "a cancelled job's emit was sent"); + } + /// Review P1: a rejected reservation must not build a guard whose drop /// releases a credit that was never taken. #[test] From 8730a670e3d0c57fad178564f450532fb67e31bf Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 11:17:12 -0300 Subject: [PATCH 100/208] fix(jobs): return a refused emit's credit once, through its guard JobCore::enqueue_emit returned the emit credit itself when the job was cancelled or notified between the credit's acquisition and the enqueue, and the uncalled enqueue closure then dropped the EmitCredit guard, which returned it again. A job could end with more credits than it was given, and with `emit_credits = u16::MAX` the second return overflowed. The guard is now the only owner of the return: the refusal drops the closure (and its guard) after releasing the job's lock. The Loom model now emits through a guard like production instead of returning credits by hand. Tests: tasks::actor::jobs::tests::a_refused_emit_returns_its_credit_once and threads::actor::jobs::tests::a_refused_emit_returns_its_credit_once (both failed before: the credit came back twice); job_core_loom passes. --- concurrency/src/reply/job_core.rs | 13 +++++----- concurrency/src/reply/jobs.rs | 6 +++++ concurrency/src/tasks/jobs.rs | 39 ++++++++++++++++++++++++++++++ concurrency/src/threads/jobs.rs | 35 +++++++++++++++++++++++++++ concurrency/tests/job_core_loom.rs | 17 +++++++++++++ 5 files changed, 104 insertions(+), 6 deletions(-) diff --git a/concurrency/src/reply/job_core.rs b/concurrency/src/reply/job_core.rs index 168014fa..062c8946 100644 --- a/concurrency/src/reply/job_core.rs +++ b/concurrency/src/reply/job_core.rs @@ -234,18 +234,19 @@ impl JobCore { } /// Emit step 2: under the job's lock, enqueue unless the job is - /// cancelled or notified (then the credit returns here). An envelope the - /// mailbox refused comes back to be dropped after the lock is released; - /// its credit guard returns the credit then. + /// cancelled or notified. The caller's credit guard is the only owner of + /// the credit's return: it travels inside `enqueue` and returns the + /// credit when that closure is dropped uncalled (refused here) or when + /// an envelope the mailbox refused is dropped after the lock is released. pub(crate) fn enqueue_emit( &self, enqueue: impl FnOnce() -> Result<(), X>, ) -> Result<(), (JobGone, Option)> { - let mut inner = lock(&self.inner); + let inner = lock(&self.inner); if !inner.accepting() { - inner.emit_credits += 1; drop(inner); - self.emitters.notify_all(); + // `enqueue` (and the credit guard it owns) drops here, unlocked. + drop(enqueue); return Err((JobGone, None)); } enqueue().map_err(|unsent| (JobGone, Some(unsent))) diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 1a9635ad..0e98e836 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -423,6 +423,12 @@ impl JobState { self.core.is_running() } + /// Free emit credits (tests). + #[cfg(test)] + pub(crate) fn emit_credits(&self) -> u16 { + self.core.emit_credits() + } + /// Execution of a cancelled async job ended without a notification (the /// model's `Finish` when the notification was suppressed). #[cfg(feature = "async")] diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index 09993d8b..c6f42adb 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -910,3 +910,42 @@ where { Box::new(move |outcome| done(outcome, token)) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::tasks::ActorStart; + + struct Owner; + impl Actor for Owner {} + struct Note; + impl Message for Note { + type Result = (); + } + impl Handler for Owner { + async fn handle(&mut self, _: Note, _: PendingReply<'_, ()>, _: &Context) {} + } + + /// Review P1 (Codex 13): an emit refused after its credit was taken + /// (the job was cancelled in between) returns that credit once, through + /// its guard. It used to be returned by the refusal and again by the + /// guard, so the job ended with more credits than it was given (and + /// `u16::MAX` credits overflowed). + #[tokio::test] + async fn a_refused_emit_returns_its_credit_once() { + let owner = Owner.start(); + for credits in [1, u16::MAX] { + let state = JobState::new(owner.id(), credits); + let job = JobCtx { + state: state.clone(), + ctx: owner.context(), + }; + state.acquire_blocking().unwrap(); + state.cancel(); + assert_eq!(job.enqueue(Note), Err(JobGone)); + assert_eq!(state.emit_credits(), credits, "credit returned twice"); + } + owner.context().stop(); + owner.join().await; + } +} diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index 78f2421f..d4d3c140 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -566,3 +566,38 @@ impl Context { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::threads::ActorStart; + + struct Owner; + impl Actor for Owner {} + struct Note; + impl Message for Note { + type Result = (); + } + impl Handler for Owner { + fn handle(&mut self, _: Note, _: PendingReply<'_, ()>, _: &Context) {} + } + + /// Review P1 (Codex 13), threads mode: see the tasks-mode test. + #[test] + fn a_refused_emit_returns_its_credit_once() { + let owner = Owner.start(); + for credits in [1, u16::MAX] { + let state = JobState::new(owner.id(), credits); + let job = JobCtx { + state: state.clone(), + ctx: owner.context(), + }; + state.acquire_blocking().unwrap(); + state.cancel(); + assert_eq!(job.enqueue(Note), Err(JobGone)); + assert_eq!(state.emit_credits(), credits, "credit returned twice"); + } + owner.context().stop(); + owner.join(); + } +} diff --git a/concurrency/tests/job_core_loom.rs b/concurrency/tests/job_core_loom.rs index 46036b48..34adf5a6 100644 --- a/concurrency/tests/job_core_loom.rs +++ b/concurrency/tests/job_core_loom.rs @@ -181,6 +181,17 @@ enum Msg { Terminal(u32), } +/// The production emit credit guard (`EmitCredit`): returns the credit when +/// dropped unless it travelled with an enqueued message. +struct CreditGuard(Option>); +impl Drop for CreditGuard { + fn drop(&mut self) { + if let Some(job) = self.0.take() { + job.core.return_emit_credit(); + } + } +} + #[derive(Default)] struct Mailbox(Mutex>); impl Mailbox { @@ -325,8 +336,14 @@ fn emit_vs_notification_and_cancel() { loom::thread::spawn(move || -> Result<(), JobGone> { for i in 0..2 { job.core.acquire_blocking()?; + // The production credit guard: it moves into the + // envelope when enqueued and returns the credit when + // dropped otherwise (the only owner of the return). + let credit = CreditGuard(Some(job.clone())); job.core .enqueue_emit(|| { + let mut credit = credit; + credit.0 = None; // travels with the message mailbox.push(Msg::Emit(0, i)); Ok::<(), ()>(()) }) From 903be60c8c1e8adda393b914bc70074dab82075d Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 11:19:50 -0300 Subject: [PATCH 101/208] fix(tasks): catch a panic while building a handler's future as a handler panic A manual tasks-mode `Handler` may be a plain `fn` returning `impl Future` that panics before returning it. The run loop evaluated `envelope.handle(actor, &ctx)` before `catch_unwind` was installed, so the panic escaped the turn: the outer lifecycle guard caught it, skipped the turn end and `stopped()`, and reported "actor lifecycle unwound" instead of the handler-panic reason. The envelope is now dispatched inside the guarded future, as threads mode already does. Test: tasks::actor::tests::panic_building_the_handler_future_is_a_handler_panic (failed before: no `stopped()`, lifecycle-unwound reason). --- concurrency/src/tasks/actor.rs | 58 +++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 434793e9..bbc6185c 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -1792,7 +1792,11 @@ async fn run_actor_lifecycle( .observation() .map(|observation| observation.begin()); ctx.shared.replies.begin_turn(); - let result = AssertUnwindSafe(envelope.handle(actor, &ctx)) + // The handler's future is built inside the guarded future: a + // manual `Handler` may panic before it returns one, and that + // is a handler panic like any other (threads mode catches the + // same call). + let result = AssertUnwindSafe(async { envelope.handle(actor, &ctx).await }) .catch_unwind() .await; ctx.shared.replies.end_turn(result.is_err()); @@ -2662,6 +2666,58 @@ mod tests { }); } + /// Review P1 (Codex 9): a manual `Handler` may be a plain `fn` that + /// panics before it returns its future. That is a handler panic like + /// any other: the turn ends, `stopped()` runs, and the caller and + /// `wait_exit` see the handler-panic reason. The future used to be built + /// outside the turn's `catch_unwind`, so the panic unwound the whole + /// lifecycle (no `stopped()`, an "actor lifecycle unwound" reason). + #[test] + pub fn panic_building_the_handler_future_is_a_handler_panic() { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + struct Eager(Arc); + struct Explode; + impl Message for Explode { + type Result = (); + } + impl Actor for Eager { + async fn stopped(&mut self, _ctx: &Context) { + self.0.store(true, Ordering::SeqCst); + } + } + impl Handler for Eager { + #[allow(unreachable_code, clippy::diverging_sub_expression)] + fn handle( + &mut self, + _msg: Explode, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) -> impl Future + Send { + panic!("boom building the handler future"); + async {} + } + } + + let stopped = Arc::new(atomic::AtomicBool::new(false)); + let actor = Eager(stopped.clone()).start(); + let reply = actor.request(Explode).await; + let reason = actor.wait_exit().await; + let ExitReason::Panic(message) = &reason else { + panic!("unexpected reason {reason:?}"); + }; + assert!( + message.starts_with("panic in handler: boom building"), + "not a handler panic: {message}" + ); + assert!( + matches!(&reply, Err(ActorError::ActorStopped(r)) if *r == reason), + "caller saw {reply:?}" + ); + assert!(stopped.load(Ordering::SeqCst), "stopped() was skipped"); + }); + } + #[test] pub fn panic_in_stopped_still_completes() { let runtime = rt::Runtime::new().unwrap(); From 49ce7530598fc26a37ddc90cdb184c782085e557 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 11:23:45 -0300 Subject: [PATCH 102/208] fix(jobs): a result held for scheduled emits keeps its deadline and credit While a job's result waits for its scheduled emits (`emit_at`), two things the job owned were released when the work returned: - the deadline: the run held the only strong `Arc` as a local of the executing task (the heap holds a `Weak`), so a hard `JobOptions::deadline` never fired and the emit and `Ok` arrived at the emit's instant instead of `DeadlineExceeded` at the deadline (Codex 8, agy 1); - the `MAX_OUTSTANDING` credit (or the linked slot's job count): returned before `finish`, so held jobs and their scheduled emits were not bounded (agy 5). The run now owns its deadline, and takes the execution's hold when the work ends; the terminal message's enqueue releases it (still before the message can be dispatched, so `done`'s turn sees the credit back), as does dropping the run on cancellation. A deadline that fires while the work still runs leaves the hold with the execution, as before. An `emit_at` that is already due and is refused at once now returns `JobGone`. Tests (deferred_api, both modes): a_deadline_overtakes_a_result_held_for_scheduled_emits, a_held_result_keeps_its_credit_until_the_terminal_message; all four failed before. --- concurrency/src/reply/jobs.rs | 16 +++- concurrency/src/tasks/jobs.rs | 62 ++++++++++------ concurrency/src/threads/jobs.rs | 45 +++++++---- concurrency/tests/deferred_api.rs | 119 ++++++++++++++++++++++++++++++ 4 files changed, 201 insertions(+), 41 deletions(-) diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 0e98e836..1339cfd3 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -385,6 +385,11 @@ impl JobState { /// when the deadline fires, fenced like any emit, and the job's terminal /// message waits for it. `track` arms the owner's deadline heap. On /// `Err` the caller's credit guard (inside `send`) returns the credit. + /// + /// `JobGone` also when the emit will never be delivered: the job was + /// cancelled while registering, or an emit already due was refused. An + /// `at` past the job's deadline is accepted: the deadline takes the + /// notification first and the emit is dropped at its instant. pub(crate) fn schedule_emit( self: &Arc, at: u64, @@ -411,7 +416,9 @@ impl JobState { return Err(JobGone); } if at <= super::now_ticks() { - emit.expire(at); + if !emit.settle(self, true) { + return Err(JobGone); + } } else { let weak: std::sync::Weak = Arc::downgrade(&emit) as _; track(at, weak); @@ -747,9 +754,10 @@ pub(crate) struct ScheduledEmit { impl ScheduledEmit { /// Fire (`deliver`) or discard: step 2 of the emit protocol under the /// job's lock, then remove it from the job, then run a held terminal. - fn settle(&self, job: &JobState, deliver: bool) { + /// Returns whether the message was enqueued. + fn settle(&self, job: &JobState, deliver: bool) -> bool { let Some(send) = lock(&self.send).take() else { - return; + return false; }; // A discarded message (and its credit guard) drops at the end, // outside the trace section and every lock. @@ -766,6 +774,7 @@ impl ScheduledEmit { } (result, held) }); + let delivered = result.is_ok(); // A refused envelope (and its credit guard) drops here, unlocked. drop(result); lock(&job.scheduled).retain(|emit| !std::ptr::eq(Arc::as_ptr(emit), self)); @@ -774,6 +783,7 @@ impl ScheduledEmit { } #[cfg(feature = "async")] job.wake_emitters(); + delivered } fn discard(&self, job: &JobState) { diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index c6f42adb..e75f3352 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -121,6 +121,14 @@ struct Run { state: Arc, ctx: Context, done: Mutex>>, + /// The job's deadline (the heap holds a `Weak`): owned until the run is + /// dropped, so it still fires while a result waits for scheduled emits. + deadline: std::sync::OnceLock>, + /// What execution held (its credit, or its place in a linked slot), + /// handed over when execution ends and released when the terminal + /// message is enqueued (or the run is dropped): a result waiting for + /// scheduled emits keeps it, so `MAX_OUTSTANDING` still bounds it. + hold: Mutex>>, } fn end_of(outcome: &JobOutcome) -> &'static str { @@ -147,6 +155,12 @@ where fn enqueue_terminal(&self, outcome: JobOutcome) -> Option + Send>> { let done = self.done.lock().unwrap_or_else(|p| p.into_inner()).take()?; + // Execution ended (or is still running past its deadline, and then + // keeps its hold): return the credit before the terminal message + // can be dispatched, so `done`'s turn can start the next job at the + // bound. Only atomics run here (a credit or the slot's job count). + let hold = self.hold.lock().unwrap_or_else(|p| p.into_inner()).take(); + drop(hold); let envelope: Box + Send> = Box::new(TerminalEnvelope { job: self.state.clone(), done, @@ -157,8 +171,10 @@ where /// The work's result: notify now, or, while delayed emits are still /// scheduled, hold it for the last of them so the terminal message - /// stays last. - fn finish(self: &Arc, outcome: JobOutcome) { + /// stays last. `hold` is what execution held; the terminal message + /// releases it. + fn finish(self: &Arc, outcome: JobOutcome, hold: Box) { + *self.hold.lock().unwrap_or_else(|p| p.into_inner()) = Some(hold); let end = end_of(&outcome); let outcome = std::cell::Cell::new(Some(outcome)); let unsent = self.state.finish( @@ -535,17 +551,15 @@ impl Context { }) } - fn arm_deadline( - &self, - opts: &JobOptions, - run: &Arc>, - ) -> Option> + fn arm_deadline(&self, opts: &JobOptions, run: &Arc>) where A: Handler, M: Message, T: Send + 'static, { - let at = spawned_rt::time::to_ticks(opts.deadline?); + let Some(at) = opts.deadline.map(spawned_rt::time::to_ticks) else { + return; + }; let weak = Arc::downgrade(run); let deadline = Arc::new(JobDeadline { at, @@ -556,13 +570,14 @@ impl Context { } }))), }); + // The run owns it; the heap only refers to it. + let _ = run.deadline.set(deadline.clone()); if at <= now_ticks() { deadline.expire(at); } else { let item: Weak = Arc::downgrade(&deadline) as Weak; self.shared.replies.track(at, item); } - Some(deadline) } fn launch_async( @@ -588,14 +603,14 @@ impl Context { state: state.clone(), ctx: self.clone(), done: Mutex::new(Some(done)), + deadline: std::sync::OnceLock::new(), + hold: Mutex::new(None), }); - let deadline = self.arm_deadline(&opts, &run); + self.arm_deadline(&opts, &run); let ended = Ended(self.clone(), state.clone()); let handle = JobHandle::new(state.clone()); self.spawn_framework(async move { let _ended = ended; - let _hold = hold; - let _deadline = deadline; let cancelled = pin!(state.cancellation().cancelled()); let gone = pin!(async { match &caller_gone { @@ -608,13 +623,10 @@ impl Context { let work = pin!(AssertUnwindSafe(work).catch_unwind()); let stop = future::select(cancelled, gone); match future::select(work, stop).await { - future::Either::Left((result, _)) => { - // Execution ended: return its credit (and linked-job - // count) before the terminal message can be dispatched, - // so `done`'s turn can start the next job at the bound. - drop(_hold); - run.finish(caught(result)) - } + // Execution ended: its credit (and linked-job count) goes to + // the run, which returns it when the terminal message is + // enqueued. + future::Either::Left((result, _)) => run.finish(caught(result), Box::new(hold)), // Cancelled (or the opt-in caller give-up): the work future is // dropped here, and with it any downstream receiver. future::Either::Right(_) => { @@ -674,8 +686,10 @@ impl Context { state: state.clone(), ctx: self.clone(), done: Mutex::new(Some(done)), + deadline: std::sync::OnceLock::new(), + hold: Mutex::new(None), }); - let deadline = self.arm_deadline(&opts, &run); + self.arm_deadline(&opts, &run); let ended = Ended(self.clone(), state.clone()); let job = JobCtx { state: state.clone(), @@ -683,11 +697,11 @@ impl Context { }; self.spawn_native(move || { let _ended = ended; - let _deadline = deadline; let result = std::panic::catch_unwind(AssertUnwindSafe(|| work(&job))); - // Execution ended: release the native slot and credit first. - drop((native, hold)); - run.finish(caught(result)); + // Execution ended: release the native slot now; the run returns + // the credit when the terminal message is enqueued. + drop(native); + run.finish(caught(result), Box::new(hold)); }); JobHandle::new(state) } diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index d4d3c140..6bc518fa 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -99,6 +99,14 @@ struct Run { state: Arc, ctx: Context, done: Mutex>>, + /// The job's deadline (the heap holds a `Weak`): owned until the run is + /// dropped, so it still fires while a result waits for scheduled emits. + deadline: std::sync::OnceLock>, + /// What execution held (its credit, or its place in a linked slot), + /// handed over when execution ends and released when the terminal + /// message is enqueued (or the run is dropped): a result waiting for + /// scheduled emits keeps it, so `MAX_OUTSTANDING` still bounds it. + hold: Mutex>>, } fn end_of(outcome: &JobOutcome) -> &'static str { @@ -125,6 +133,12 @@ where fn enqueue_terminal(&self, outcome: JobOutcome) -> Option> { let done = self.done.lock().unwrap_or_else(|p| p.into_inner()).take()?; + // Execution ended (or is still running past its deadline, and then + // keeps its hold): return the credit before the terminal message + // can be dispatched, so `done`'s turn can start the next job at the + // bound. Only atomics run here (a credit or the slot's job count). + let hold = self.hold.lock().unwrap_or_else(|p| p.into_inner()).take(); + drop(hold); let envelope: Box + Send> = Box::new(TerminalEnvelope { job: self.state.clone(), done, @@ -139,8 +153,10 @@ where /// The work's result: notify now, or, while delayed emits are still /// scheduled, hold it for the last of them so the terminal message - /// stays last. - fn finish(self: &Arc, outcome: JobOutcome) { + /// stays last. `hold` is what execution held; the terminal message + /// releases it. + fn finish(self: &Arc, outcome: JobOutcome, hold: Box) { + *self.hold.lock().unwrap_or_else(|p| p.into_inner()) = Some(hold); let end = end_of(&outcome); let outcome = std::cell::Cell::new(Some(outcome)); let unsent = self.state.finish( @@ -400,17 +416,15 @@ impl Context { }) } - fn arm_deadline( - &self, - opts: &JobOptions, - run: &Arc>, - ) -> Option> + fn arm_deadline(&self, opts: &JobOptions, run: &Arc>) where A: Handler, M: Message, T: Send + 'static, { - let at = spawned_rt::time::to_ticks(opts.deadline?); + let Some(at) = opts.deadline.map(spawned_rt::time::to_ticks) else { + return; + }; let weak = Arc::downgrade(run); let deadline = Arc::new(JobDeadline { at, @@ -421,13 +435,14 @@ impl Context { } }))), }); + // The run owns it; the heap only refers to it. + let _ = run.deadline.set(deadline.clone()); if at <= now_ticks() { deadline.expire(at); } else { let item: Weak = Arc::downgrade(&deadline) as Weak; self.replies.track(at, item); } - Some(deadline) } fn launch_blocking( @@ -452,8 +467,10 @@ impl Context { state: state.clone(), ctx: self.clone(), done: Mutex::new(Some(done)), + deadline: std::sync::OnceLock::new(), + hold: Mutex::new(None), }); - let deadline = self.arm_deadline(&opts, &run); + self.arm_deadline(&opts, &run); let ended = Ended(self.clone(), state.clone()); let job = JobCtx { state: state.clone(), @@ -461,11 +478,11 @@ impl Context { }; let task: NativeTask = Box::new(move |native| { let _ended = ended; - let _deadline = deadline; let result = std::panic::catch_unwind(AssertUnwindSafe(|| work(&job))); - // Execution ended: release the native slot and credit first. - drop((native, hold)); - run.finish(caught(result)); + // Execution ended: release the native slot now; the run returns + // the credit when the terminal message is enqueued. + drop(native); + run.finish(caught(result), Box::new(hold)); }); // The worker thread is parked on `recv` until this send (or its // sender's drop); it cannot have exited. diff --git a/concurrency/tests/deferred_api.rs b/concurrency/tests/deferred_api.rs index f0f0cddd..07ac2c8c 100644 --- a/concurrency/tests/deferred_api.rs +++ b/concurrency/tests/deferred_api.rs @@ -274,6 +274,69 @@ mod tasks_mode { actor.join().await; } + /// Review P1 (Codex 8, agy 1): a hard deadline overtakes a result held + /// for a scheduled emit. The run used to drop its only strong reference + /// to the deadline when the work returned, so the deadline never fired: + /// the emit and `Ok` arrived at the emit's instant instead. + #[spawned_macros::sim_test] + async fn a_deadline_overtakes_a_result_held_for_scheduled_emits() { + let (actor, log) = start(); + in_actor(&actor, |_, ctx| { + let opts = JobOptions::default().deadline_after(Duration::from_millis(40)); + ctx.pipe_with_ctx( + opts, + |job| async move { + let at = Instant::now() + Duration::from_millis(300); + job.emit_at(at, Note("late".into())).await.unwrap(); + 7 + }, + |o| describe("job", o), + ) + .unwrap(); + }) + .await; + spawned_rt::tasks::sleep(Duration::from_millis(150)).await; + assert_eq!(log.snapshot(), ["job:deadline"]); + // The scheduled emit is dropped at its instant (the job is over). + spawned_rt::tasks::sleep(Duration::from_millis(300)).await; + assert_eq!(log.snapshot(), ["job:deadline"]); + let outstanding = in_actor(&actor, |_, ctx| ctx.deferred_outstanding()).await; + assert_eq!(outstanding, 0); + actor.context().stop(); + actor.join().await; + } + + /// Review P1 (agy 5): a result held for scheduled emits keeps the job's + /// `MAX_OUTSTANDING` credit until its terminal message is enqueued. It + /// used to return the credit when the work returned, so held jobs (and + /// their scheduled emits) were not bounded by `MAX_OUTSTANDING`. + #[spawned_macros::sim_test] + async fn a_held_result_keeps_its_credit_until_the_terminal_message() { + let (actor, log) = start(); + in_actor(&actor, |_, ctx| { + ctx.pipe_with_ctx( + JobOptions::default(), + |job| async move { + let at = Instant::now() + Duration::from_millis(200); + job.emit_at(at, Note("late".into())).await.unwrap(); + 1 + }, + |o| describe("job", o), + ) + .unwrap(); + }) + .await; + spawned_rt::tasks::sleep(Duration::from_millis(50)).await; + assert!(log.snapshot().is_empty()); + let outstanding = in_actor(&actor, |_, ctx| ctx.deferred_outstanding()).await; + assert_eq!(outstanding, 1, "a held result returned its credit early"); + assert_eq!(wait_for(&log, 2).await, ["late", "job:ok:1"]); + let outstanding = in_actor(&actor, |_, ctx| ctx.deferred_outstanding()).await; + assert_eq!(outstanding, 0); + actor.context().stop(); + actor.join().await; + } + #[spawned_macros::sim_test] async fn send_handler_impls_answer_requests_with_unit() { let (actor, log) = start(); @@ -395,6 +458,62 @@ mod threads_mode { actor.join(); } + /// Review P1 (Codex 8, agy 1), threads mode: a hard deadline overtakes + /// a result held for a scheduled emit. + #[test] + fn a_deadline_overtakes_a_result_held_for_scheduled_emits() { + let log = Arc::new(Log::default()); + let actor = Owner { log: log.clone() }.start(); + in_actor(&actor, |_, ctx| { + let opts = JobOptions::default().deadline_after(Duration::from_millis(40)); + ctx.pipe_blocking_with( + opts, + |job| { + let at = Instant::now() + Duration::from_millis(300); + job.emit_at_blocking(at, Note("late".into())) + }, + |o| label("job", o), + ) + .unwrap(); + }); + std::thread::sleep(Duration::from_millis(150)); + assert_eq!(log.snapshot(), ["job:Err(DeadlineExceeded)"]); + std::thread::sleep(Duration::from_millis(300)); + assert_eq!(log.snapshot(), ["job:Err(DeadlineExceeded)"]); + assert_eq!(in_actor(&actor, |_, ctx| ctx.deferred_outstanding()), 0); + actor.context().stop(); + actor.join(); + } + + /// Review P1 (agy 5), threads mode: a held result keeps its credit. + #[test] + fn a_held_result_keeps_its_credit_until_the_terminal_message() { + let log = Arc::new(Log::default()); + let actor = Owner { log: log.clone() }.start(); + in_actor(&actor, |_, ctx| { + ctx.pipe_blocking_with( + JobOptions::default(), + |job| { + let at = Instant::now() + Duration::from_millis(200); + job.emit_at_blocking(at, Note("late".into())) + }, + |o| label("job", o), + ) + .unwrap(); + }); + std::thread::sleep(Duration::from_millis(50)); + assert!(log.snapshot().is_empty()); + assert_eq!( + in_actor(&actor, |_, ctx| ctx.deferred_outstanding()), + 1, + "a held result returned its credit early" + ); + assert_eq!(wait_for(&log, 2), ["late", "job:Ok(Ok(()))"]); + assert_eq!(in_actor(&actor, |_, ctx| ctx.deferred_outstanding()), 0); + actor.context().stop(); + actor.join(); + } + #[test] fn send_handler_impls_answer_requests_with_unit() { let log = Arc::new(Log::default()); From 1d2dcb7d5eb2beda06c3fc2cdfabf386307d6917 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 11:26:32 -0300 Subject: [PATCH 103/208] fix(jobs): a shortened token deadline no longer cancels a caller-gone job `pipe_reply_blocking*` with `cancel_on_caller_gone` registers a watcher waker on the linked slot. Watchers are also woken when the token's deadline is shortened (so timers re-arm), and `CancelJob::wake` cancelled the job unconditionally, while the caller was still waiting. A blocking linked job's deadline hands the token back while the work runs, so a handler shortening it cancelled live work. The watcher now rechecks the slot (weakly referenced) and re-registers while it is open, as an awaiting watcher re-polls. Test (deferred_api, both modes): shortening_the_deadline_does_not_cancel_a_caller_gone_job (failed before); it also checks that a caller leaving still cancels the job, which no test covered for blocking jobs. --- concurrency/src/reply/jobs.rs | 38 ++++- concurrency/tests/deferred_api.rs | 248 ++++++++++++++++++++++++++++++ 2 files changed, 280 insertions(+), 6 deletions(-) diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 1339cfd3..9fd52c26 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -832,15 +832,41 @@ impl Drop for CleanedGuard { /// Opt-in caller-give-up cancellation for jobs that cannot await /// (`pipe_reply_blocking*` with `cancel_on_caller_gone`): a watcher waker on /// the linked slot cancels the job when the slot closes. -pub(crate) fn cancel_when_closed(slot: &super::slot::Slot, state: &Arc) { - struct CancelJob(Arc); - impl std::task::Wake for CancelJob { +/// +/// Watchers are also woken when the token's deadline is shortened (so timers +/// re-arm): the waker rechecks the slot and re-registers while it is open, +/// as an awaiting watcher would re-poll. It refers to the slot weakly (the +/// slot stores it), and a slot that is gone has no caller left. +pub(crate) fn cancel_when_closed( + slot: &Arc>, + state: &Arc, +) { + struct CancelJob { + slot: std::sync::Weak>, + state: Arc, + } + impl CancelJob { + /// Register as the slot's watcher; `true` once the slot is closed. + fn closed(self: &Arc) -> bool { + let Some(slot) = self.slot.upgrade() else { + return true; + }; + let waker = std::task::Waker::from(self.clone()); + slot.poll_closed(&waker, &super::now_ticks).is_ready() + } + } + impl std::task::Wake for CancelJob { fn wake(self: Arc) { - self.0.cancel(); + if self.closed() { + self.state.cancel(); + } } } - let waker = std::task::Waker::from(Arc::new(CancelJob(state.clone()))); - if slot.poll_closed(&waker, &super::now_ticks).is_ready() { + let watcher = Arc::new(CancelJob { + slot: Arc::downgrade(slot), + state: state.clone(), + }); + if watcher.closed() { state.cancel(); } } diff --git a/concurrency/tests/deferred_api.rs b/concurrency/tests/deferred_api.rs index 07ac2c8c..507d557b 100644 --- a/concurrency/tests/deferred_api.rs +++ b/concurrency/tests/deferred_api.rs @@ -525,3 +525,251 @@ mod threads_mode { actor.join(); } } + +/// Review P2 (Codex 14, agy 3): shortening a delegated token's deadline +/// wakes its watchers so they re-arm; the caller-gone watcher of a blocking +/// `pipe_reply` job must recheck the slot, not cancel the job. A blocking +/// linked job's deadline hands the token back while the work keeps running; +/// the handler then shortens the token and checks the job. +#[cfg(all(feature = "async", not(spawned_sim)))] // native jobs are not simulated +mod caller_gone_vs_shortening_tasks { + use spawned_concurrency::message::Message; + use spawned_concurrency::tasks::{ + Actor, ActorStart, Context, Handler, Instant, JobError, JobHandle, JobOptions, JobOutcome, + PendingReply, ReplyTo, + }; + use std::time::Duration; + + #[derive(Default)] + struct Delegator { + job: Option, + } + impl Actor for Delegator {} + struct Ask; + impl Message for Ask { + type Result = (bool, bool); + } + struct Delegated(JobOutcome<()>, ReplyTo<(bool, bool)>); + impl Message for Delegated { + type Result = (); + } + impl Handler for Delegator { + async fn handle(&mut self, _: Ask, reply: PendingReply<'_, (bool, bool)>, ctx: &Context) { + let token = reply.defer().unwrap(); + let opts = JobOptions { + cancel_on_caller_gone: true, + ..JobOptions::default() + } + .deadline_after(Duration::from_millis(20)); + let job = ctx + .pipe_reply_blocking_with( + opts, + token, + |_| std::thread::sleep(Duration::from_millis(300)), + Delegated, + ) + .unwrap(); + self.job = Some(job); + } + } + struct Leave; + impl Message for Leave { + type Result = (); + } + struct Probe; + impl Message for Probe { + type Result = bool; + } + struct Ended(ReplyTo<()>); + impl Message for Ended { + type Result = (); + } + impl Handler for Delegator { + async fn handle(&mut self, msg: Ended, _: PendingReply<'_, ()>, _: &Context) { + msg.0.reply(()); + } + } + impl Handler for Delegator { + async fn handle(&mut self, _: Leave, reply: PendingReply<'_, ()>, ctx: &Context) { + let token = reply.defer().unwrap(); + let opts = JobOptions { + cancel_on_caller_gone: true, + ..JobOptions::default() + }; + let job = ctx + .pipe_reply_blocking_with( + opts, + token, + |job| { + while !job.is_cancelled() { + std::thread::sleep(Duration::from_millis(5)); + } + }, + |_, token| Ended(token), + ) + .unwrap(); + self.job = Some(job); + } + } + impl Handler for Delegator { + async fn handle(&mut self, _: Probe, reply: PendingReply<'_, bool>, _: &Context) { + reply.reply(self.job.as_ref().unwrap().is_cancelled()); + } + } + impl Handler for Delegator { + async fn handle(&mut self, msg: Delegated, _: PendingReply<'_, ()>, _: &Context) { + let Delegated(outcome, token) = msg; + let deadline = matches!(outcome.into_result(), Err(JobError::DeadlineExceeded)); + token.with_deadline(Instant::now() + Duration::from_secs(30)); + let cancelled = self.job.as_ref().unwrap().is_cancelled(); + token.reply((deadline, cancelled)); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn shortening_the_deadline_does_not_cancel_a_caller_gone_job() { + let actor = Delegator::default().start(); + let (deadline, cancelled) = actor + .request_with_timeout(Ask, Duration::from_secs(60)) + .await + .unwrap(); + assert!(deadline, "the job deadline handed the token back"); + assert!(!cancelled, "a shortened deadline cancelled the job"); + // The caller leaving still cancels a blocking linked job. + let left = actor.request_raw(Leave).unwrap(); + assert!(!actor.request(Probe).await.unwrap(), "cancelled before the caller left"); + drop(left); + let mut cancelled = false; + for _ in 0..500 { + cancelled = actor.request(Probe).await.unwrap(); + if cancelled { + break; + } + tokio::time::sleep(Duration::from_millis(2)).await; + } + assert!(cancelled, "the caller left but the job was not cancelled"); + actor.context().stop(); + actor.join().await; + } +} + +#[cfg(not(spawned_sim))] +mod caller_gone_vs_shortening_threads { + use spawned_concurrency::message::Message; + use spawned_concurrency::threads::{ + Actor, ActorStart, Context, Handler, Instant, JobError, JobHandle, JobOptions, JobOutcome, + PendingReply, ReplyTo, + }; + use std::time::Duration; + + #[derive(Default)] + struct Delegator { + job: Option, + } + impl Actor for Delegator {} + struct Ask; + impl Message for Ask { + type Result = (bool, bool); + } + struct Delegated(JobOutcome<()>, ReplyTo<(bool, bool)>); + impl Message for Delegated { + type Result = (); + } + impl Handler for Delegator { + fn handle(&mut self, _: Ask, reply: PendingReply<'_, (bool, bool)>, ctx: &Context) { + let token = reply.defer().unwrap(); + let opts = JobOptions { + cancel_on_caller_gone: true, + ..JobOptions::default() + } + .deadline_after(Duration::from_millis(20)); + let job = ctx + .pipe_reply_blocking_with( + opts, + token, + |_| std::thread::sleep(Duration::from_millis(300)), + Delegated, + ) + .unwrap(); + self.job = Some(job); + } + } + struct Leave; + impl Message for Leave { + type Result = (); + } + struct Probe; + impl Message for Probe { + type Result = bool; + } + struct Ended(ReplyTo<()>); + impl Message for Ended { + type Result = (); + } + impl Handler for Delegator { + fn handle(&mut self, msg: Ended, _: PendingReply<'_, ()>, _: &Context) { + msg.0.reply(()); + } + } + impl Handler for Delegator { + fn handle(&mut self, _: Leave, reply: PendingReply<'_, ()>, ctx: &Context) { + let token = reply.defer().unwrap(); + let opts = JobOptions { + cancel_on_caller_gone: true, + ..JobOptions::default() + }; + let job = ctx + .pipe_reply_blocking_with( + opts, + token, + |job| { + while !job.is_cancelled() { + std::thread::sleep(Duration::from_millis(5)); + } + }, + |_, token| Ended(token), + ) + .unwrap(); + self.job = Some(job); + } + } + impl Handler for Delegator { + fn handle(&mut self, _: Probe, reply: PendingReply<'_, bool>, _: &Context) { + reply.reply(self.job.as_ref().unwrap().is_cancelled()); + } + } + impl Handler for Delegator { + fn handle(&mut self, msg: Delegated, _: PendingReply<'_, ()>, _: &Context) { + let Delegated(outcome, token) = msg; + let deadline = matches!(outcome.into_result(), Err(JobError::DeadlineExceeded)); + token.with_deadline(Instant::now() + Duration::from_secs(30)); + let cancelled = self.job.as_ref().unwrap().is_cancelled(); + token.reply((deadline, cancelled)); + } + } + + #[test] + fn shortening_the_deadline_does_not_cancel_a_caller_gone_job() { + let actor = Delegator::default().start(); + let (deadline, cancelled) = actor + .request_with_timeout(Ask, Duration::from_secs(60)) + .unwrap(); + assert!(deadline, "the job deadline handed the token back"); + assert!(!cancelled, "a shortened deadline cancelled the job"); + // The caller leaving still cancels a blocking linked job. + let left = actor.request_raw(Leave).unwrap(); + assert!(!actor.request(Probe).unwrap(), "cancelled before the caller left"); + drop(left); + let mut cancelled = false; + for _ in 0..500 { + cancelled = actor.request(Probe).unwrap(); + if cancelled { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } + assert!(cancelled, "the caller left but the job was not cancelled"); + actor.context().stop(); + actor.join(); + } +} From 13204795b22a6f201b058dfadece9dd6c8f55b5c Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 11:28:10 -0300 Subject: [PATCH 104/208] fix(reply): a shortened deadline wakes the waiting caller to re-arm its timer `ReplyTo::with_deadline` woke only the token-side watchers. The caller's `ReplyFuture` kept its runtime timer armed for the deadline fixed at send, and a `recv_blocking` thread kept its timed park; the owner's deadline heap still expired the slot on time in real time, but a paused Tokio clock auto-advanced to the old deadline. A lowered deadline now also wakes the caller's receiver (moved out of the slot, not cloned under its lock) and notifies parked threads, which re-arm for it. agy's second point (raw `.await` uses the owner's heap, not the runtime's timer) is a deliberate optimisation (c134b6f2); the design doc and T4 claimed otherwise. They now say which waits follow paused Tokio time and that the heap also holds awaited raw receivers. Test: deferred_replies tasks_mode::a_shortened_deadline_rearms_the_callers_timer (paused clock; failed before: the caller slept 60 s instead of 50 ms). reply_slot_loom passes. --- concurrency/src/reply/slot.rs | 33 +++++++++++++++--------- concurrency/tests/deferred_replies.rs | 36 +++++++++++++++++++++++++++ docs/design/deferred-replies.md | 23 ++++++++++++----- docs/design/semantics.md | 8 +++++- 4 files changed, 81 insertions(+), 19 deletions(-) diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 76151ff6..789eae0e 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -422,26 +422,35 @@ impl Slot { /// finds `WRITING` (or later) is a no-op; one that lowers the deadline is /// seen by any reply that has not read it yet. pub(crate) fn shorten(&self, at: u64, now: &impl Fn() -> u64) -> bool { - let lowered = { - let mut _inner = lock(&self.inner); + let (lowered, watchers, waker, parked) = { + let mut inner = lock(&self.inner); let lower = state(self.word()) == OPEN && at < self.deadline(); if lower { self.deadline.store(at, Ordering::Release); + // Moved out, not cloned: no executor code under the lock. A + // woken receiver re-registers when it polls again. + ( + true, + std::mem::take(&mut inner.watchers), + inner.waker.take(), + inner.parked > 0, + ) + } else { + (false, Vec::new(), None, false) } - ( - lower, - if lower { - std::mem::take(&mut _inner.watchers) - } else { - Vec::new() - }, - ) }; - let (lowered, watchers) = lowered; - // Token-side waiters re-arm their timers for the earlier deadline. + // Both sides re-arm their timers for the earlier deadline: the + // token's waiters, the caller's receiver and parked `recv_blocking` + // threads (which recompute their wait). for watcher in watchers { watcher.wake(); } + if let Some(waker) = waker { + waker.wake(); + } + if parked { + self.parked.notify_all(); + } self.check_deadline(now); lowered } diff --git a/concurrency/tests/deferred_replies.rs b/concurrency/tests/deferred_replies.rs index 0cb1a955..ca2190b4 100644 --- a/concurrency/tests/deferred_replies.rs +++ b/concurrency/tests/deferred_replies.rs @@ -441,6 +441,42 @@ mod tasks_mode { assert!(token.is_closed()); } + /// Review P1 (agy 4): shortening a token's deadline wakes an awaiting + /// caller, so its own timer re-arms for the earlier deadline. Without + /// the wake it kept sleeping until the deadline fixed at send; real time + /// was saved by the owner's heap, but a paused Tokio clock auto-advanced + /// to the old deadline (here 60 s instead of 50 ms). + #[cfg(not(spawned_sim))] + #[tokio::test(start_paused = true)] + async fn a_shortened_deadline_rearms_the_callers_timer() { + struct Shortener(Option>); + impl Actor for Shortener {} + struct Shorten; + impl spawned_concurrency::message::Message for Shorten { + type Result = (); + } + impl Handler for Shortener { + async fn handle(&mut self, _: Shorten, reply: PendingReply<'_, ()>, _: &Context) { + let token = reply.defer().unwrap(); + token.with_deadline(spawned_rt::time::Instant::now() + Duration::from_millis(50)); + self.0 = Some(token); + } + } + let actor = Shortener(None).start(); + let began = tokio::time::Instant::now(); + let result = actor + .request_with_timeout(Shorten, Duration::from_secs(60)) + .await; + assert!(matches!(result, Err(ActorError::RequestTimeout)), "{result:?}"); + assert!( + began.elapsed() < Duration::from_secs(5), + "the caller slept until the old deadline: {:?}", + began.elapsed() + ); + actor.context().stop(); + actor.join().await; + } + #[spawned_macros::sim_test] async fn request_timeout_starts_at_send() { let (actor, _) = start(); diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index 34ad7b8c..a69f8e9c 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -204,8 +204,14 @@ absolute instant, and OTP's `call` starts waiting when it sends. **Who expires a slot.** -- **A polling receiver.** It sleeps until the slot's deadline (rt clock) and - then `CAS OPEN -> TIMED_OUT`. +- **A polling receiver.** `request`, `request_with_timeout` and + `Response::from` sleep on the runtime's timer until the slot's deadline + and then `CAS OPEN -> TIMED_OUT` (so they follow paused Tokio clocks). A + raw receiver awaited directly (`request_raw(..)?.await`) instead puts its + deadline once in the owner's heap, which expires it on the service's clock + (real time, or the simulated clock under `spawned_sim`), saving a timer + per request. A shortened deadline (`ReplyTo::with_deadline`) wakes the + awaiting or parked receiver, which re-arms for it. - **`defer()`, the `pipe_reply` attach and `is_closed()`.** They check the deadline first, so an unpolled, expired request reports `Closed`. - **The owner's deadline heap,** for deferred tokens and for job deadlines. @@ -232,7 +238,9 @@ handles a call even after the caller timed out, and the late reply is dropped. `with_deadline`). - **Compaction.** The heap is rebuilt when stale entries exceed live ones, per actor. So storage is at most about 2 × (live deferred tokens + live - jobs with deadlines), at most about 2 × (`MAX_OUTSTANDING` + job count). + jobs with deadlines + awaited raw receivers); the first two are bounded by + `MAX_OUTSTANDING` + job count, the last by the requests callers are + awaiting (each also holds a queued envelope or an open slot). - **Ownership.** Entries hold `Weak` references, so the heap owns no slot and no job. The heap is dropped at the owner's death, after the registry close. - **Service.** A per-system service holds one entry per actor: its earliest @@ -972,9 +980,12 @@ semantics above. lock that a shortening takes, which makes a shortening after the `WRITING` CAS a no-op (Loom: `reply_vs_shortening`). - **Receiver and timer.** `ReplyReceiver` is one pointer and implements - `IntoFuture`; the deadline timer lives in the awaiting `ReplyFuture`. When - the runtime's timer fires the receiver expires the slot, so paused Tokio - clocks keep working. + `IntoFuture`; the deadline timer lives in the awaiting `ReplyFuture`. On + the `request` and `Response` paths, when the runtime's timer fires the + receiver expires the slot, so paused Tokio clocks keep working. A raw + receiver awaited directly registers its deadline in the owner's heap + instead (no runtime timer per request), so it follows the deadline + service's clock, not a paused Tokio clock. - **Deadline service.** Unmanaged and managed actors share one process-wide service thread (not a task on the `ActorSystem` control runtime); a dead owner is forgotten at its registry close. Under `spawned_sim` each owner's diff --git a/docs/design/semantics.md b/docs/design/semantics.md index cebc6daa..28d1ea81 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -584,7 +584,13 @@ is tasks-only). **T4. Clocks.** Delays use monotonic time (Tokio timers in tasks, so they follow paused Tokio time in tests; `recv_timeout` in threads). threads: each timer is -one OS thread. Nothing is persisted across process restarts. +one OS thread. Nothing is persisted across process restarts. Request +deadlines are instants on the `spawned_rt` clock: `request`, +`request_with_timeout` and `Response::from` wait on the runtime's timer (paused +Tokio time applies, and a shortened deadline re-arms it); token and job +deadlines, `emit_at` and a raw receiver awaited directly are serviced by the +owner's deadline heap on the real clock (the simulated clock under +`spawned_sim`), not by paused Tokio time. ## 9. Panics From 2c76646f1d8596374aa93a730bb49f39688e8ae9 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 11:32:21 -0300 Subject: [PATCH 105/208] fix(reply): contain user payload on the process-wide deadline service The deadline service is one thread per process; its `OnceLock` keeps the service after the thread dies. A job's scheduled emit discarded at its instant (after the job's deadline took the notification) dropped the user's message on that thread, and inside `JobCore::fire` under the job's lock: a panicking destructor killed the service, so every later raw-request deadline, deferred-credit expiry, job deadline and scheduled emit in the process stopped. - `JobCore::fire` hands an uncalled enqueue back instead of dropping it under the job's lock. - A scheduled emit drops every payload (discarded, refused or uncalled message, and itself if dropped unsettled) through the owner's failure sink, so the emit still settles (a held result still runs). - A job's run contains its other payload drops that can land on the service: a refused terminal message, a held outcome dropped unrun, and `done` when the last reference goes. - `Registry::service_deadlines` runs each due entry, and its last reference, inside `catch_unwind`, and the service thread guards each visit, so one failing entry neither skips the others nor ends servicing. Test: deadline_service::deadline_service_survives_a_panicking_payload_destructor (subprocess; before the fix the service thread died and another actor's request deadline never fired). job_core_loom passes with the new `fire`. --- concurrency/src/reply/deadline.rs | 13 ++- concurrency/src/reply/job_core.rs | 25 +++-- concurrency/src/reply/jobs.rs | 62 ++++++++++-- concurrency/src/reply/registry.rs | 27 ++++- concurrency/src/tasks/jobs.rs | 24 ++++- concurrency/src/threads/jobs.rs | 24 ++++- concurrency/tests/deadline_service.rs | 136 ++++++++++++++++++++++++++ concurrency/tests/job_core_loom.rs | 4 +- 8 files changed, 280 insertions(+), 35 deletions(-) create mode 100644 concurrency/tests/deadline_service.rs diff --git a/concurrency/src/reply/deadline.rs b/concurrency/src/reply/deadline.rs index eccd84e9..1bd7bea0 100644 --- a/concurrency/src/reply/deadline.rs +++ b/concurrency/src/reply/deadline.rs @@ -246,11 +246,16 @@ mod thread_service { } state.actors.remove(&key); drop(state); - if let Some(registry) = weak.upgrade() { - if let Some(next) = registry.service_deadlines(now) { - schedule(®istry, next); + // Entries are contained one by one; this also covers the + // visit itself and dropping the upgraded registry, so the + // one process-wide service never stops. + super::super::registry::contain_entry(|| { + if let Some(registry) = weak.upgrade() { + if let Some(next) = registry.service_deadlines(now) { + schedule(®istry, next); + } } - } + }); state = lock(service); } Some(Reverse((at, _))) => { diff --git a/concurrency/src/reply/job_core.rs b/concurrency/src/reply/job_core.rs index 062c8946..87e71b45 100644 --- a/concurrency/src/reply/job_core.rs +++ b/concurrency/src/reply/job_core.rs @@ -266,19 +266,24 @@ impl JobCore { /// Delayed emit, step 2 at its deadline, or when dropped unfired /// (`enqueue` is `None`): under the job's lock, enqueue if the - /// notification is still pending. A message not enqueued returns its - /// credit through its guard, after the lock. The last scheduled emit - /// hands back a held terminal, to run outside the lock. - pub(crate) fn fire( + /// notification is still pending. A message not enqueued (the refused + /// envelope, or the uncalled `enqueue` handed back as the third value) + /// returns its credit through its guard when the caller drops it, after + /// the lock: it is user payload. The last scheduled emit hands back a + /// held terminal, to run outside the lock. + pub(crate) fn fire Result<(), X>>( &self, - enqueue: Option Result<(), X>>, - ) -> (EmitResult, Option) { + enqueue: Option, + ) -> (EmitResult, Option, Option) { let mut inner = lock(&self.inner); debug_assert!(inner.scheduled > 0, "scheduled emit underflow"); inner.scheduled = inner.scheduled.saturating_sub(1); - let result = match enqueue { - Some(enqueue) if inner.open() => enqueue().map_err(|unsent| (JobGone, Some(unsent))), - _ => Err((JobGone, None)), + let (result, unused) = match enqueue { + Some(enqueue) if inner.open() => ( + enqueue().map_err(|unsent| (JobGone, Some(unsent))), + None, + ), + unused => (Err((JobGone, None)), unused), }; let held = if inner.scheduled == 0 { inner.held.take() @@ -287,7 +292,7 @@ impl JobCore { }; drop(inner); self.emitters.notify_all(); - (result, held) + (result, held, unused) } #[allow(dead_code)] // used by the Loom model diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 9fd52c26..8e138e3b 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -394,6 +394,7 @@ impl JobState { self: &Arc, at: u64, send: SendScheduled, + failures: Arc, track: impl FnOnce(u64, std::sync::Weak), ) -> Result<(), JobGone> { self.core.schedule()?; @@ -403,6 +404,7 @@ impl JobState { at, job: Arc::downgrade(self), send: Mutex::new(Some(send)), + failures, }); lock(&self.scheduled).push(emit.clone()); if self.is_cancelled() { @@ -739,16 +741,56 @@ impl Expirable for JobDeadline { } } +/// A value (user payload) whose drop is contained by the owner's failure +/// sink: it may be dropped on the deadline service thread. +pub(crate) struct Contained { + value: Option, + failures: Arc, +} + +impl Contained { + pub(crate) fn new(value: T, failures: Arc) -> Self { + Self { + value: Some(value), + failures, + } + } + pub(crate) fn into_inner(mut self) -> Option { + self.value.take() + } +} + +impl Drop for Contained { + fn drop(&mut self) { + if let Some(value) = self.value.take() { + self.failures.discard(value); + } + } +} + /// Enqueues a delayed emit's envelope; a refused envelope comes back erased, /// to be dropped outside locks. pub(crate) type SendScheduled = Box Result<(), Box> + Send>; /// One delayed emit (`JobCtx::emit_at`), serviced by the owner's deadline /// heap. Holds its message and emit credit until it fires or is discarded. +/// +/// Its message is user payload and is usually dropped on the process-wide +/// deadline service thread: every drop goes through the owner's failure +/// sink, which contains a panicking destructor (recorded as a cleanup +/// failure), so the emit still settles and the service keeps running. pub(crate) struct ScheduledEmit { at: u64, job: std::sync::Weak, send: Mutex>, + failures: Arc, +} + +impl Drop for ScheduledEmit { + fn drop(&mut self) { + let send = self.send.get_mut().unwrap_or_else(|p| p.into_inner()).take(); + self.failures.discard(send); + } } impl ScheduledEmit { @@ -759,24 +801,26 @@ impl ScheduledEmit { let Some(send) = lock(&self.send).take() else { return false; }; - // A discarded message (and its credit guard) drops at the end, - // outside the trace section and every lock. - let (deliver, _discarded) = if deliver { + // A discarded message (and its credit guard) drops below, outside the + // trace section and every lock, through the failure sink. + let (deliver, discarded) = if deliver { (Some(send), None) } else { (None, Some(send)) }; - let (result, held) = trace::atomic(|| { - let (result, held) = job.core.fire(deliver); + let (result, held, unused) = trace::atomic(|| { + let (result, held, unused) = job.core.fire(deliver); if result.is_ok() { crate::instrumentation::hooks::job_emitted(); job.record(|id| trace::JobEvent::new("Emit", id)); } - (result, held) + (result, held, unused) }); let delivered = result.is_ok(); - // A refused envelope (and its credit guard) drops here, unlocked. - drop(result); + // Whatever was not enqueued (the refused envelope, the uncalled send + // or the discarded one, each with its credit guard) drops here, + // unlocked and contained. + self.failures.discard((result, unused, discarded)); lock(&job.scheduled).retain(|emit| !std::ptr::eq(Arc::as_ptr(emit), self)); if let Some(held) = held { held(); @@ -905,7 +949,7 @@ mod tests { let _ = sent_tx.send(()); Ok(()) }); - let result = state.schedule_emit(u64::MAX - 1, send, |_, _| {}); + let result = state.schedule_emit(u64::MAX - 1, send, Arc::default(), |_, _| {}); let _ = done_tx.send((result, state.core.scheduled())); }); let (result, scheduled) = done_rx diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index 6d7260da..c08da3f2 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -215,12 +215,21 @@ impl Registry { /// The service visits: expire every due entry outside the lock, and /// return the next earliest deadline. + /// + /// Entries run framework code that can reach user code (a waker, a + /// payload destructor the entry did not route to its failure sink, the + /// last reference to a job's `done`). Each runs contained, so one + /// failing entry neither skips the others nor ends the shared service. pub(crate) fn service_deadlines(&self, now: u64) -> Option { let due = lock(&self.lists).deadlines.due(now); for item in due { - if item.pending_deadline().is_some_and(|d| d <= now) { - item.expire(now); - } + contain_entry(|| { + if item.pending_deadline().is_some_and(|d| d <= now) { + item.expire(now); + } + // Possibly the last reference: dropped inside the guard. + drop(item); + }); } lock(&self.lists).deadlines.earliest() } @@ -361,6 +370,18 @@ impl Registry { } } +/// Run one deadline-service step; a panic is logged and disposed of. +pub(crate) fn contain_entry(step: impl FnOnce()) { + if let Err(payload) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(step)) { + let message = crate::error::panic_message(&*payload); + crate::mailbox::dispose_panic(payload); + crate::actor_cleanup::log_safely(|| { + tracing::error!(event = "deadline_entry_failed", %message, + "a deadline entry panicked on the deadline service; servicing continues"); + }); + } +} + impl Drop for Registry { fn drop(&mut self) { // Every handle to the instance is gone; nothing can still answer. diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index e75f3352..d11e4ccd 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -150,7 +150,9 @@ where let (_, unsent) = self .state .notify(end_of(&outcome), || self.enqueue_terminal(outcome)); - drop(unsent); + // A refused terminal message (`done` and the outcome) may drop on + // the deadline service thread: contained. + self.ctx.shared.sender.failures().discard(unsent); } fn enqueue_terminal(&self, outcome: JobOutcome) -> Option + Send>> { @@ -182,15 +184,28 @@ where || self.enqueue_terminal(outcome.take()?), || { let run = self.clone(); - let outcome = outcome.take(); + // Dropped unrun (a deadline or a cancel overtook it) possibly + // on the deadline service thread: contained. + let outcome = outcome + .take() + .map(|outcome| crate::reply::jobs::Contained::new(outcome, self.ctx.shared.sender.failures())); Box::new(move || { - if let Some(outcome) = outcome { + if let Some(outcome) = outcome.and_then(|o| o.into_inner()) { run.notify(outcome); } }) }, ); - drop(unsent); + self.ctx.shared.sender.failures().discard(unsent); + } +} + +impl Drop for Run { + /// The last reference may go on the deadline service thread (a deadline + /// or a held result firing): `done` and its captures drop contained. + fn drop(&mut self) { + let done = self.done.get_mut().unwrap_or_else(|p| p.into_inner()).take(); + self.ctx.shared.sender.failures().discard(done); } } @@ -320,6 +335,7 @@ impl JobCtx { &self.state, spawned_rt::time::to_ticks(at), send, + self.ctx.shared.sender.failures(), |at, item| self.ctx.shared.replies.track(at, item), ) } diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index 6bc518fa..e74520b9 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -128,7 +128,9 @@ where let (_, unsent) = self .state .notify(end_of(&outcome), || self.enqueue_terminal(outcome)); - drop(unsent); + // A refused terminal message (`done` and the outcome) may drop on + // the deadline service thread: contained. + self.ctx.sender.failures().discard(unsent); } fn enqueue_terminal(&self, outcome: JobOutcome) -> Option> { @@ -164,15 +166,28 @@ where || self.enqueue_terminal(outcome.take()?), || { let run = self.clone(); - let outcome = outcome.take(); + // Dropped unrun (a deadline or a cancel overtook it) possibly + // on the deadline service thread: contained. + let outcome = outcome + .take() + .map(|outcome| crate::reply::jobs::Contained::new(outcome, self.ctx.sender.failures())); Box::new(move || { - if let Some(outcome) = outcome { + if let Some(outcome) = outcome.and_then(|o| o.into_inner()) { run.notify(outcome); } }) }, ); - drop(unsent); + self.ctx.sender.failures().discard(unsent); + } +} + +impl Drop for Run { + /// The last reference may go on the deadline service thread (a deadline + /// or a held result firing): `done` and its captures drop contained. + fn drop(&mut self) { + let done = self.done.get_mut().unwrap_or_else(|p| p.into_inner()).take(); + self.ctx.sender.failures().discard(done); } } @@ -276,6 +291,7 @@ impl JobCtx { &self.state, spawned_rt::time::to_ticks(at), send, + self.ctx.sender.failures(), |at, item| self.ctx.replies.track(at, item), ) } diff --git a/concurrency/tests/deadline_service.rs b/concurrency/tests/deadline_service.rs new file mode 100644 index 00000000..b9a32b40 --- /dev/null +++ b/concurrency/tests/deadline_service.rs @@ -0,0 +1,136 @@ +//! Review P0 (Codex 2): user payload dropped on the process-wide deadline +//! service thread must not end the service. A job's deadline takes its +//! terminal notification while the work runs; the job's scheduled emit is +//! then discarded at its instant on the service thread, and its message's +//! destructor panics. Another actor's request deadline must still fire. +//! +//! The service is process-wide, so the scenario runs in a child process. +#![cfg(all(feature = "async", not(spawned_sim)))] // native jobs are not simulated +use spawned_concurrency::{ + message::Message, + tasks::{Actor, ActorStart, Context, Handler, Instant, JobOptions, JobOutcome}, + ActorError, Deadline, PendingReply, ReplyTo, +}; +use std::time::Duration; +#[path = "support/drop_subprocess.rs"] +mod subprocess; + +/// A message whose destructor panics while armed. +struct Bomb(bool); +impl Message for Bomb { + type Result = (); +} +impl Drop for Bomb { + fn drop(&mut self) { + if self.0 { + panic!("scheduled emit destructor"); + } + } +} + +struct Done; +impl Message for Done { + type Result = (); +} + +/// Starts the job; parks requests (deferred, never answered). +#[derive(Default)] +struct Owner(Vec>); +impl Actor for Owner {} +impl Handler for Owner { + async fn handle(&mut self, mut msg: Bomb, _: PendingReply<'_, ()>, _: &Context) { + msg.0 = false; // delivered: not the scenario under test + } +} +impl Handler for Owner { + async fn handle(&mut self, _: Done, _: PendingReply<'_, ()>, _: &Context) {} +} +struct Park; +impl Message for Park { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: Park, reply: PendingReply<'_, ()>, _: &Context) { + self.0.push(reply.defer().unwrap()); + } +} + +#[test] +fn deadline_service_survives_a_panicking_payload_destructor() { + subprocess::run("deadline_service_child", "deadline service survived"); +} + +#[test] +#[ignore = "subprocess helper exercised by the enabled parent"] +fn deadline_service_child() { + if std::env::var_os("SPAWNED_MAILBOX_DROP_CHILD").is_none() { + return; + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let owner = Owner::default().start(); + owner + .__in_turn(|_, ctx| { + let opts = JobOptions::default().deadline_after(Duration::from_millis(20)); + ctx.pipe_blocking_with( + opts, + |job| { + let at = Instant::now() + Duration::from_millis(40); + job.emit_at_blocking(at, Bomb(true)).unwrap(); + std::thread::sleep(Duration::from_millis(150)); + }, + |_: JobOutcome<()>| Done, + ) + .unwrap(); + }) + .unwrap() + .await + .unwrap(); + // The deadline (20 ms) took the notification; the emit is discarded + // at 40 ms on the service thread and its destructor panics. + tokio::time::sleep(Duration::from_millis(100)).await; + + // Another actor's deadlines, serviced by the same thread. + let other = Owner::default().start(); + let parked = other + .request_raw_until(Park, Deadline::after(Duration::from_millis(50))) + .unwrap(); + match tokio::time::timeout(Duration::from_secs(5), parked).await { + Ok(Err(ActorError::RequestTimeout)) => {} + Ok(other) => panic!("unexpected outcome {other:?}"), + Err(_) => panic!("the deadline service stopped: a parked request never expired"), + } + // A job deadline still fires too. + let fired = other + .__in_turn(|_, ctx| { + let (tx, rx) = std::sync::mpsc::channel(); + let opts = JobOptions::default().deadline_after(Duration::from_millis(20)); + ctx.pipe_blocking_with( + opts, + |_| std::thread::sleep(Duration::from_millis(300)), + move |outcome: JobOutcome<()>| { + let _ = tx.send(outcome.into_result().is_err()); + Done + }, + ) + .unwrap(); + rx + }) + .unwrap() + .await + .unwrap(); + let deadline = tokio::task::spawn_blocking(move || fired.recv_timeout(Duration::from_secs(5))) + .await + .unwrap(); + assert_eq!(deadline, Ok(true), "a job deadline did not fire"); + other.context().stop(); + other.join().await; + owner.context().stop(); + owner.join().await; + }); + println!("deadline service survived"); +} diff --git a/concurrency/tests/job_core_loom.rs b/concurrency/tests/job_core_loom.rs index 34adf5a6..851fe07a 100644 --- a/concurrency/tests/job_core_loom.rs +++ b/concurrency/tests/job_core_loom.rs @@ -410,10 +410,12 @@ fn scheduled_emit_vs_finish_and_cancel() { let timer = { let (job, mailbox) = (job.clone(), mailbox.clone()); loom::thread::spawn(move || { - let (result, held) = job.core.fire(Some(|| { + let (result, held, unused) = job.core.fire(Some(|| { mailbox.push(Msg::Emit(0, 0)); Ok::<(), ()>(()) })); + // Not enqueued: handed back to drop outside the lock. + assert_eq!(result.is_err(), unused.is_some()); if result.is_err() { // Not enqueued: the credit guard returns it. job.core.return_emit_credit(); From 20642c78e7dbbb39c0b7371dcfc5508cf6beb6e2 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 11:38:19 -0300 Subject: [PATCH 106/208] fix(tasks): reserve a blocking job's worker and scope lease with its native slot Two gaps in where a tasks-mode blocking job runs, both fixed by reserving everything the launch needs together with the native slot, as threads mode already does: - Outside a Tokio runtime (and under the simulator) the job's OS thread was spawned only at launch, after the permit was granted and a `pipe_reply*` token consumed. A spawn failure was only logged: the caller got a `JobHandle` and `done` never ran (Codex 12, agy 6). The thread is now spawned and parked at reservation; a failure is `PipeErrorKind::SpawnFailed` from `reserve_blocking_job`, `pipe_blocking_with` and `pipe_reply_blocking_with` (which hands the token back). - A managed actor's blocking job ran on the group's blocking pool without a scope lease. Once the actor's bounded job join expired, the root released the last lease while the native work continued, so a supervisor saw the cleanup complete and started a replacement next to the old job (with `shutdown_timeout` above the 5 s join limit), and a shutdown could report completion with jobs running (Codex 7). The job now holds a lease until its body ends: cleanup stays incomplete (`owned cleanup stuck` at the supervisor's deadline) and ownership is kept. A closing scope refuses the reservation (`OwnerStopping`). Tests (native_jobs): - a_supervisor_does_not_replace_a_child_whose_blocking_job_still_runs (failed before: a replacement started at the 5 s join limit); - a_failed_job_thread_spawn_is_a_reservation_error (subprocess with RLIMIT_NPROC = 0; failed before: the reservation succeeded); - shutdown_waits_for_a_managed_blocking_job and its borrowed-runtime variant (coverage of the shutdown path; they pass before too, as the shutdown grace and the job join expire together). --- concurrency/src/system.rs | 5 + concurrency/src/tasks/jobs.rs | 125 ++++++++++++---- concurrency/tests/native_jobs.rs | 250 +++++++++++++++++++++++++++++++ 3 files changed, 350 insertions(+), 30 deletions(-) create mode 100644 concurrency/tests/native_jobs.rs diff --git a/concurrency/src/system.rs b/concurrency/src/system.rs index 3c8f1db7..43b41f2d 100644 --- a/concurrency/src/system.rs +++ b/concurrency/src/system.rs @@ -1175,6 +1175,11 @@ impl Scope { pub(crate) fn actor_lease(self: &Arc) -> Result { self.lease(true) } + /// A blocking job's lease, held until its native body ends: the scope + /// is not joined (cleanup incomplete) while the work still runs. + pub(crate) fn job_lease(self: &Arc) -> Result { + self.lease(false) + } /// Dispatch the actor root to its reserved worker or runtime shard. pub(crate) fn dispatch_actor(&self, future: F) -> Result<(), SystemError> where diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index d11e4ccd..f64604ca 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -224,10 +224,57 @@ enum Hold { } /// Native slot (`MAX_NATIVE_JOBS`) held by a blocking job. -struct NativeSlot(Context); +struct NativeSlot { + ctx: Context, + /// Where the job will run, reserved with the slot (`None` once used). + runner: Option, +} impl Drop for NativeSlot { fn drop(&mut self) { - self.0.shared.jobs.release_native(); + self.ctx.shared.jobs.release_native(); + } +} + +/// Where a blocking job's body runs, fixed when its native slot is reserved +/// so that launching it cannot fail (threads mode reserves its thread the +/// same way). +struct NativeRunner { + /// A managed actor's scope lease, held until the body physically ends: + /// the actor's cleanup is not complete (no supervisor replacement, no + /// complete system shutdown) while its native work still runs. + lease: Option, + pool: RunnerPool, +} + +enum RunnerPool { + /// Tokio's blocking pool: the managed actor's group runtime, or the + /// current one. + #[cfg(not(spawned_sim))] + Tokio(spawned_rt::tasks::Handle), + /// Outside a Tokio runtime, and under the simulator: an owned thread, + /// spawned at reservation and parked until the job starts (dropping the + /// sender unused lets it exit). + Thread(std::sync::mpsc::Sender>), +} + +impl NativeRunner { + /// Run `task`, then release the lease (after the body, and everything + /// it drops, ended). + fn run(self, task: impl FnOnce() + Send + 'static) { + let NativeRunner { lease, pool } = self; + let task = move || { + task(); + drop(lease); + }; + match pool { + #[cfg(not(spawned_sim))] + RunnerPool::Tokio(handle) => drop(handle.spawn_blocking(task)), + // The thread is parked on `recv` until this send (or the + // sender's drop); it cannot have exited. + RunnerPool::Thread(start) => { + let _ = start.send(Box::new(task)); + } + } } } @@ -530,11 +577,53 @@ impl Context { }) } + /// Reserve a native slot (`MAX_NATIVE_JOBS`) and where the job will run: + /// the blocking pool, or an owned thread spawned now (`SpawnFailed` + /// here, never at launch), plus a managed actor's scope lease + /// (`OwnerStopping` once its scope is closing). A failure releases the + /// slot. fn reserve_native(&self) -> Result, PipeErrorKind> { if !self.shared.jobs.reserve_native() { return Err(PipeErrorKind::NativeBudget); } - Ok(NativeSlot(self.clone())) + let mut slot = NativeSlot { + ctx: self.clone(), + runner: None, + }; + let lease = match &self.shared.managed { + Some(scope) => Some( + scope + .job_lease() + .map_err(|_| PipeErrorKind::OwnerStopping)?, + ), + None => None, + }; + #[cfg(not(spawned_sim))] + let handle = match &self.shared.managed { + Some(scope) => Some(scope.handle.clone()), + None => spawned_rt::tasks::Handle::try_current().ok(), + }; + #[cfg(spawned_sim)] + let handle: Option<()> = None; + let pool = match handle { + #[cfg(not(spawned_sim))] + Some(handle) => RunnerPool::Tokio(handle), + #[allow(unreachable_patterns)] + _ => { + let (start, started) = std::sync::mpsc::channel::>(); + std::thread::Builder::new() + .name("spawned-job".into()) + .spawn(move || { + if let Ok(task) = started.recv() { + task(); + } + }) + .map_err(PipeErrorKind::SpawnFailed)?; + RunnerPool::Thread(start) + } + }; + slot.runner = Some(NativeRunner { lease, pool }); + Ok(slot) } /// Reserve an unlinked job's admission (the owner is running, one @@ -654,37 +743,12 @@ impl Context { handle } - /// Run a blocking job's body on Tokio's blocking pool (the managed - /// actor's group runtime, or the current one). Outside a Tokio runtime, - /// and under the simulator, on an owned thread. - fn spawn_native(&self, task: impl FnOnce() + Send + 'static) { - #[cfg(not(spawned_sim))] - { - let handle = match &self.shared.managed { - Some(scope) => Some(scope.handle.clone()), - None => spawned_rt::tasks::Handle::try_current().ok(), - }; - if let Some(handle) = handle { - drop(handle.spawn_blocking(task)); - return; - } - } - if let Err(error) = std::thread::Builder::new() - .name("spawned-job".into()) - .spawn(task) - { - // The task (its native slot, credit and `done`) dropped with the - // failed spawn: the job ends without a terminal message. - tracing::error!(%error, actor = %self.id(), "could not spawn a native job thread"); - } - } - fn launch_blocking( &self, opts: JobOptions, state: Arc, hold: Hold, - native: NativeSlot, + mut native: NativeSlot, work: W, done: Done, ) -> JobHandle @@ -711,7 +775,8 @@ impl Context { state: state.clone(), ctx: self.clone(), }; - self.spawn_native(move || { + let runner = native.runner.take().expect("a native slot is reserved with its runner"); + runner.run(move || { let _ended = ended; let result = std::panic::catch_unwind(AssertUnwindSafe(|| work(&job))); // Execution ended: release the native slot now; the run returns diff --git a/concurrency/tests/native_jobs.rs b/concurrency/tests/native_jobs.rs new file mode 100644 index 00000000..fe22d607 --- /dev/null +++ b/concurrency/tests/native_jobs.rs @@ -0,0 +1,250 @@ +//! Tasks-mode blocking jobs: where their native body runs is reserved with +//! the native slot (so a failure is an error, never a lost job), and a +//! managed actor's job keeps its scope owned until the body ends. +#![cfg(all(feature = "async", not(spawned_sim)))] // native jobs are not simulated +use spawned_concurrency::{ + message::Message, + system::ActorSystem, + tasks::{Actor, Context, Handler, JobOptions, JobOutcome, PendingReply}, +}; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + mpsc, Arc, Mutex, +}; +use std::time::Duration; +#[cfg(target_os = "linux")] +#[path = "support/drop_subprocess.rs"] +mod subprocess; + +struct Done; +impl Message for Done { + type Result = (); +} + +/// Starts one blocking job in `started()` that ignores cancellation and +/// runs until the test releases it. +struct Gated { + gate: Mutex>>, + running: Arc, +} +impl Actor for Gated { + async fn started(&mut self, ctx: &Context) { + let gate = self.gate.lock().unwrap().take().unwrap(); + let running = self.running.clone(); + ctx.pipe_blocking_with( + JobOptions::default(), + move |_| { + running.store(true, Ordering::SeqCst); + let _ = gate.recv(); + running.store(false, Ordering::SeqCst); + }, + |_: JobOutcome<()>| Done, + ) + .unwrap(); + } +} +impl Handler for Gated { + async fn handle(&mut self, _: Done, _: PendingReply<'_, ()>, _: &Context) {} +} + +/// Review P1 (Codex 7): a managed actor's blocking job holds a scope lease +/// until its body ends, so a system shutdown is not complete (and a +/// supervisor does not see the cleanup done) while the native work still +/// runs. The job used to run on the blocking pool without a lease: once the +/// actor's bounded job join expired, shutdown reported completion with the +/// job still executing, on an owned group and on a borrowed runtime alike. +async fn shutdown_waits_for_the_blocking_job(borrowed: bool) { + let system = ActorSystem::new().unwrap(); + let group = if borrowed { + system + .async_group_on("borrowed-jobs", &tokio::runtime::Handle::current()) + .unwrap() + } else { + system.async_group("owned-jobs", 1).unwrap() + }; + let (release, gate) = mpsc::channel(); + let running = Arc::new(AtomicBool::new(false)); + let actor = group + .start(Gated { + gate: Mutex::new(Some(gate)), + running: running.clone(), + }) + .unwrap(); + actor.ready().await.unwrap(); + for _ in 0..500 { + if running.load(Ordering::SeqCst) { + break; + } + tokio::time::sleep(Duration::from_millis(2)).await; + } + assert!(running.load(Ordering::SeqCst), "the job never started"); + let report = system.shutdown(Duration::from_millis(300)).await; + assert!(running.load(Ordering::SeqCst)); + assert!( + !report.complete, + "shutdown completed while a managed blocking job still ran" + ); + release.send(()).unwrap(); + let report = report.wait(Duration::from_secs(5)).await; + assert!(report.complete, "shutdown did not complete after the job ended"); + assert!(!running.load(Ordering::SeqCst)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shutdown_waits_for_a_managed_blocking_job() { + shutdown_waits_for_the_blocking_job(false).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shutdown_on_a_borrowed_runtime_waits_for_a_managed_blocking_job() { + shutdown_waits_for_the_blocking_job(true).await; +} + +/// Review P1 (Codex 7): with a `shutdown_timeout` above the scope's default +/// job-join limit (5 s), the actor's root released its lease after that +/// limit while its blocking job still ran; the supervisor saw the cleanup +/// complete and started a replacement next to the old job's effects. The +/// job's lease keeps the old scope owned: no replacement until it ends. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_supervisor_does_not_replace_a_child_whose_blocking_job_still_runs() { + use spawned_concurrency::supervision::{ChildSpec, SupervisorConfig}; + use std::sync::atomic::AtomicUsize; + /// The first instance starts a gated job and exits (`Normal`; the + /// child is `Permanent`). + struct Child(Option); + impl Actor for Child { + async fn started(&mut self, ctx: &Context) { + if let Some(gated) = &mut self.0 { + let gate = gated.gate.lock().unwrap().take().unwrap(); + let running = gated.running.clone(); + ctx.pipe_blocking_with( + JobOptions::default(), + move |_| { + running.store(true, Ordering::SeqCst); + let _ = gate.recv(); + running.store(false, Ordering::SeqCst); + }, + |_: JobOutcome<()>| Done, + ) + .unwrap(); + // Exit after readiness (a startup exit is a startup failure). + ctx.send(Leave).unwrap(); + } + } + } + struct Leave; + impl Message for Leave { + type Result = (); + } + impl Handler for Child { + async fn handle(&mut self, _: Leave, _: PendingReply<'_, ()>, ctx: &Context) { + ctx.stop(); + } + } + impl Handler for Child { + async fn handle(&mut self, _: Done, _: PendingReply<'_, ()>, _: &Context) {} + } + let system = ActorSystem::new().unwrap(); + let group = system.async_group("supervised-jobs", 2).unwrap(); + let (release, gate) = mpsc::channel(); + let running = Arc::new(AtomicBool::new(false)); + let starts = Arc::new(AtomicUsize::new(0)); + let first = Arc::new(Mutex::new(Some(Gated { + gate: Mutex::new(Some(gate)), + running: running.clone(), + }))); + let mut builder = system.supervisor(SupervisorConfig::default()).unwrap(); + let factory_starts = starts.clone(); + builder + .add( + ChildSpec::new("gated", &group, move |_| { + factory_starts.fetch_add(1, Ordering::SeqCst); + let gated = first.lock().unwrap().take(); + async move { Ok::<_, String>(Child(gated)) } + }) + .shutdown_timeout(Duration::from_secs(10)), + ) + .unwrap(); + let _supervisor = builder.start().await.unwrap(); + // Past the scope's 5 s job-join limit, with the job still running. + tokio::time::sleep(Duration::from_millis(5600)).await; + assert!(running.load(Ordering::SeqCst), "the job ended early"); + assert_eq!( + starts.load(Ordering::SeqCst), + 1, + "a replacement started while the old blocking job still ran" + ); + release.send(()).unwrap(); + for _ in 0..1000 { + if starts.load(Ordering::SeqCst) == 2 { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + assert_eq!(starts.load(Ordering::SeqCst), 2, "no replacement after the job ended"); + let _ = system.shutdown(Duration::from_secs(5)).await; +} + +/// Review P1 (Codex 12, agy 6): outside a Tokio runtime a tasks actor's +/// blocking job runs on an owned thread. Its creation is part of the +/// reservation, so a failure is `SpawnFailed` (and `pipe_reply*` hands the +/// token back), as in threads mode. The thread used to be spawned after the +/// permit was granted and the token consumed: a failure was only logged, +/// the caller got a `JobHandle` and `done` never ran. +#[cfg(target_os = "linux")] +#[test] +fn a_failed_job_thread_spawn_is_a_reservation_error() { + subprocess::run("job_thread_spawn_failure_child", "job thread spawn failure reported"); +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "subprocess helper exercised by the enabled parent"] +fn job_thread_spawn_failure_child() { + use nix::sys::resource::{getrlimit, setrlimit, Resource}; + use spawned_concurrency::tasks::{ActorStart, Backend, PipeErrorKind}; + if std::env::var_os("SPAWNED_MAILBOX_DROP_CHILD").is_none() { + return; + } + struct Owner; + impl Actor for Owner {} + impl Handler for Owner { + async fn handle(&mut self, _: Done, _: PendingReply<'_, ()>, _: &Context) {} + } + // The actor runs on its own thread; this thread has no Tokio runtime. + let actor = Owner.start_with_backend(Backend::Thread); + let ctx = actor.context(); + assert!(tokio::runtime::Handle::try_current().is_err()); + let (soft, hard) = getrlimit(Resource::RLIMIT_NPROC).unwrap(); + // No new threads for this user: the job thread cannot be created. + setrlimit(Resource::RLIMIT_NPROC, 0, hard).unwrap(); + if let Ok(probe) = std::thread::Builder::new().spawn(|| ()) { + // Not enforced here (a privileged user): nothing to observe. + let _ = probe.join(); + setrlimit(Resource::RLIMIT_NPROC, soft, hard).unwrap(); + ctx.stop(); + println!("job thread spawn failure reported (RLIMIT_NPROC not enforced)"); + return; + } + let reserved = ctx.reserve_blocking_job(); + let piped = ctx.pipe_blocking_with(JobOptions::default(), |_| (), |_| Done); + setrlimit(Resource::RLIMIT_NPROC, soft, hard).unwrap(); + match reserved { + Err(error) => assert!( + matches!(error.kind, PipeErrorKind::SpawnFailed(_)), + "{error}" + ), + Ok(_) => panic!("the reservation succeeded without a job thread"), + } + assert!( + matches!(piped, Err(ref e) if matches!(e.kind, PipeErrorKind::SpawnFailed(_))), + "pipe_blocking_with returned {piped:?}" + ); + assert_eq!(ctx.deferred_outstanding(), 0, "a refused job kept its credit"); + // With threads available again, the same call runs the job. + let permit = ctx.reserve_blocking_job().expect("a job thread"); + drop(permit); + ctx.stop(); + std::thread::sleep(Duration::from_millis(50)); + println!("job thread spawn failure reported"); +} From 6d1bc4b39359789b0f10aa826b1783312e96af85 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:37:08 +0000 Subject: [PATCH 107/208] fix(supervision): charge the restart budget once per recovery round OTP charges one restart per restart action (add_restart), however many children a one_for_all or rest_for_one restart rebuilds. Spawned charged every rebuilt child, so a three-child OneForAll supervisor with max_restarts = 2 failed while rebuilding its first round (ROADMAP 5f, Codex review finding 11). The coordinator now takes one grant when it selects a OneForAll/RestForOne round, before stopping the cohort; if none is left the supervisor fails with RestartBudget naming the round's origin child. The round's rebuilds (origin and collateral) take no grant of their own. A failed replacement requests a new round, which is charged again (try_again_restart); a superseded round keeps its charge. OneForOne retries and retries before the tree is ready keep the per-child reserve-then-grant protocol. Tests: cohort_reconstruction_charges_one_restart_per_round (both strategies; replaces cohort_reconstruction_consumes_shared_factory_attempt_budget), policy_tests::cohort_round_takes_one_grant_for_all_rebuilt_children and supersession_charges_each_selected_round_once (replace the per-child contract tests); canary_feedback's cohort give-up case uses max_restarts = 0. Docs: max_restarts, semantics V5, guide, migration, otp-exit-signals design, research note, ROADMAP 5f, CHANGELOG. --- CHANGELOG.md | 11 ++ ROADMAP.md | 2 +- concurrency/src/supervision.rs | 99 +++++++++++++--- concurrency/tests/canary_feedback.rs | 13 ++- concurrency/tests/supervision_strategies.rs | 122 +++++++++++++------- docs/GUIDE.md | 10 +- docs/MIGRATION.md | 6 + docs/TESTING.md | 2 +- docs/design/otp-exit-signals.md | 8 +- docs/design/semantics.md | 22 ++-- docs/research/supervision-models.md | 7 +- 11 files changed, 218 insertions(+), 84 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c567343c..eda0d48f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ ## Unreleased +- **Breaking (0.8):** the restart budget counts restart actions, as OTP's + intensity does (ROADMAP rank 5f): a OneForAll or RestForOne recovery round + takes one `max_restarts` grant when the coordinator selects it, before the + cohort is stopped, however many children it rebuilds (v0.7 charged every + rebuilt child, so a six-child OneForAll round needed six grants). A failed + replacement inside a round requests a new round, charged again (OTP's + `try_again_restart`); a superseded round keeps its charge; OneForOne and + retries before the tree is ready still take one grant per child retry. An + exhausted budget fails the supervisor when the round would be selected, + naming the round's origin child. `max_restarts` documentation, semantics + V5, the guide and the migration notes follow. - **Breaking (0.8), in progress:** exit signals, startup and runtime-drop semantics as in OTP (ROADMAP ranks 5e, 5f, 5g), implementing [the design](docs/design/otp-exit-signals.md) (#240, stacked on rank 11). diff --git a/ROADMAP.md b/ROADMAP.md index a69b145a..a630ef20 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -535,7 +535,7 @@ Small, independent of the scheduler and lifecycle design, and visible to users. |---|---|---| | 5d | One semantics contract | A single `docs/design/semantics.md` stating every guarantee Spawned gives (message ordering and delivery, request outcomes, exit propagation through links and monitors, panic handling, restart and budget rules, shutdown order, timers, what `panic=abort` disables), each linked to the TLA+ invariant, test or example that checks it, and listing what is explicitly not guaranteed. Today these are spread across the guide, design contracts, TESTING.md and the models. New designs are reviewed against it. Erlang's semantics fit on a few pages; ours should too. | | 5e | Exit signals as in OTP (v0.8) | Align every exit-signal rule with OTP, as one design with a TLA+ model (the planned base-lifecycle model: links, monitors, `trap_exit`) and negative controls. (1) A direct `kill` stays untrappable, but the exit it causes propagates to linked actors as a distinct `Killed` reason (OTP `killed`), which they can trap. (2) `exit(Pid, normal)` semantics: a `Normal` exit signal from another actor is ignored by an actor that is not trapping exits; other reasons, including `Shutdown`, are delivered as trappable `Exit` messages to a trapping actor, and only `Kill` cannot be trapped. Today every direct stop ignores `trap_exit`. (3) A supervisor's `Shutdown` to a trapping child runs its normal termination path, as gen_server calls `terminate` on an exit from its parent. (4) Linking to or monitoring an actor that has already exited reports `NoProc` (OTP `noproc`) instead of its earlier exit reason. Breaking for exhaustive matches on `ExitReason`: pair with `#[non_exhaustive]` (as rank 11 does for `ActorError`), and update the semantics contract, migration notes and trace mapping. | -| 5f | Restart budget counts recovery rounds (OTP alignment, v0.8) | Today every rebuilt child counts against the budget, including the siblings OneForAll and RestForOne rebuild, so those strategies exhaust a budget N times faster than OTP users expect. Count one restart per recovery round, as OTP does. Also align initial +| 5f | Restart budget counts recovery rounds (OTP alignment, v0.8) | **Done in the 0.8 stack (#240):** a OneForAll/RestForOne recovery round takes one grant when the coordinator selects it (OTP's `add_restart` once per restart action); its rebuilds take none; a failed replacement requests a new, charged round. Before, every rebuilt child counted, so those strategies exhausted a budget N times faster than OTP users expect. Remaining: `Cohorts.tla` still abstracts budgets (`Supervision.tla`'s `BudgetBounded` covers OneForOne, where a child retry is the restart action); modelling the per-round charge with a per-child negative control is follow-up work. Also align initial startup: OTP aborts `start_link` on any child start error, while Spawned retries a Permanent or Transient child's failed factory, panic or startup timeout during the first start (semantics §11; found reviewing #233). Behaviour change: extend the `Supervision`/`Cohorts` TLA+ models and their Apalache inductive checks (`BudgetBounded`) to the new rule, with a negative control for counting per child, and update the Kani restart-window proofs and the guide. | diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs index 0cbb6eb2..8aa2ce3b 100644 --- a/concurrency/src/supervision.rs +++ b/concurrency/src/supervision.rs @@ -147,9 +147,11 @@ pub enum SupervisorStrategy { #[derive(Debug, Clone)] pub struct SupervisorConfig { pub strategy: SupervisorStrategy, - /// Supervisor-wide child retry grants per window, including collateral - /// OneForAll/RestForOne replacements. A six-child OneForAll round requires - /// six available grants. This retains v0.7's per-child accounting. + /// Supervisor-wide restarts per `restart_window` (OTP's intensity and + /// period). One restart is one restart action: a OneForOne child retry, + /// or one OneForAll/RestForOne recovery round, however many children + /// the round rebuilds. A failed replacement inside a round requests a + /// new round, which is charged again (OTP's `try_again_restart`). pub max_restarts: usize, pub restart_window: Duration, pub max_children: usize, @@ -986,6 +988,15 @@ impl Core { .flatten() }) } + /// One OneForAll/RestForOne recovery round is one restart action (OTP's + /// `add_restart` runs once per restart, before the cohort is rebuilt): + /// take its grant now. The round's controllers then rebuild without a + /// grant of their own. Pending per-child reservations (retries during + /// the initial start) keep their capacity. Returns whether it was taken. + fn charge_round(&self) -> bool { + let mut budget = system::lock(&self.budget); + budget.take(Instant::now()) + } /// Caller holds the admission gate and slot lock; charge atomically with /// generation admission, so placement failure or supersession refunds it. fn retry_under_gate(&self, reservation: &mut RetryReservation<'_>) -> bool { @@ -1629,6 +1640,12 @@ async fn run_recoveries(core: Arc) { } let round = trace::atomic(|| { let mut queue = system::lock(&core.recovery); + // The round is the restart action: charge it once, before any + // child of the cohort is stopped or rebuilt. + let origin = queue.pending.iter().position(|pending| *pending)?; + if !core.charge_round() { + return Some(Err(origin)); + } queue .select(core.config.strategy == SupervisorStrategy::OneForAll) .map(|(origin, start, id)| { @@ -1642,10 +1659,21 @@ async fn run_recoveries(core: Arc) { .round(id) .count(start + 1) }); - (start, meta) + Ok((start, meta)) }) }); - let Some((start, meta)) = round else { + if let Some(Err(origin)) = round { + // Exhausted: a concurrent stop wins; only an open supervisor fails. + let snapshot = children[origin].snapshot(); + let message = format!("restart budget exhausted by {}", snapshot.name); + core.fail_record_if_open_as( + snapshot_failure(snapshot, FailureKind::RestartBudget, message), + || None, + ); + core.log_failure_once(); + return; + } + let Some(Ok((start, meta))) = round else { if matches!( select( Box::pin(core.admission_closed.cancelled()), @@ -1767,6 +1795,8 @@ async fn run_child(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc>(), - vec![2, 2, 2, 2, 2, 1] + vec![2; 6] ); assert!(system.shutdown(limit).await.complete); } diff --git a/concurrency/tests/canary_feedback.rs b/concurrency/tests/canary_feedback.rs index b31db1ca..e1867c6e 100644 --- a/concurrency/tests/canary_feedback.rs +++ b/concurrency/tests/canary_feedback.rs @@ -252,13 +252,14 @@ async fn supervision_logs_restart_budget_exhaustion_and_give_up() { // Not simulated: the cohort case blocks a spawn_blocking helper. #[cfg(not(spawned_sim))] { - // The healthy collateral child takes the last retry grant. Its failing - // sibling must give up before a 60-second backoff, and lookups must report - // owner failure even while the healthy sibling's owned cleanup is pending. + // The round is the restart action: with no grant left the supervisor + // gives up when it would select the round, before the origin's + // 60-second backoff, and lookups must report owner failure even while + // the healthy sibling's owned cleanup is pending. let mut cohort = system .supervisor(SupervisorConfig { strategy: SupervisorStrategy::OneForAll, - max_restarts: 1, + max_restarts: 0, backoff: Backoff { initial: Duration::from_secs(60), maximum: Duration::from_secs(60), @@ -275,7 +276,7 @@ async fn supervision_logs_restart_budget_exhaustion_and_give_up() { ChildSpec::new("canary-healthy", &group, move |ctx| { let gate = gate.clone(); async move { - if ctx.generation() == 2 { + if ctx.generation() == 1 { let gate = gate.lock().unwrap().take().unwrap(); let (entered, arrival) = oneshot::channel(); ctx.spawn_blocking_helper(move || { @@ -317,7 +318,7 @@ async fn supervision_logs_restart_budget_exhaustion_and_give_up() { healthy.current(), Err(ChildUnavailable(ChildState::Failed)) )); - assert_eq!(healthy.snapshot().generation, 2); + assert_eq!(healthy.snapshot().generation, 1); release.send(()).unwrap(); spawned_rt::tasks::timeout(LIMIT, cohort.join()) .await diff --git a/concurrency/tests/supervision_strategies.rs b/concurrency/tests/supervision_strategies.rs index 03ecdd8c..df9bbf72 100644 --- a/concurrency/tests/supervision_strategies.rs +++ b/concurrency/tests/supervision_strategies.rs @@ -336,49 +336,91 @@ async fn failed_cohort_construction_rebuilds_the_whole_cohort_like_otp() { assert!(system.shutdown(LIMIT).await.complete); } +/// OTP charges one restart per restart action: a OneForAll or RestForOne +/// recovery round costs one grant however many children it rebuilds. With +/// three children and `max_restarts = 2`, two rounds rebuild the cohort and +/// the third failure exhausts the budget (per-child charging would have +/// failed the supervisor while rebuilding the first round). #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] -async fn cohort_reconstruction_consumes_shared_factory_attempt_budget() { - let system = ActorSystem::new().unwrap(); - let group = system.async_group("cohort-budget", 1).unwrap(); - let mut config = policy(SupervisorStrategy::OneForAll); - config.max_restarts = 2; - let mut tree = system.supervisor(config).unwrap(); - let mut children = Vec::new(); - let live = Arc::new((0..3).map(|_| AtomicUsize::new(0)).collect::>()); - let events = Arc::new(Mutex::new(Vec::new())); - for index in 0..3 { - let live = live.clone(); - let events = events.clone(); - children.push( - tree.add(ChildSpec::new( - format!("child-{index}"), - &group, - move |ctx| { - let live = live.clone(); - let events = events.clone(); - async move { - assert_eq!(live[index].fetch_add(1, Ordering::SeqCst), 0); - Ok::<_, String>(Probe { - index, - generation: ctx.generation() as usize, - value: 0, - live, - events, - }) - } - }, - )) - .unwrap(), +async fn cohort_reconstruction_charges_one_restart_per_round() { + for strategy in [ + SupervisorStrategy::OneForAll, + SupervisorStrategy::RestForOne, + ] { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("cohort-budget", 1).unwrap(); + let mut config = policy(strategy); + config.max_restarts = 2; + let mut tree = system.supervisor(config).unwrap(); + let mut children = Vec::new(); + let live = Arc::new((0..3).map(|_| AtomicUsize::new(0)).collect::>()); + let events = Arc::new(Mutex::new(Vec::new())); + for index in 0..3 { + let live = live.clone(); + let events = events.clone(); + children.push( + tree.add(ChildSpec::new( + format!("child-{index}"), + &group, + move |ctx| { + let live = live.clone(); + let events = events.clone(); + async move { + assert_eq!(live[index].fetch_add(1, Ordering::SeqCst), 0); + Ok::<_, String>(Probe { + index, + generation: ctx.generation() as usize, + value: 0, + live, + events, + }) + } + }, + )) + .unwrap(), + ); + } + let supervisor = tree.start().await.unwrap(); + // Two rounds, each rebuilding all three children (the first child + // is the origin, so RestForOne rebuilds the whole tail too). + for round in 1..=2u64 { + assert!(children[0].current().unwrap().request(Crash).await.is_err()); + for child in &children { + spawned_rt::tasks::timeout(LIMIT, child.wait_for_generation(round)) + .await + .unwrap() + .unwrap_or_else(|e| { + panic!( + "{strategy:?} round {round}: {e:?} {:?}", + supervisor.failure() + ) + }); + } + assert!( + supervisor.failure().is_none(), + "{strategy:?}: round {round} charged per rebuilt child: {:?}", + supervisor.failure() + ); + } + assert!(children + .iter() + .all(|child| child.snapshot().generation == 3)); + // The third failure is the third restart action in the window. + assert!(children[0].current().unwrap().request(Crash).await.is_err()); + spawned_rt::tasks::timeout(LIMIT, supervisor.join()) + .await + .unwrap(); + let failure = supervisor.failure().unwrap(); + assert!( + failure.contains("restart budget exhausted by child-0"), + "{strategy:?}: {failure}" ); + assert!(children + .iter() + .all(|child| child.snapshot().generation == 3)); + assert!(system.shutdown(LIMIT).await.complete); + assert!(live.iter().all(|n| n.load(Ordering::SeqCst) == 0)); } - let supervisor = tree.start().await.unwrap(); - assert!(children[0].current().unwrap().request(Crash).await.is_err()); - spawned_rt::tasks::timeout(LIMIT, supervisor.join()) - .await - .unwrap(); - assert!(supervisor.failure().unwrap().contains("budget exhausted")); - assert!(system.shutdown(LIMIT).await.complete); - assert!(live.iter().all(|n| n.load(Ordering::SeqCst) == 0)); } // Not simulated: children meet at a std::sync::Barrier, which needs OS threads. diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 8d74da07..a55c1205 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -1083,10 +1083,12 @@ Use an owned `async_group` for independent scheduling capacity and runtime lifetime guarantees. Ambient `.start()` remains unmanaged, including inside a managed handler; use `ctx.execution_group().start(...)` for owned child work. -The restart limit counts admitted child retry attempts across the supervisor, -including collateral OneForAll/RestForOne replacements, retaining v0.7 behavior. -A six-child OneForAll round needs six available grants. This differs from OTP -intensity accounting per recovery round; it is not changed silently in this patch. +The restart limit counts restart actions across the supervisor, as OTP's +intensity does: a OneForOne child retry is one, and a OneForAll/RestForOne +recovery round is one however many children it rebuilds (a six-child +OneForAll round needs one available grant). The round takes its grant when it +is selected, before the cohort is stopped; a replacement that fails inside the +round requests a new round, which takes another. Retry capacity is checked and reserved before a backoff. An exhausted window fails immediately, even if an older grant would expire during the proposed wait. diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 6f7a694a..9ce5fd00 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -12,6 +12,12 @@ table is the full list). What to edit: - **`match` on `ExitReason`**: add a wildcard arm (`#[non_exhaustive]`, new `Killed` and `NoProc`). +- **`max_restarts` with OneForAll/RestForOne counts rounds** (rank 5f): one + recovery round takes one grant, however many children it rebuilds (v0.7 + charged each rebuilt child). A cohort supervisor now tolerates more + failures for the same `max_restarts`; if you had raised it to cover + collateral rebuilds (for example `failures × children`), lower it to the + number of failures you mean to allow. OneForOne is unchanged. - **Kill from outside**: `ChildHandle::kill()` is the untrappable kill; the actor ends `Killed` and `stopped()` is skipped unless already running. `stop_with_reason(Kill)` stays an administrative stop ending `Kill`. diff --git a/docs/TESTING.md b/docs/TESTING.md index 686c98ce..1b1830ee 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -210,7 +210,7 @@ This generated suite uses zero backoff and a high, unexhausted restart budget; it does not claim budget-exhaustion or shutdown-during-backoff coverage. Those are controlled cases in `supervision` (`restart_budget_is_shared_between_siblings_and_exhaustion_closes_tree`, `shutdown_during_backoff_prevents_new_factory_invocation`) and -`supervision_strategies` (`cohort_reconstruction_consumes_shared_factory_attempt_budget`). +`supervision_strategies` (`cohort_reconstruction_charges_one_restart_per_round`). ```sh cargo test --locked -p spawned-concurrency --test supervisor_policies diff --git a/docs/design/otp-exit-signals.md b/docs/design/otp-exit-signals.md index 4af32c43..5db41da4 100644 --- a/docs/design/otp-exit-signals.md +++ b/docs/design/otp-exit-signals.md @@ -6,8 +6,12 @@ maintainer settled every open question (see [Decisions](#decisions)). - Covers ROADMAP rank 5e (exit signals), the startup part of rank 5f, rank 5g (runtime-dropped actors) and the supervisor `start()` ordering found by the - quality pass (#223). Counting the restart budget per recovery round (the - other half of 5f) is a separate change. + quality pass (#223). The other half of 5f, counting the restart budget per + recovery round, shipped in the same 0.8 stack: a OneForAll/RestForOne + round takes one grant when the coordinator selects it (OTP's + `add_restart` once per restart action), its rebuilds take none, and a + failed replacement requests a new round that is charged again (semantics + V5). - Target: v0.8. Changes L3, L4, L6, L7, X1-X7, V3, V7, V9 and §11 of the [semantics contract](semantics.md), and refines the death boundary of [deferred-replies.md §5](deferred-replies.md#5-owner-lifecycle) (rank 11, diff --git a/docs/design/semantics.md b/docs/design/semantics.md index 28d1ea81..62c5f369 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -422,14 +422,20 @@ Checked by: `Supervision!PublishedIsLive`, `Cohorts!PublishedIsLive`; [`supervisor_policies`][supervisor_policies] (request fences), [`supervision_recovery`][ex_supervision_recovery] example. -**V5. Restart budget.** At most `max_restarts` retry grants per rolling -`restart_window` (default 5 per 30 s), shared by the whole supervisor. Initial -construction is free; every later factory call counts, including failed -factories and collateral OneForAll/RestForOne children (unlike OTP, which counts -rounds). Capacity is reserved before backoff, refunded if the backoff is -cancelled or superseded, and stamped as a grant at admission. An exhausted -window or `max_restarts = 0` fails at once, without waiting for backoff. -Exhaustion fails the supervisor and stops its children in reverse order. +**V5. Restart budget.** At most `max_restarts` restart actions per rolling +`restart_window` (default 5 per 30 s), shared by the whole supervisor, as OTP's +intensity and period. Initial construction is free. A OneForOne child retry +(and any retry before the tree is ready) is one action: its capacity is +reserved before backoff, refunded if the backoff is cancelled or superseded, +and stamped as a grant at admission, so a failed factory or `started()` after +admission still counts. A OneForAll/RestForOne recovery round is one action, +however many children it rebuilds: the coordinator takes its grant when it +selects the round, before stopping the cohort (OTP's `add_restart`), and the +round's rebuilds take none. A failed replacement inside a round requests a new +round, charged again (`try_again_restart`); a superseded round keeps its +charge. An exhausted window or `max_restarts = 0` fails at once, without +waiting for backoff. Exhaustion fails the supervisor and stops its children in +reverse order. Checked by: `Supervision!BudgetBounded`; Kani `restart_window_admits_exactly_within_budget`; [`supervision::restart_budget_is_shared_between_siblings_and_exhaustion_closes_tree`][supervision], `disabled_retries_fail_without_waiting_for_a_long_backoff`, diff --git a/docs/research/supervision-models.md b/docs/research/supervision-models.md index 126c34c9..045c68a4 100644 --- a/docs/research/supervision-models.md +++ b/docs/research/supervision-models.md @@ -125,9 +125,10 @@ Read from `concurrency/src/supervision.rs`, `supervision/nested.rs` and the abnormal exit), `Temporary`: OTP's three types. Supervisor or system shutdown suppresses every mode. Stopping a Permanent child individually restarts it. - **Intensity:** `max_restarts` 5 per rolling `restart_window` 30 s, **shared by - the supervisor**, as in OTP. Initial construction is free; every later factory - call counts, including collateral reconstruction in a cohort and failed - factories. Exhaustion fails the whole supervisor and stops its children; + the supervisor**, as in OTP. Initial construction is free; a OneForOne retry + (including a failed factory) counts once, and since 0.8 a OneForAll or + RestForOne recovery round counts once however many children it rebuilds + (0.7 charged each collateral reconstruction). Exhaustion fails the whole supervisor and stops its children; `max_restarts = 0` fails at once. A nested subtree's failure "escalates through the adapter to its parent's restart policy" (`SupervisorSpec`). - **Backoff:** per child, capped exponential (`initial` 10 ms, `maximum` 5 s) From 9a00d319a71462bd748f9992284b83d9b6c74c96 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 11:39:25 -0300 Subject: [PATCH 108/208] style: rustfmt the review fixes --- concurrency/src/reply/job_core.rs | 7 +++---- concurrency/src/reply/jobs.rs | 14 +++++++++++--- concurrency/src/tasks/jobs.rs | 17 ++++++++++++----- concurrency/src/threads/jobs.rs | 12 ++++++++---- concurrency/tests/deadline_service.rs | 7 ++++--- concurrency/tests/deferred_api.rs | 17 ++++++++++++++--- concurrency/tests/deferred_replies.rs | 5 ++++- concurrency/tests/native_jobs.rs | 22 ++++++++++++++++++---- 8 files changed, 74 insertions(+), 27 deletions(-) diff --git a/concurrency/src/reply/job_core.rs b/concurrency/src/reply/job_core.rs index 87e71b45..986342ba 100644 --- a/concurrency/src/reply/job_core.rs +++ b/concurrency/src/reply/job_core.rs @@ -279,10 +279,9 @@ impl JobCore { debug_assert!(inner.scheduled > 0, "scheduled emit underflow"); inner.scheduled = inner.scheduled.saturating_sub(1); let (result, unused) = match enqueue { - Some(enqueue) if inner.open() => ( - enqueue().map_err(|unsent| (JobGone, Some(unsent))), - None, - ), + Some(enqueue) if inner.open() => { + (enqueue().map_err(|unsent| (JobGone, Some(unsent))), None) + } unused => (Err((JobGone, None)), unused), }; let held = if inner.scheduled == 0 { diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 8e138e3b..28b7021c 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -788,7 +788,11 @@ pub(crate) struct ScheduledEmit { impl Drop for ScheduledEmit { fn drop(&mut self) { - let send = self.send.get_mut().unwrap_or_else(|p| p.into_inner()).take(); + let send = self + .send + .get_mut() + .unwrap_or_else(|p| p.into_inner()) + .take(); self.failures.discard(send); } } @@ -944,7 +948,8 @@ mod tests { let (sent_tx, sent_rx) = std::sync::mpsc::channel::<()>(); let (done_tx, done_rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { - AFTER_SCHEDULE.with(|hook| *hook.borrow_mut() = Some(Box::new(move || cancelled.cancel()))); + AFTER_SCHEDULE + .with(|hook| *hook.borrow_mut() = Some(Box::new(move || cancelled.cancel()))); let send: SendScheduled = Box::new(move || { let _ = sent_tx.send(()); Ok(()) @@ -957,7 +962,10 @@ mod tests { .expect("schedule_emit deadlocked discarding its own emit"); assert_eq!(result, Err(JobGone)); assert_eq!(scheduled, 0, "the discarded emit is no longer scheduled"); - assert!(sent_rx.try_recv().is_err(), "a cancelled job's emit was sent"); + assert!( + sent_rx.try_recv().is_err(), + "a cancelled job's emit was sent" + ); } /// Review P1: a rejected reservation must not build a guard whose drop diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index f64604ca..40c16968 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -186,9 +186,9 @@ where let run = self.clone(); // Dropped unrun (a deadline or a cancel overtook it) possibly // on the deadline service thread: contained. - let outcome = outcome - .take() - .map(|outcome| crate::reply::jobs::Contained::new(outcome, self.ctx.shared.sender.failures())); + let outcome = outcome.take().map(|outcome| { + crate::reply::jobs::Contained::new(outcome, self.ctx.shared.sender.failures()) + }); Box::new(move || { if let Some(outcome) = outcome.and_then(|o| o.into_inner()) { run.notify(outcome); @@ -204,7 +204,11 @@ impl Drop for Run { /// The last reference may go on the deadline service thread (a deadline /// or a held result firing): `done` and its captures drop contained. fn drop(&mut self) { - let done = self.done.get_mut().unwrap_or_else(|p| p.into_inner()).take(); + let done = self + .done + .get_mut() + .unwrap_or_else(|p| p.into_inner()) + .take(); self.ctx.shared.sender.failures().discard(done); } } @@ -775,7 +779,10 @@ impl Context { state: state.clone(), ctx: self.clone(), }; - let runner = native.runner.take().expect("a native slot is reserved with its runner"); + let runner = native + .runner + .take() + .expect("a native slot is reserved with its runner"); runner.run(move || { let _ended = ended; let result = std::panic::catch_unwind(AssertUnwindSafe(|| work(&job))); diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index e74520b9..1bca196b 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -168,9 +168,9 @@ where let run = self.clone(); // Dropped unrun (a deadline or a cancel overtook it) possibly // on the deadline service thread: contained. - let outcome = outcome - .take() - .map(|outcome| crate::reply::jobs::Contained::new(outcome, self.ctx.sender.failures())); + let outcome = outcome.take().map(|outcome| { + crate::reply::jobs::Contained::new(outcome, self.ctx.sender.failures()) + }); Box::new(move || { if let Some(outcome) = outcome.and_then(|o| o.into_inner()) { run.notify(outcome); @@ -186,7 +186,11 @@ impl Drop for Run { /// The last reference may go on the deadline service thread (a deadline /// or a held result firing): `done` and its captures drop contained. fn drop(&mut self) { - let done = self.done.get_mut().unwrap_or_else(|p| p.into_inner()).take(); + let done = self + .done + .get_mut() + .unwrap_or_else(|p| p.into_inner()) + .take(); self.ctx.sender.failures().discard(done); } } diff --git a/concurrency/tests/deadline_service.rs b/concurrency/tests/deadline_service.rs index b9a32b40..00e76915 100644 --- a/concurrency/tests/deadline_service.rs +++ b/concurrency/tests/deadline_service.rs @@ -123,9 +123,10 @@ fn deadline_service_child() { .unwrap() .await .unwrap(); - let deadline = tokio::task::spawn_blocking(move || fired.recv_timeout(Duration::from_secs(5))) - .await - .unwrap(); + let deadline = + tokio::task::spawn_blocking(move || fired.recv_timeout(Duration::from_secs(5))) + .await + .unwrap(); assert_eq!(deadline, Ok(true), "a job deadline did not fire"); other.context().stop(); other.join().await; diff --git a/concurrency/tests/deferred_api.rs b/concurrency/tests/deferred_api.rs index 507d557b..6e61e7a9 100644 --- a/concurrency/tests/deferred_api.rs +++ b/concurrency/tests/deferred_api.rs @@ -554,7 +554,12 @@ mod caller_gone_vs_shortening_tasks { type Result = (); } impl Handler for Delegator { - async fn handle(&mut self, _: Ask, reply: PendingReply<'_, (bool, bool)>, ctx: &Context) { + async fn handle( + &mut self, + _: Ask, + reply: PendingReply<'_, (bool, bool)>, + ctx: &Context, + ) { let token = reply.defer().unwrap(); let opts = JobOptions { cancel_on_caller_gone: true, @@ -637,7 +642,10 @@ mod caller_gone_vs_shortening_tasks { assert!(!cancelled, "a shortened deadline cancelled the job"); // The caller leaving still cancels a blocking linked job. let left = actor.request_raw(Leave).unwrap(); - assert!(!actor.request(Probe).await.unwrap(), "cancelled before the caller left"); + assert!( + !actor.request(Probe).await.unwrap(), + "cancelled before the caller left" + ); drop(left); let mut cancelled = false; for _ in 0..500 { @@ -758,7 +766,10 @@ mod caller_gone_vs_shortening_threads { assert!(!cancelled, "a shortened deadline cancelled the job"); // The caller leaving still cancels a blocking linked job. let left = actor.request_raw(Leave).unwrap(); - assert!(!actor.request(Probe).unwrap(), "cancelled before the caller left"); + assert!( + !actor.request(Probe).unwrap(), + "cancelled before the caller left" + ); drop(left); let mut cancelled = false; for _ in 0..500 { diff --git a/concurrency/tests/deferred_replies.rs b/concurrency/tests/deferred_replies.rs index ca2190b4..bfedc5ed 100644 --- a/concurrency/tests/deferred_replies.rs +++ b/concurrency/tests/deferred_replies.rs @@ -467,7 +467,10 @@ mod tasks_mode { let result = actor .request_with_timeout(Shorten, Duration::from_secs(60)) .await; - assert!(matches!(result, Err(ActorError::RequestTimeout)), "{result:?}"); + assert!( + matches!(result, Err(ActorError::RequestTimeout)), + "{result:?}" + ); assert!( began.elapsed() < Duration::from_secs(5), "the caller slept until the old deadline: {:?}", diff --git a/concurrency/tests/native_jobs.rs b/concurrency/tests/native_jobs.rs index fe22d607..ca067a17 100644 --- a/concurrency/tests/native_jobs.rs +++ b/concurrency/tests/native_jobs.rs @@ -86,7 +86,10 @@ async fn shutdown_waits_for_the_blocking_job(borrowed: bool) { ); release.send(()).unwrap(); let report = report.wait(Duration::from_secs(5)).await; - assert!(report.complete, "shutdown did not complete after the job ended"); + assert!( + report.complete, + "shutdown did not complete after the job ended" + ); assert!(!running.load(Ordering::SeqCst)); } @@ -181,7 +184,11 @@ async fn a_supervisor_does_not_replace_a_child_whose_blocking_job_still_runs() { } tokio::time::sleep(Duration::from_millis(5)).await; } - assert_eq!(starts.load(Ordering::SeqCst), 2, "no replacement after the job ended"); + assert_eq!( + starts.load(Ordering::SeqCst), + 2, + "no replacement after the job ended" + ); let _ = system.shutdown(Duration::from_secs(5)).await; } @@ -194,7 +201,10 @@ async fn a_supervisor_does_not_replace_a_child_whose_blocking_job_still_runs() { #[cfg(target_os = "linux")] #[test] fn a_failed_job_thread_spawn_is_a_reservation_error() { - subprocess::run("job_thread_spawn_failure_child", "job thread spawn failure reported"); + subprocess::run( + "job_thread_spawn_failure_child", + "job thread spawn failure reported", + ); } #[cfg(target_os = "linux")] @@ -240,7 +250,11 @@ fn job_thread_spawn_failure_child() { matches!(piped, Err(ref e) if matches!(e.kind, PipeErrorKind::SpawnFailed(_))), "pipe_blocking_with returned {piped:?}" ); - assert_eq!(ctx.deferred_outstanding(), 0, "a refused job kept its credit"); + assert_eq!( + ctx.deferred_outstanding(), + 0, + "a refused job kept its credit" + ); // With threads available again, the same call runs the job. let permit = ctx.reserve_blocking_job().expect("a job thread"); drop(permit); From f0013b61c4454ee4818a51699e6ab1284024f789 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 11:42:40 -0300 Subject: [PATCH 109/208] fix(jobs): admit a job in one step with its owner's exit closing admission `start_job` checked `is_exiting()` and `JobSetCore::admit` later listed the job; a cloned context (or `pipe_with_ctx`, whose `work` closure is user code in between) could pass the check, let the owner exit, drain `cancel_all` and even finish its join, and then admit a job nobody would cancel or join: blocking work ran uncancelled after the owner's death. Threads mode had the same check/use window. `cancel_all` now closes admission under the job list's lock, and `admit` checks the gate, lists the job and counts it as running in one step under that lock (refusing with `false`). Admission happens after every reversible reservation and before anything irreversible: a refused `pipe_reply*` hands its token back (`OwnerStopping`), a refused permit returns a cancelled handle without running anything. This is the atomicity `DeferredJobs.tla`'s `Start` (guarded by `alive`) assumes. Tests: native_jobs::a_job_admitted_after_its_owner_exited_never_runs (the owner dies while `pipe_with_ctx` builds the work; failed before: the job ran uncancelled); Loom job_core_loom::admission_vs_owner_exit. --- concurrency/src/reply/job_core.rs | 66 ++++++++++++++++++++++-------- concurrency/src/reply/jobs.rs | 18 +++++--- concurrency/src/tasks/jobs.rs | 49 ++++++++++++++++++---- concurrency/src/threads/jobs.rs | 28 ++++++++++--- concurrency/tests/job_core_loom.rs | 39 ++++++++++++++++-- concurrency/tests/native_jobs.rs | 47 +++++++++++++++++++++ 6 files changed, 206 insertions(+), 41 deletions(-) diff --git a/concurrency/src/reply/job_core.rs b/concurrency/src/reply/job_core.rs index 986342ba..b79071da 100644 --- a/concurrency/src/reply/job_core.rs +++ b/concurrency/src/reply/job_core.rs @@ -323,9 +323,18 @@ pub(crate) trait Job { /// two above it). const COMPACT_AT: usize = 32; +/// The job list and the admission gate, under one lock: admission checks +/// the gate and lists the job in one step, so an admitted job is always in +/// the list the owner's exit takes and cancels. +struct JobList { + jobs: Vec>, + /// The owner's exit closed admission (`cancel_all`). + closed: bool, +} + /// An owner instance's jobs: key map, running count, native budget. pub(crate) struct JobSetCore { - jobs: Mutex>>, + jobs: Mutex>, keys: Mutex>>, running: Mutex, idle: Condvar, @@ -336,7 +345,10 @@ pub(crate) struct JobSetCore { impl JobSetCore { pub(crate) fn new(max_native: usize) -> Self { Self { - jobs: Mutex::new(Vec::new()), + jobs: Mutex::new(JobList { + jobs: Vec::new(), + closed: false, + }), keys: Mutex::new(HashMap::new()), running: Mutex::new(0), idle: Condvar::new(), @@ -364,18 +376,24 @@ impl JobSetCore { self.native.load(Ordering::Acquire) } - /// Admit a job: replace (cancel) the previous one with the same key, - /// whatever its notification state, and count it as running. The - /// replaced job and the references compaction upgraded are dropped after - /// the locks are released. - pub(crate) fn admit(&self, state: &Arc, key: Option) { - let previous = key.and_then(|key| lock(&self.keys).insert(key, state.clone())); - if let Some(previous) = &previous { - previous.cancel(); - } + /// Admit a job: refused (`false`) once the owner's exit closed + /// admission. Otherwise list it, count it as running and replace + /// (cancel) the previous job with the same key, whatever its + /// notification state. The check, the listing and the count are one + /// step under the list's lock, the same lock `cancel_all` closes and + /// drains under, so no admitted job escapes the owner's cancellation and + /// the owner's join counts it. The replaced job and the references + /// compaction upgraded are cancelled and dropped after the locks are + /// released. + pub(crate) fn admit(&self, state: &Arc, key: Option) -> bool { let mut upgraded: Vec> = Vec::new(); - { - let mut jobs = lock(&self.jobs); + let previous = { + let mut list = lock(&self.jobs); + if list.closed { + return false; + } + let previous = key.and_then(|key| lock(&self.keys).insert(key, state.clone())); + let jobs = &mut list.jobs; if jobs.len() >= COMPACT_AT && jobs.len().is_power_of_two() { jobs.retain(|job| match job.upgrade() { Some(job) => { @@ -387,10 +405,15 @@ impl JobSetCore { }); } jobs.push(Arc::downgrade(state)); + *lock(&self.running) += 1; + previous + }; + if let Some(previous) = &previous { + previous.cancel(); } - *lock(&self.running) += 1; drop(upgraded); drop(previous); + true } /// A job's execution ended. Returns whether the set became idle (the @@ -415,11 +438,18 @@ impl JobSetCore { *lock(&self.running) } - /// Owner exit: cancel every job (flag set, blocked emitters woken). The - /// key map's references and the upgraded jobs drop outside the locks. + /// Owner exit: close admission and cancel every job (flag set, blocked + /// emitters woken). The key map's references and the upgraded jobs drop + /// outside the locks. pub(crate) fn cancel_all(&self) { - let jobs = std::mem::take(&mut *lock(&self.jobs)); - let keys = std::mem::take(&mut *lock(&self.keys)); + let (jobs, keys) = { + let mut list = lock(&self.jobs); + list.closed = true; + ( + std::mem::take(&mut list.jobs), + std::mem::take(&mut *lock(&self.keys)), + ) + }; for job in jobs.iter().filter_map(Weak::upgrade) { job.cancel(); } diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 28b7021c..a5afdd53 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -609,18 +609,24 @@ impl JobSet { } /// Admit a job: replace (cancel) the previous one with the same key and - /// count it as running (the model's `Start`). - pub(crate) fn admit(&self, state: &Arc, key: Option, deadline: bool) { + /// count it as running (the model's `Start`). Refused (`false`, nothing + /// recorded) once the owner's exit closed admission: the caller must not + /// run it. + #[must_use] + pub(crate) fn admit(&self, state: &Arc, key: Option, deadline: bool) -> bool { trace::atomic(|| { let keyed = key.is_some(); - self.core.admit(state, key); + if !self.core.admit(state, key) { + return false; + } crate::instrumentation::hooks::job_started(); state.record(|job| trace::JobEvent { keyed, deadline, ..trace::JobEvent::new("Start", job) }); - }); + true + }) } /// A job's execution ended. @@ -990,10 +996,10 @@ mod tests { fn replacement_cancels_a_finished_keyed_job() { let set = JobSet::new(ActorId::next(), 1); let old = JobState::new(set.owner(), 1); - set.admit(&old, Some(JobKey::new("k")), false); + assert!(set.admit(&old, Some(JobKey::new("k")), false)); set.ended(&old); let new = JobState::new(set.owner(), 1); - set.admit(&new, Some(JobKey::new("k")), false); + assert!(set.admit(&new, Some(JobKey::new("k")), false)); assert!(old.is_cancelled(), "the finished job escaped replacement"); assert!(!new.is_cancelled()); } diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index 40c16968..823abc1e 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -484,6 +484,9 @@ impl JobPermit { let Some(state) = ctx.start_job(&opts) else { return ctx.refused_job(&opts, (work, done)); }; + if !ctx.admit_job(&state, &opts) { + return ctx.refuse(state, (work, done, credit)); + } ctx.launch_async::( opts, state, @@ -513,6 +516,11 @@ impl JobPermit { ctx: ctx.clone(), }; let work = work(job); + // Admitted after `work` was built (user code): an owner that exited + // meanwhile refuses the job, which then never runs. + if !ctx.admit_job(&state, &opts) { + return ctx.refuse(state, (work, done, credit)); + } ctx.launch_async::( opts, state, @@ -544,6 +552,9 @@ impl BlockingJobPermit { let Some(state) = ctx.start_job(&opts) else { return ctx.refused_job(&opts, (work, done, native)); }; + if !ctx.admit_job(&state, &opts) { + return ctx.refuse(state, (work, done, native, credit)); + } ctx.launch_blocking::( opts, state, @@ -564,12 +575,27 @@ impl Context { /// A permit used after the owner began exiting: nothing runs, the parts /// drop here (outside locks), and the handle reports cancelled. fn refused_job

(&self, opts: &JobOptions, parts: P) -> JobHandle { + self.refuse(JobState::new(self.id(), opts.emit_credits), parts) + } + + /// `state`'s admission was refused: as [`refused_job`](Self::refused_job). + fn refuse

(&self, state: Arc, parts: P) -> JobHandle { drop(parts); - let state = JobState::new(self.id(), opts.emit_credits); state.never_started(); JobHandle::new(state) } + /// Admit `state` into the owner's job set (the model's `Start`), after + /// every reversible reservation and before anything that cannot be + /// undone: refused once the owner's exit closed admission, in one step + /// with that close, so an admitted job is always cancelled (and joined) + /// by the owner's exit. + fn admit_job(&self, state: &Arc, opts: &JobOptions) -> bool { + self.shared + .jobs + .admit(state, opts.key.clone(), opts.deadline.is_some()) + } + fn reserve_credit(&self) -> Result { if self.shared.replies.is_exiting() { return Err(PipeError { @@ -705,9 +731,6 @@ impl Context { R: Send + 'static, W: Future + Send + 'static, { - self.shared - .jobs - .admit(&state, opts.key.clone(), opts.deadline.is_some()); let run = Arc::new(Run { state: state.clone(), ctx: self.clone(), @@ -763,9 +786,6 @@ impl Context { R: Send + 'static, W: FnOnce(&JobCtx) -> T + Send + 'static, { - self.shared - .jobs - .admit(&state, opts.key.clone(), opts.deadline.is_some()); let run = Arc::new(Run { state: state.clone(), ctx: self.clone(), @@ -937,6 +957,14 @@ impl Context { let (state, slot, token) = self.link_reply(&opts, token)?; let watch = opts.cancel_on_caller_gone.then(|| slot.clone()); let hold = Hold::Linked(LinkedJob(slot)); + if !self.admit_job(&state, &opts) { + // Dropping the hold detaches the linked job from the slot. + drop((hold, watch)); + return Err(PipeReplyError::Failed { + token, + kind: PipeErrorKind::OwnerStopping, + }); + } Ok(self.launch_async(opts, state, hold, work, linked_done(token, done), watch)) } @@ -967,6 +995,13 @@ impl Context { return Err(PipeReplyError::Failed { token, kind }); } }; + if !self.admit_job(&state, &opts) { + drop((hold, native)); + return Err(PipeReplyError::Failed { + token, + kind: PipeErrorKind::OwnerStopping, + }); + } if opts.cancel_on_caller_gone { crate::reply::jobs::cancel_when_closed(&slot, &state); } diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index 1bca196b..44548d49 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -370,13 +370,14 @@ impl BlockingJobPermit { credit, worker, } = self; - if ctx.replies.is_exiting() { - drop((work, done, worker)); - let state = JobState::new(ctx.id(), opts.emit_credits); + let state = JobState::new(ctx.id(), opts.emit_credits); + // Refused once the owner began exiting, in one step with the exit's + // close of admission (see `admit_job`). + if ctx.replies.is_exiting() || !ctx.admit_job(&state, &opts) { + drop((work, done, worker, credit)); state.never_started(); return JobHandle::new(state); } - let state = JobState::new(ctx.id(), opts.emit_credits); ctx.launch_blocking::( opts, state, @@ -389,6 +390,16 @@ impl BlockingJobPermit { } impl Context { + /// Admit `state` into the owner's job set (the model's `Start`), after + /// every reversible reservation and before anything that cannot be + /// undone: refused once the owner's exit closed admission, in one step + /// with that close, so an admitted job is always cancelled (and joined) + /// by the owner's exit. + fn admit_job(&self, state: &Arc, opts: &JobOptions) -> bool { + self.jobs + .admit(state, opts.key.clone(), opts.deadline.is_some()) + } + fn reserve_credit(&self) -> Result { if self.replies.is_exiting() { return Err(PipeErrorKind::OwnerStopping); @@ -481,8 +492,6 @@ impl Context { R: Send + 'static, W: FnOnce(&JobCtx) -> T + Send + 'static, { - self.jobs - .admit(&state, opts.key.clone(), opts.deadline.is_some()); let run = Arc::new(Run { state: state.clone(), ctx: self.clone(), @@ -566,6 +575,13 @@ impl Context { return Err(PipeReplyError::Failed { token, kind }); } }; + if !self.admit_job(&state, &opts) { + drop((hold, worker)); + return Err(PipeReplyError::Failed { + token, + kind: PipeErrorKind::OwnerStopping, + }); + } if opts.cancel_on_caller_gone { crate::reply::jobs::cancel_when_closed(&slot, &state); } diff --git a/concurrency/tests/job_core_loom.rs b/concurrency/tests/job_core_loom.rs index 851fe07a..fc805703 100644 --- a/concurrency/tests/job_core_loom.rs +++ b/concurrency/tests/job_core_loom.rs @@ -219,7 +219,7 @@ fn admission_rollback_vs_job_end() { // A running job holds the credit and the native slot. assert!(pool.reserve(1) && set.reserve_native()); let running = TestJob::new(0, 1, None, &destroyed); - set.admit(&running, None); + assert!(set.admit(&running, None)); holding.fetch_add(1, Ordering::AcqRel); let ender = { @@ -259,7 +259,7 @@ fn admission_rollback_vs_job_end() { "native bound exceeded" ); let job = TestJob::new(1, 1, None, &destroyed); - set.admit(&job, None); + assert!(set.admit(&job, None)); second = Some(job); } else { pool.release(1); @@ -289,7 +289,7 @@ fn completed_key_replacement() { let mailbox = Arc::new(Mailbox::default()); let destroyed = Arc::new(AtomicUsize::new(0)); let old = TestJob::new(0, 1, None, &destroyed); - set.admit(&old, Some(7)); + assert!(set.admit(&old, Some(7))); let worker = { let (set, mailbox, old) = (set.clone(), mailbox.clone(), old.clone()); loom::thread::spawn(move || { @@ -301,7 +301,7 @@ fn completed_key_replacement() { }) }; let new = TestJob::new(1, 1, None, &destroyed); - set.admit(&new, Some(7)); + assert!(set.admit(&new, Some(7))); assert!( old.core.is_cancelled(), "the replaced job was not cancelled" @@ -321,6 +321,37 @@ fn completed_key_replacement() { }); } +/// Admission races the owner's exit (review P1, Codex 6): `admit` checks +/// the gate and lists the job in the step `cancel_all` closes and drains +/// under, so a job is either refused or listed in time to be cancelled, and +/// the running count the owner's join waits for includes it. +#[test] +fn admission_vs_owner_exit() { + model(|| { + let set = Arc::new(Set::new(1)); + let destroyed = Arc::new(AtomicUsize::new(0)); + let job = TestJob::new(0, 1, None, &destroyed); + let admitter = { + let (set, job) = (set.clone(), job.clone()); + loom::thread::spawn(move || set.admit(&job, Some(1))) + }; + set.cancel_all(); + if admitter.join().unwrap() { + assert!( + job.core.is_cancelled(), + "an admitted job escaped the owner's exit" + ); + assert_eq!(set.running(), 1, "the owner's join misses the job"); + set.ended(&job); + } else { + assert_eq!(set.running(), 0); + } + let late = TestJob::new(1, 1, None, &destroyed); + assert!(!set.admit(&late, None), "admission reopened after exit"); + assert_eq!(set.running(), 0); + }); +} + /// An emitter (one credit, two emits) races the terminal notification and, /// optionally, a cancel. No emit is enqueued after the terminal message, a /// blocked emitter is always released, and every credit comes back. diff --git a/concurrency/tests/native_jobs.rs b/concurrency/tests/native_jobs.rs index ca067a17..b895508d 100644 --- a/concurrency/tests/native_jobs.rs +++ b/concurrency/tests/native_jobs.rs @@ -192,6 +192,53 @@ async fn a_supervisor_does_not_replace_a_child_whose_blocking_job_still_runs() { let _ = system.shutdown(Duration::from_secs(5)).await; } +/// Review P1 (Codex 6): job admission is one step with the owner's exit. +/// `pipe_with_ctx` builds the work (user code) between its exit check and +/// the admission; an owner that exits meanwhile refuses the job, so it +/// never runs. It used to be admitted into the job set the exit had already +/// drained: uncancelled work outliving its owner. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_job_admitted_after_its_owner_exited_never_runs() { + use spawned_concurrency::tasks::ActorStart; + struct Owner; + impl Actor for Owner {} + impl Handler for Owner { + async fn handle(&mut self, _: Done, _: PendingReply<'_, ()>, _: &Context) {} + } + let actor = Owner.start(); + let ctx = actor.context(); + let owner = actor.clone(); + let ran = Arc::new(AtomicBool::new(false)); + let work_ran = ran.clone(); + let handle = tokio::task::spawn_blocking(move || { + ctx.pipe_with_ctx( + JobOptions::default(), + |job| { + // While the work is built, the owner exits and dies. + owner.context().stop(); + while owner.exit_reason().is_none() { + std::thread::sleep(Duration::from_millis(1)); + } + async move { + work_ran.store(true, Ordering::SeqCst); + job.cancelled().await; + } + }, + |_: JobOutcome<()>| Done, + ) + .unwrap() + }) + .await + .unwrap(); + assert!( + handle.is_cancelled(), + "a job admitted after its owner's exit escaped the exit's cancellation" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!ran.load(Ordering::SeqCst), "a refused job ran"); + actor.join().await; +} + /// Review P1 (Codex 12, agy 6): outside a Tokio runtime a tasks actor's /// blocking job runs on an owned thread. Its creation is part of the /// reservation, so a failure is `SpawnFailed` (and `pipe_reply*` hands the From f87645c7f654e3433f212ae2a0f9b20e69716706 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:02:28 -0300 Subject: [PATCH 110/208] fix: one death boundary: answer every caller and fence replies, signals and jobs before destruction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The death point D was split across the run loops and drop guards, and user destructors ran inside it: - Queued requests were answered while the mailbox was dropped, envelope by envelope, each payload before its own request: a blocking payload destructor left its caller and every later queued caller unanswered (Codex 1, P0). - A deferred token's `reply()` was admitted until the registry closed the slot, after D sealed admission: a reply could succeed after `NoProc` was observable (Codex 4). - When the runtime dropped the actor, its state (never-polled: the whole startup payload) was destroyed before D, and the registry closed after publication (Codex 5). - `ctx.exit` from a cloned context checked the sender was alive and then delivered: the sender's death notice could overtake the signal at a trapping target (Codex 10). `ActorCleanup` is now the single boundary, whatever drives the death (run loop, `ExitOnDrop` for a dropped running actor, `ActorStartup` for a never-polled one): exit pending for dropped replies; job admission closed and jobs cancelled; outgoing replies and exit signals fenced (registry gates: in-flight ones finish, later ones are refused, the reply as `Delivery::OwnerExited` with its slot left to the close); then, in one trace section, the final reason with the link table closed and admission sealed; then every deferred and exit-pending slot closed. The mode then discharges the mailbox (closes it, answers every queued request, including the threads-mode envelope received after the stop) and only then destroys queued payloads and the state; publication stays last. No user code runs before the answers. `CloseOnDrop` is gone. Models: `Lifecycle.tla`'s `Death` (and `DeferredReplies.tla`'s) are atomic; the code's answers follow the seal but run no user code and nothing in between depends on them, so they commute to it: documented in the model and in deferred-replies.md §5 (no model change). `ExitSignal` requiring a live sender and `Start` requiring `alive` now hold atomically in the code. Tests (all failed before): - death_boundary: queued_requests_are_answered_before_payload_destructors (tasks and threads), a_never_polled_actor_answers_before_its_state_is_destroyed, a_dropped_running_actor_answers_before_its_state_is_destroyed; - failpoints: no_reply_succeeds_after_noproc_is_observable (new site actor.at_death), an_exit_signal_is_not_overtaken_by_its_senders_death_notice (new site actor.exit_before_deliver). --- CHANGELOG.md | 10 ++ concurrency/src/actor_cleanup.rs | 109 ++++++++---- concurrency/src/failpoints.rs | 17 ++ concurrency/src/mailbox.rs | 48 ++++- concurrency/src/reply/mod.rs | 10 +- concurrency/src/reply/registry.rs | 58 +++--- concurrency/src/reply/slot.rs | 23 ++- concurrency/src/tasks/actor.rs | 139 ++++++++++----- concurrency/src/threads/actor.rs | 117 +++++++++--- concurrency/tests/death_boundary.rs | 264 ++++++++++++++++++++++++++++ concurrency/tests/failpoints.rs | 158 +++++++++++++++++ docs/TESTING.md | 2 + docs/design/deferred-replies.md | 27 ++- docs/design/semantics.md | 18 +- verification/tla/Lifecycle.tla | 5 + 15 files changed, 856 insertions(+), 149 deletions(-) create mode 100644 concurrency/tests/death_boundary.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index eda0d48f..75ac5fbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,16 @@ exhausted budget fails the supervisor when the round would be selected, naming the round's origin child. `max_restarts` documentation, semantics V5, the guide and the migration notes follow. +- The death point is one boundary, however the actor dies (run loop or + runtime drop): job admission closes, outgoing replies and `ctx.exit` + signals are fenced, the reason is sealed, and every deferred token and + queued request is answered `ActorStopped(final reason)` before any queued + payload or state destructor runs and before death is published. A + `ReplyTo::reply` once `NoProc` is observable returns + `Delivery::OwnerExited` (the caller gets the final reason); a signal a + cloned context sends before its actor's death reaches the target before + the death notice. Previously a blocking payload destructor, or a runtime + dropping the actor, could leave callers unanswered while it ran. - **Breaking (0.8), in progress:** exit signals, startup and runtime-drop semantics as in OTP (ROADMAP ranks 5e, 5f, 5g), implementing [the design](docs/design/otp-exit-signals.md) (#240, stacked on rank 11). diff --git a/concurrency/src/actor_cleanup.rs b/concurrency/src/actor_cleanup.rs index 4343abf3..b4bc2adb 100644 --- a/concurrency/src/actor_cleanup.rs +++ b/concurrency/src/actor_cleanup.rs @@ -36,6 +36,14 @@ pub(crate) fn log_stopped_skipped( /// Constructed before dispatch, so even a never-polled actor has cleanup. /// [`die`](Self::die) is the death point D; dropping it is publication. +/// +/// D is one boundary, whatever drives it (the run loop, or the runtime +/// dropping the actor): exit pending for dropped replies, job admission +/// closed and every job cancelled, outgoing replies and exit signals fenced, +/// the final reason fixed with the link table closed and admission sealed, +/// and every open deferred reply answered. The caller then answers its +/// queued requests (the mode's mailbox discharge) and only then destroys +/// payloads and state; publication comes last, when this is dropped. pub(crate) struct ActorCleanup { id: ActorId, metrics: crate::instrumentation::ActorObservation, @@ -49,6 +57,8 @@ pub(crate) struct ActorCleanup { failures: Option>, /// The reply registry, sealed with the final reason at D. replies: Option>, + /// The owner's jobs: admission closes and they are cancelled at D. + jobs: Option>, } impl ActorCleanup { pub(crate) fn new( @@ -71,6 +81,7 @@ impl ActorCleanup { cancel, failures: None, replies: None, + jobs: None, } } /// Seal `replies` with the final reason at D, inside the exit-table @@ -83,6 +94,10 @@ impl ActorCleanup { self.replies = Some(replies); self } + pub(crate) fn with_jobs(mut self, jobs: std::sync::Arc) -> Self { + self.jobs = Some(jobs); + self + } pub(crate) fn with_mailbox_failures( mut self, failures: std::sync::Arc, @@ -94,17 +109,61 @@ impl ActorCleanup { /// table closes, taking one notice per link (delivered at publication). /// Idempotent; returns the final reason. pub(crate) fn die(&mut self) -> ExitReason { + self.reach_death(false) + } + + /// D reached because the runtime dropped the actor before it got there + /// (G1): `Killed` unless a non-`Normal` reason was already requested. + pub(crate) fn die_dropped(&mut self) -> ExitReason { + self.reach_death(true) + } + + fn reach_death(&mut self, dropped: bool) -> ExitReason { if let Some(reason) = &self.reason { return reason.clone(); } + // Exit pending: dropped replies wait for D; jobs stop being admitted + // and are cancelled (the job join comes after publication). + if let Some(replies) = &self.replies { + replies.exiting(); + } + if let Some(jobs) = &self.jobs { + jobs.cancel_all(); + } + // Fence outgoing replies and exit signals before the final reason is + // visible (outside any trace section: an in-flight signal may need + // one). Later ones are refused; those in flight finish first, so a + // signal sent before death precedes the death notices. + if let Some(replies) = &self.replies { + replies.fence(); + } let id = self.id.raw(); - trace::life_section(id, || { - let reason = crate::link::lock_request(&self.cause).die(); + let mut skipped = false; + let reason = trace::life_section(id, || { + let mut request = crate::link::lock_request(&self.cause); + let reason = if dropped { + skipped = !request.hook_begun(); + request.runtime_drop() + } else { + request.die() + }; + drop(request); trace::life_emit(id, || { - trace::LifeEvent::new("Die", id).x(trace::life_reason(&reason)) + trace::LifeEvent::new(if dropped { "Drop" } else { "Die" }, id) + .x(trace::life_reason(&reason)) }); self.close(reason) - }) + }); + if skipped { + log_stopped_skipped(self.id, "", "runtime_drop", Some(&reason)); + } + fail_point!("actor.at_death"); + // Every open deferred reply (and every exit-pending one) is answered + // with the final reason, before any destructor runs. + if let Some(replies) = &self.replies { + replies.close(); + } + reason } fn close(&mut self, reason: ExitReason) -> ExitReason { let replies = self.replies.as_deref(); @@ -131,28 +190,9 @@ impl ActorCleanup { } impl Drop for ActorCleanup { fn drop(&mut self) { - // Never reached D: the runtime dropped the actor (G1). - let reason = match self.reason.clone() { - Some(reason) => reason, - None => { - let id = self.id.raw(); - let mut skipped = false; - let reason = trace::life_section(id, || { - let mut request = crate::link::lock_request(&self.cause); - skipped = !request.hook_begun(); - let reason = request.runtime_drop(); - drop(request); - trace::life_emit(id, || { - trace::LifeEvent::new("Drop", id).x(trace::life_reason(&reason)) - }); - self.close(reason) - }); - if skipped { - log_stopped_skipped(self.id, "", "runtime_drop", Some(&reason)); - } - reason - } - }; + // Normally D was reached already; otherwise the runtime dropped the + // actor (G1) and nothing of it is left to destroy. + let reason = self.die_dropped(); if let Some(message) = self.failures.as_ref().and_then(|f| f.first()) { // Subscriber code must not unwind through a receiver/actor teardown. log_safely(|| { @@ -196,18 +236,21 @@ pub(crate) fn finish_actor_state(actor: A) -> Option { Some(format!("actor state destructor: {message}")) } -/// Captured as one value by the dispatch closure. Declaration order makes -/// never-polled payload destruction precede completion publication, without -/// relying on the compiler's order for disjoint closure captures. +/// Captured as one value by the dispatch closure, so a never-polled actor +/// dropped with its runtime still reaches D, then answers its queued +/// requests (`discharge`), then destroys its payload, then publishes, +/// without relying on the compiler's order for disjoint closure captures. pub(crate) struct ActorStartup { payload: Option

, cleanup: Option>, + discharge: fn(&mut P), } impl ActorStartup { - pub(crate) fn new(payload: P, cleanup: ActorCleanup) -> Self { + pub(crate) fn new(payload: P, cleanup: ActorCleanup, discharge: fn(&mut P)) -> Self { Self { payload: Some(payload), cleanup: Some(cleanup), + discharge, } } pub(crate) fn into_parts(mut self) -> (P, ActorCleanup) { @@ -216,7 +259,11 @@ impl ActorStartup { } impl Drop for ActorStartup { fn drop(&mut self) { - if let Some(payload) = self.payload.take() { + if let Some(cleanup) = &mut self.cleanup { + cleanup.die_dropped(); + } + if let Some(mut payload) = self.payload.take() { + (self.discharge)(&mut payload); if let Err(panic) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| drop(payload))) { diff --git a/concurrency/src/failpoints.rs b/concurrency/src/failpoints.rs index 5ff3f586..65532f54 100644 --- a/concurrency/src/failpoints.rs +++ b/concurrency/src/failpoints.rs @@ -141,6 +141,23 @@ pub const SITES: &[Site] = &[ description: "An owned registration is being removed during scope cleanup. Delay \ checks that a replacement only starts after the old name is gone.", }, + Site { + name: "actor.at_death", + has_error: false, + description: "Right after the death point D (the final reason is fixed, admission \ + refuses with `NoProc`), before the open deferred replies are closed \ + (tasks and threads). Delay widens the window in which an external \ + `ReplyTo::reply` races the death: it must not succeed once `NoProc` \ + was observed.", + }, + Site { + name: "actor.exit_before_deliver", + has_error: false, + description: "`ctx.exit(target, reason)` after its sender-lifetime check, before the \ + signal is delivered (tasks and threads). Delay lets the sender reach \ + its death point meanwhile: its death notice must not overtake the \ + signal.", + }, Site { name: "link.before_install", has_error: false, diff --git a/concurrency/src/mailbox.rs b/concurrency/src/mailbox.rs index 117faf04..89667e75 100644 --- a/concurrency/src/mailbox.rs +++ b/concurrency/src/mailbox.rs @@ -44,7 +44,8 @@ pub(crate) fn dispose_panic(payload: Box) { #[path = "mailbox_admission.rs"] mod admission; -use admission::SendGate; +/// Also the death point's fences (reply and exit-signal admission). +pub(crate) use admission::{SendGate, SendPermit}; #[derive(Default)] struct Admission { gate: SendGate, @@ -105,17 +106,34 @@ pub(crate) mod tasks { &mut self.inner } } - impl Drop for Receiver { - fn drop(&mut self) { + impl Receiver { + /// Close admission and take everything accepted, without dropping + /// it: the death point answers queued requests before any payload + /// destructor runs ([`discard_all`](Self::discard_all) then). + pub(crate) fn drain(&mut self) -> Vec { self.inner.close(); + let mut items = Vec::new(); // Sends remain synchronous and unbounded. EOF waits only for raw // reservations to publish, never for this runtime thread to run work. // Suppress Tokio's coop deferral while the owning task is destroyed. block_on_drop(spawned_rt::tasks::unconstrained(async { while let Some(value) = self.inner.recv().await { - self.failures.discard(value); + items.push(value); } })); + items + } + /// Destroy drained items; a destructor panic is a cleanup failure. + pub(crate) fn discard_all(&self, items: Vec) { + for item in items { + self.failures.discard(item); + } + } + } + impl Drop for Receiver { + fn drop(&mut self) { + let items = self.drain(); + self.discard_all(items); } } pub(crate) fn channel() -> (Sender, Receiver) { @@ -177,14 +195,30 @@ pub(crate) mod threads { &self.inner } } - impl Drop for Receiver { - fn drop(&mut self) { + impl Receiver { + /// Close admission (waiting for in-flight sends) and take everything + /// accepted, without dropping it (see the async receiver). + pub(crate) fn drain(&mut self) -> Vec { self.admission.close(); + let mut items = Vec::new(); while let Ok(value) = self.inner.try_recv() { - self.admission.failures.discard(value); + items.push(value); + } + items + } + /// Destroy drained items; a destructor panic is a cleanup failure. + pub(crate) fn discard_all(&self, items: Vec) { + for item in items { + self.admission.failures.discard(item); } } } + impl Drop for Receiver { + fn drop(&mut self) { + let items = self.drain(); + self.discard_all(items); + } + } pub(crate) fn channel() -> (Sender, Receiver) { let (send, recv) = raw::channel(); let admission = Arc::new(Admission::default()); diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 77e47bc1..6dd301fe 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -381,7 +381,10 @@ impl ReplyTo { .slot .take() .expect("ReplyTo holds its slot until consumed"); - let (delivery, unsent) = slot.reply(value, &now_ticks); + // Admitted by the owner's death-point fence: once `NoProc` is + // observable no reply succeeds (the owner's close answers the slot). + let (delivery, unsent) = + slot.reply_admitted(value, &now_ticks, || self.registry.reply_permit()); drop(unsent); record(delivery); if delivery == Delivery::Delivered { @@ -391,9 +394,12 @@ impl ReplyTo { } /// Answer the caller `Err(Overloaded)`, e.g. on `InsertError::Full`. + /// Past the owner's death point the caller gets its final reason. pub fn overloaded(mut self) { if let Some(slot) = self.slot.take() { - slot.terminate(Terminal::Overloaded); + if let Some(_permit) = self.registry.reply_permit() { + slot.terminate(Terminal::Overloaded); + } } } diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index c08da3f2..f8fc0d38 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -85,6 +85,14 @@ pub(crate) struct Registry { /// The owner's final exit reason, set once at its death point. Set means /// the mailbox refuses admission (`NoProc`). final_reason: OnceLock, + /// Reply admission for deferred tokens answered off the owner's turn: + /// closed at the death point, before the final reason is visible, so no + /// reply succeeds once `NoProc` is observable. + replies: crate::mailbox::SendGate, + /// Exit-signal admission for this owner's contexts (`ctx.exit`): closed + /// at the death point, before the death notices are taken, so a signal + /// sent before death is delivered before them. + signals: crate::mailbox::SendGate, #[cfg(all(spawned_sim, feature = "async"))] sim: deadline::SimService, } @@ -108,6 +116,8 @@ impl Registry { }), exit_began: Mutex::new(None), final_reason: OnceLock::new(), + replies: crate::mailbox::SendGate::default(), + signals: crate::mailbox::SendGate::default(), #[cfg(all(spawned_sim, feature = "async"))] sim: deadline::SimService::default(), }) @@ -284,6 +294,28 @@ impl Registry { } } + /// Admit a deferred reply's linearization (its `OPEN -> WRITING` CAS), + /// or `None` once the owner reached its death point: the slot then stays + /// listed and the registry's close answers it with the final reason. + pub(crate) fn reply_permit(&self) -> Option> { + self.replies.enter() + } + + /// Admit an exit signal sent by this owner, held until it is delivered; + /// `None` once the owner reached its death point (the signal is inert). + pub(crate) fn signal_permit(&self) -> Option> { + self.signals.enter() + } + + /// The death point's fences, before the final reason is visible: refuse + /// later replies and exit signals, and wait for those in flight (a CAS, + /// or one signal delivery: no user code). Never call it inside a + /// lifecycle-trace section: an in-flight delivery may need one. + pub(crate) fn fence(&self) { + self.replies.close(); + self.signals.close(); + } + /// The death point D (called inside the owner's exit-table critical /// section): fix the final reason callers will see and refuse admission. /// Idempotent: the first reason stays. @@ -391,32 +423,6 @@ impl Drop for Registry { } } -/// Closes the registry and cancels the jobs if the owner's lifecycle is -/// dropped without completing (for example, its runtime was destroyed). -/// Idempotent after a normal death. -#[cfg_attr(not(feature = "async"), allow(dead_code))] -pub(crate) struct CloseOnDrop { - replies: Arc, - jobs: Arc, -} - -#[cfg_attr(not(feature = "async"), allow(dead_code))] -impl CloseOnDrop { - pub(crate) fn new(replies: Arc, jobs: Arc) -> Self { - Self { replies, jobs } - } -} - -impl Drop for CloseOnDrop { - fn drop(&mut self) { - if !self.replies.is_dead() { - self.replies.exiting(); - self.jobs.cancel_all(); - self.replies.close(); - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 789eae0e..6b9e6f6e 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -534,13 +534,32 @@ impl Slot { /// otherwise write the value and `WRITING -> REPLIED`. A value that was /// not published is returned so the caller drops it outside every lock. pub(crate) fn reply(&self, value: R, now: &impl Fn() -> u64) -> (Delivery, Option) { - if let Err(Some(s)) = self.cas(None, |w| { + self.reply_admitted(value, now, || Some(())) + } + + /// [`reply`](Self::reply) admitted by the owner: `admit` returns a + /// permit held across the linearizing `OPEN -> WRITING` CAS (and + /// released before the value is written), or `None` once the owner is + /// past its death point. Refused, the slot is left to the owner's close + /// (`OwnerExited`). + pub(crate) fn reply_admitted

( + &self, + value: R, + now: &impl Fn() -> u64, + admit: impl FnOnce() -> Option

, + ) -> (Delivery, Option) { + let Some(permit) = admit() else { + return (Delivery::OwnerExited, Some(value)); + }; + let linearized = self.cas(None, |w| { if state(w) == OPEN { Ok(with_state(w, WRITING)) } else { Err(state(w)) } - }) { + }); + drop(permit); + if let Err(Some(s)) = linearized { let delivery = match s { TIMED_OUT => Delivery::Expired, STOPPED => Delivery::OwnerExited, diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index bbc6185c..3b49a95c 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -156,6 +156,11 @@ trait Envelope: Send { #[cfg(feature = "metrics")] fn reject(self: Box) {} + /// The owner reached its death point with this envelope undispatched: + /// answer its request now (the final reason), before any payload of the + /// mailbox is destroyed. No user code runs. + fn discharge(&mut self) {} + fn handle<'a>( self: Box, actor: &'a mut A, @@ -163,9 +168,32 @@ trait Envelope: Send { ) -> Pin + Send + 'a>>; } +/// Answer every request queued in a dead owner's mailbox, then destroy the +/// queued payloads (a destructor panic is a cleanup failure). Admission is +/// closed first, so nothing is queued after. +fn discharge_mailbox(rx: &mut mpsc::Receiver + Send>>) { + let mut queued = rx.drain(); + for envelope in &mut queued { + envelope.discharge(); + } + rx.discard_all(queued); +} + +/// A never-polled actor's startup payload, dropped with its runtime. +type StartupPayload = ( + A, + Context, + mpsc::Receiver + Send>>, + CancellationToken, +); +fn discharge_startup(payload: &mut StartupPayload) { + discharge_mailbox(&mut payload.2); +} + struct MessageEnvelope { - msg: M, + // Before the message: a plain drop also answers the request first. slot: SlotHandle, + msg: M, #[cfg(feature = "metrics")] observation: Option, } @@ -194,6 +222,9 @@ where observation.reject(); } } + fn discharge(&mut self) { + drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); + } fn handle<'a>( self: Box, actor: &'a mut A, @@ -257,8 +288,8 @@ where /// A closure run in one of the actor's turns (`spawned_test::tasks::in_actor`). type InTurn = Box) -> R + Send>; struct InTurnEnvelope { - f: InTurn, slot: SlotHandle, + f: InTurn, } impl Envelope for InTurnEnvelope { @@ -266,6 +297,9 @@ impl Envelope for InTurnEnvelope { fn message_type(&self) -> &'static str { "InTurn" } + fn discharge(&mut self) { + drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); + } fn handle<'a>( self: Box, actor: &'a mut A, @@ -547,9 +581,16 @@ impl Context { /// inert after it. A signal to this actor's own handle is a signal, not a /// self-exit: a non-trapping actor ignores `Normal`. pub fn exit(&self, target: &ChildHandle, reason: ExitReason) { + // Held until delivered: the death point waits for it, so a signal + // sent before death precedes the death notices (and none is sent + // after it). + let Some(_sending) = self.shared.replies.signal_permit() else { + return; + }; if link::lock_request(&self.shared.linked_reason).is_dead() { return; } + fail_point!("actor.exit_before_deliver"); target.exit_from(self.shared.id, reason, false); } @@ -1436,20 +1477,20 @@ impl ActorRef { move || cleanup_token.cancel(), ) .with_mailbox_failures(actor_ref.shared.sender.failures()) - .with_replies(actor_ref.shared.replies.clone()); - let startup = - crate::actor_cleanup::ActorStartup::new((actor, ctx, rx, cancellation_token), cleanup); + .with_replies(actor_ref.shared.replies.clone()) + .with_jobs(actor_ref.shared.jobs.clone()); + let startup = crate::actor_cleanup::ActorStartup::new( + (actor, ctx, rx, cancellation_token), + cleanup, + discharge_startup::, + ); let job_owner = actor_ref.shared.clone(); let cleaned_tx = cleaned_tx; - // Destruction-safe close: if the runtime drops this future before it - // completes, the guard still cancels the jobs and closes the registry, - // so no deferred caller waits for an instance that can never answer. - let close_guard = crate::reply::registry::CloseOnDrop::new( - actor_ref.shared.replies.clone(), - actor_ref.shared.jobs.clone(), - ); + // Destruction-safe: if the runtime drops this future before it + // completes, `ActorStartup` (never polled) or `ExitOnDrop` (polled) + // still reach D before destroying anything, so no caller waits for an + // instance that can never answer. let inner_future = async move { - let _close_guard = close_guard; let ((actor, ctx, rx, cancellation_token), mut cleanup) = startup.into_parts(); #[cfg(feature = "backend-experiments")] let _borrowed_runtime = borrowed_runtime; @@ -1619,43 +1660,59 @@ impl MessageTurn { } } -/// The actor's state while its lifecycle runs. If the lifecycle future is -/// dropped mid-way (its runtime was destroyed), exit begins before the state -/// is dropped, so tokens held in it answer `ActorStopped`, not `ReplyDropped`. -struct ExitOnDrop { +/// The actor's state and mailbox while its lifecycle runs. They are +/// destroyed only after the death point D: normally at the end of +/// [`run_actor`]; if the lifecycle future is dropped mid-way (its runtime was +/// destroyed), here, which reaches D first (`Killed` unless a reason was +/// requested), so tokens and queued requests answer `ActorStopped` before +/// any destructor runs. +struct ExitOnDrop<'c, A: Actor, C: Fn()> { actor: Option, - replies: Arc, - jobs: Arc, + rx: mpsc::Receiver + Send>>, + cleanup: &'c mut crate::actor_cleanup::ActorCleanup, } -impl Drop for ExitOnDrop { - fn drop(&mut self) { - if self.actor.is_some() { - self.replies.exiting(); - self.jobs.cancel_all(); +impl ExitOnDrop<'_, A, C> { + /// D, then the queued requests, then destruction (mailbox, then state). + fn finish(&mut self, dropped: bool) { + let Some(actor) = self.actor.take() else { + return; + }; + if dropped { + self.cleanup.die_dropped(); + } else { + self.cleanup.die(); + } + discharge_mailbox(&mut self.rx); + // Destruction: a destructor panic is a cleanup failure, never the reason. + if let Some(failure) = crate::actor_cleanup::finish_actor_state(actor) { + self.cleanup.cleanup_failed(failure); } } } +impl Drop for ExitOnDrop<'_, A, C> { + fn drop(&mut self) { + self.finish(true); + } +} async fn run_actor( actor: A, ctx: Context, - mut rx: mpsc::Receiver + Send>>, + rx: mpsc::Receiver + Send>>, cancellation_token: CancellationToken, cleanup: &mut crate::actor_cleanup::ActorCleanup, #[cfg(feature = "backend-experiments")] turn: Option, ) { - let replies = ctx.shared.replies.clone(); - let jobs = ctx.shared.jobs.clone(); let request = ctx.shared.linked_reason.clone(); let mut owned = ExitOnDrop { actor: Some(actor), - replies: replies.clone(), - jobs: jobs.clone(), + rx, + cleanup, }; if let Err(payload) = AssertUnwindSafe(run_actor_lifecycle( owned.actor.as_mut().expect("the state lives until death"), ctx, - &mut rx, + &mut owned.rx, cancellation_token, #[cfg(feature = "backend-experiments")] turn, @@ -1670,21 +1727,13 @@ async fn run_actor( ))); } // The death point D, only now that `stopped()` returned (or was - // skipped): the reason becomes final, the link table closes, the - // registry closes (every open token is answered) and the mailbox refuses - // admission (queued requests are answered). Early returns (startup - // panic, closed scope) and an unwound lifecycle also come here, so jobs - // never outlive their owner. - replies.exiting(); - jobs.cancel_all(); - cleanup.die(); - replies.close(); - drop(rx); - // Destruction: a destructor panic is a cleanup failure, never the reason. - let actor = owned.actor.take().expect("the state lives until death"); - if let Some(failure) = crate::actor_cleanup::finish_actor_state(actor) { - cleanup.cleanup_failed(failure); - } + // skipped): the reason becomes final, the link table closes, job + // admission and the registry close (every open token is answered), then + // the mailbox refuses admission and its queued requests are answered, + // all before any destructor. Early returns (startup panic, closed scope) + // and an unwound lifecycle also come here, so jobs never outlive their + // owner. + owned.finish(false); } async fn run_actor_lifecycle( diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 38207ce0..d4c363ae 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -91,12 +91,47 @@ trait Envelope: Send { #[cfg(feature = "metrics")] fn reject(self: Box) {} + /// The owner reached its death point with this envelope undispatched: + /// answer its request now (the final reason), before any payload of the + /// mailbox is destroyed. No user code runs. + fn discharge(&mut self) {} + fn handle(self: Box, actor: &mut A, ctx: &Context); } +/// Answer every request queued in a dead owner's mailbox (and `extra`, an +/// envelope received after the stop), then destroy the queued payloads (a +/// destructor panic is a cleanup failure). Admission is closed first +/// (waiting for in-flight sends), so nothing is queued after. +fn discharge_mailbox( + rx: &mut mpsc::Receiver>, + extra: Option + Send>>, +) { + let mut queued = rx.drain(); + queued.extend(extra.map(MailboxItem::Message)); + for item in &mut queued { + if let MailboxItem::Message(envelope) = item { + envelope.discharge(); + } + } + rx.discard_all(queued); +} + +/// A never-polled actor's startup payload (its thread was never started). +type StartupPayload = ( + A, + Context, + mpsc::Receiver>, + CancellationToken, +); +fn discharge_startup(payload: &mut StartupPayload) { + discharge_mailbox(&mut payload.2, None); +} + struct MessageEnvelope { - msg: M, + // Before the message: a plain drop also answers the request first. slot: SlotHandle, + msg: M, #[cfg(feature = "metrics")] observation: Option, } @@ -121,6 +156,9 @@ where observation.reject(); } } + fn discharge(&mut self) { + drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); + } fn handle(self: Box, actor: &mut A, ctx: &Context) { let mut this = *self; let reply = PendingReply::new(this.slot.take(), &ctx.replies); @@ -182,11 +220,14 @@ where /// A closure run in one of the actor's turns (`spawned_test::threads::in_actor`). type InTurn = Box) -> R + Send>; struct InTurnEnvelope { - f: InTurn, slot: SlotHandle, + f: InTurn, } impl Envelope for InTurnEnvelope { + fn discharge(&mut self) { + drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); + } fn handle(self: Box, actor: &mut A, ctx: &Context) { let InTurnEnvelope { f, mut slot } = *self; let reply = PendingReply::new(slot.take(), &ctx.replies); @@ -435,9 +476,16 @@ impl Context { /// Works until this actor's death point, including from `stopped()`; /// inert after it. pub fn exit(&self, target: &ChildHandle, reason: ExitReason) { + // Held until delivered: the death point waits for it, so a signal + // sent before death precedes the death notices (and none is sent + // after it). + let Some(_sending) = self.replies.signal_permit() else { + return; + }; if link::lock_request(&self.linked_reason).is_dead() { return; } + fail_point!("actor.exit_before_deliver"); target.exit_from(self.id, reason, false); } @@ -1150,9 +1198,13 @@ impl ActorRef { move || cleanup_token.cancel(), ) .with_mailbox_failures(actor_ref.sender.failures()) - .with_replies(actor_ref.replies.clone()); - let startup = - crate::actor_cleanup::ActorStartup::new((actor, ctx, rx, cancellation_token), cleanup); + .with_replies(actor_ref.replies.clone()) + .with_jobs(actor_ref.jobs.clone()); + let startup = crate::actor_cleanup::ActorStartup::new( + (actor, ctx, rx, cancellation_token), + cleanup, + discharge_startup::, + ); let replies = actor_ref.replies.clone(); let jobs = actor_ref.jobs.clone(); let cleaned = crate::reply::jobs::CleanedGuard(actor_ref.cleaned.clone()); @@ -1160,23 +1212,27 @@ impl ActorRef { let _cleaned = cleaned; let ((mut actor, ctx, mut rx, cancellation_token), mut cleanup) = startup.into_parts(); let request = ctx.linked_reason.clone(); - if let Err(payload) = catch_unwind(AssertUnwindSafe(|| { + let discarded = match catch_unwind(AssertUnwindSafe(|| { run_actor(&mut actor, ctx, &mut rx, cancellation_token) })) { - let message = panic_message(&*payload); - crate::mailbox::dispose_panic(payload); - link::lock_request(&request).panic(ExitReason::Panic(format!( - "actor lifecycle unwound: {message}" - ))); - } + Ok(discarded) => discarded, + Err(payload) => { + let message = panic_message(&*payload); + crate::mailbox::dispose_panic(payload); + link::lock_request(&request).panic(ExitReason::Panic(format!( + "actor lifecycle unwound: {message}" + ))); + None + } + }; // The death point D, only now that `stopped()` returned (or was - // skipped): the reason becomes final, the link table closes, the - // registry closes (every open token is answered) and the mailbox - // refuses admission (queued requests are answered). - replies.exiting(); - jobs.cancel_all(); + // skipped): the reason becomes final, the link table closes, job + // admission and the registry close (every open token is + // answered), then the mailbox refuses admission and its queued + // requests (and one received after the stop) are answered, all + // before any destructor. cleanup.die(); - replies.close(); + discharge_mailbox(&mut rx, discarded); drop(rx); // Destruction: a destructor panic is a cleanup failure only. if let Some(failure) = crate::actor_cleanup::finish_actor_state(actor) { @@ -1197,12 +1253,14 @@ impl ActorRef { } } +/// Returns an envelope received after the stop (undispatched; answered at +/// the death point). fn run_actor( actor: &mut A, ctx: Context, rx: &mut mpsc::Receiver>, cancellation_token: CancellationToken, -) { +) -> Option + Send>> { run_actor_receiving( actor, ctx, @@ -1219,7 +1277,7 @@ fn run_actor_receiving( ctx: Context, mut receive: impl FnMut() -> Result, std::sync::mpsc::RecvTimeoutError>, cancellation_token: CancellationToken, -) { +) -> Option + Send>> { let mut discarded = None; let request = ctx.linked_reason.clone(); let trace_id = ctx.id.raw(); @@ -1242,7 +1300,7 @@ fn run_actor_receiving( .panic(ExitReason::Panic(format!("panic in started(): {msg}"))); }); cancellation_token.cancel(); - return; + return None; } // Phase 0 ends: resolve signals held during started() with the trap // flag read now. @@ -1362,9 +1420,11 @@ fn run_actor_receiving( } // Message destructors and reply cancellation may re-enter monitor state. - // Preserve stopped() -> monitor cleanup -> undelivered-message drop. + // Preserve stopped() -> monitor cleanup -> undelivered-message drop (the + // envelope received after the stop is answered at the death point, then + // dropped with the mailbox). drop(_monitor_cleanup); - ctx.sender.failures().discard(discarded); + discarded } // --------------------------------------------------------------------------- @@ -2275,8 +2335,11 @@ mod idle_stop_regression { let trace = events.clone(); let actor = std::thread::spawn(move || { let mut first = true; - run_actor_receiving( - &mut Counter(seen, trace), + let mut counter = Counter(seen, trace); + // The envelope received after the stop comes back undispatched; + // the death point answers it and destroys it before the state. + let discarded = run_actor_receiving( + &mut counter, ctx, move || { // Pause the real receive boundary after the started/cancel @@ -2289,7 +2352,9 @@ mod idle_stop_regression { rx.recv_timeout(Duration::from_secs(5)) }, cancel, - ) + ); + drop(discarded); + drop(counter); }); entered.wait(); let (slot, response) = reply_slot(crate::reply::Deadline::Never, None); diff --git a/concurrency/tests/death_boundary.rs b/concurrency/tests/death_boundary.rs new file mode 100644 index 00000000..fdbdaa64 --- /dev/null +++ b/concurrency/tests/death_boundary.rs @@ -0,0 +1,264 @@ +//! The death point D (docs/design/semantics.md, `Lifecycle!NoOrphanRequest`): +//! every queued request and every deferred reply is answered with the final +//! reason at D, before any payload or state destructor runs and whatever +//! drives the death (the run loop, or the runtime dropping the actor). Each +//! test blocks a destructor and checks that the callers are answered while +//! it is still blocked. +#![cfg(not(spawned_sim))] // blocking destructors and OS threads +use spawned_concurrency::{message::Message, ActorError, ExitReason}; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +const LIMIT: Duration = Duration::from_secs(5); + +/// A payload whose destructor blocks until released (or `LIMIT`), and +/// reports that it started. +struct Gated { + started: Option>, + release: Option>, +} +impl Drop for Gated { + fn drop(&mut self) { + if let (Some(started), Some(release)) = (self.started.take(), self.release.take()) { + let _ = started.send(()); + let _ = release.recv_timeout(LIMIT); + } + } +} +/// A gated payload, its "destructor started" signal and its release. +fn gated() -> (Gated, mpsc::Receiver<()>, mpsc::Sender<()>) { + let (started_tx, started) = mpsc::channel(); + let (release, release_rx) = mpsc::channel(); + ( + Gated { + started: Some(started_tx), + release: Some(release_rx), + }, + started, + release, + ) +} +fn inert() -> Gated { + Gated { + started: None, + release: None, + } +} + +/// A request carrying a payload. +struct Queued(#[allow(dead_code)] Gated); +impl Message for Queued { + type Result = (); +} +/// Blocks the handler until released. +struct Block(mpsc::Receiver<()>, mpsc::Sender<()>); +impl Message for Block { + type Result = (); +} +/// Defers and keeps the token in the actor's state. +struct Park; +impl Message for Park { + type Result = (); +} + +/// Wait until `poll` yields, at most `LIMIT`. +fn eventually(what: &str, mut poll: impl FnMut() -> Option) -> T { + let began = Instant::now(); + loop { + if let Some(value) = poll() { + return value; + } + assert!(began.elapsed() < LIMIT, "{what}"); + std::thread::sleep(Duration::from_millis(2)); + } +} + +#[cfg(feature = "async")] +mod tasks_mode { + use super::*; + use spawned_concurrency::tasks::{Actor, ActorStart, Context, Handler}; + use spawned_concurrency::{PendingReply, ReplyTo}; + + struct Owner { + _state: Gated, + parked: Vec>, + } + impl Actor for Owner {} + impl Handler for Owner { + async fn handle(&mut self, _: Queued, reply: PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } + } + impl Handler for Owner { + async fn handle(&mut self, msg: Block, reply: PendingReply<'_, ()>, _: &Context) { + let _ = msg.1.send(()); + let _ = msg.0.recv_timeout(LIMIT); + reply.reply(()); + } + } + impl Handler for Owner { + async fn handle(&mut self, _: Park, reply: PendingReply<'_, ()>, _: &Context) { + self.parked.push(reply.defer().unwrap()); + } + } + + fn stopped(result: Option>, reason: &ExitReason) -> Option<()> { + match result? { + Err(ActorError::ActorStopped(r)) if r == *reason => Some(()), + Err(other) => panic!("unexpected error {other}"), + Ok(_) => panic!("a request that was never handled was answered Ok"), + } + } + + /// Review P0 (Codex 1): requests queued behind the exit are answered + /// at D, before the first queued payload's destructor runs. The mailbox + /// used to be dropped envelope by envelope, each payload before its + /// request: a blocking destructor left its own caller and every later + /// one unanswered while it ran. + #[test] + fn queued_requests_are_answered_before_payload_destructors() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let _guard = runtime.enter(); + let actor = Owner { + _state: inert(), + parked: Vec::new(), + } + .start(); + let (unblock, blocked) = mpsc::channel(); + let (entered_tx, entered) = mpsc::channel(); + actor.send(Block(blocked, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + let (payload, destroying, release) = gated(); + let mut first = actor.request_raw(Queued(payload)).unwrap(); + let mut second = actor.request_raw(Queued(inert())).unwrap(); + actor.context().stop(); + unblock.send(()).unwrap(); + destroying.recv_timeout(LIMIT).unwrap(); + // The first queued payload's destructor is running (blocked). + eventually("the queued callers were not answered at death", || { + stopped(first.try_recv(), &ExitReason::Normal) + }); + eventually( + "a later queued caller waited for an earlier destructor", + || stopped(second.try_recv(), &ExitReason::Normal), + ); + release.send(()).unwrap(); + runtime.block_on(actor.join()); + } + + /// Review P1 (Codex 5): a never-polled actor dropped with its runtime + /// reaches D (queued callers answered `Killed`) before its state and + /// mailbox are destroyed. Its startup payload was destroyed first. + #[test] + fn a_never_polled_actor_answers_before_its_state_is_destroyed() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (state, destroying, release) = gated(); + // Spawned but never polled: the runtime is never driven. + let actor = { + let _guard = runtime.enter(); + Owner { + _state: state, + parked: Vec::new(), + } + .start() + }; + let mut queued = actor.request_raw(Queued(inert())).unwrap(); + let dropper = std::thread::spawn(move || drop(runtime)); + destroying.recv_timeout(LIMIT).unwrap(); + eventually("a queued caller waited for the state destructor", || { + stopped(queued.try_recv(), &ExitReason::Killed) + }); + release.send(()).unwrap(); + dropper.join().unwrap(); + assert_eq!(actor.exit_reason(), Some(ExitReason::Killed)); + } + + /// Review P1 (Codex 5): a running actor dropped with its runtime reaches + /// D (deferred and queued callers answered `Killed`) before its state is + /// destroyed. The state was destroyed first, and the registry closed + /// after publication. + #[test] + fn a_dropped_running_actor_answers_before_its_state_is_destroyed() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (state, destroying, release) = gated(); + let (actor, mut parked) = runtime.block_on(async { + let actor = Owner { + _state: state, + parked: Vec::new(), + } + .start(); + let parked = actor.request_raw(Park).unwrap(); + // Let the actor defer it. + actor.request(Queued(inert())).await.unwrap(); + (actor, parked) + }); + let dropper = std::thread::spawn(move || drop(runtime)); + destroying.recv_timeout(LIMIT).unwrap(); + eventually("a deferred caller waited for the state destructor", || { + stopped(parked.try_recv(), &ExitReason::Killed) + }); + release.send(()).unwrap(); + dropper.join().unwrap(); + assert_eq!(actor.exit_reason(), Some(ExitReason::Killed)); + } +} + +mod threads_mode { + use super::*; + use spawned_concurrency::threads::{Actor, ActorStart, Context, Handler}; + use spawned_concurrency::PendingReply; + + struct Owner; + impl Actor for Owner {} + impl Handler for Owner { + fn handle(&mut self, _: Queued, reply: PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } + } + impl Handler for Owner { + fn handle(&mut self, msg: Block, reply: PendingReply<'_, ()>, _: &Context) { + let _ = msg.1.send(()); + let _ = msg.0.recv_timeout(LIMIT); + reply.reply(()); + } + } + + /// Review P0 (Codex 1), threads mode. + #[test] + fn queued_requests_are_answered_before_payload_destructors() { + let actor = Owner.start(); + let (unblock, blocked) = mpsc::channel(); + let (entered_tx, entered) = mpsc::channel(); + actor.send(Block(blocked, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + let (payload, destroying, release) = gated(); + let mut first = actor.request_raw(Queued(payload)).unwrap(); + let mut second = actor.request_raw(Queued(inert())).unwrap(); + actor.context().stop(); + unblock.send(()).unwrap(); + destroying.recv_timeout(LIMIT).unwrap(); + let answered = |rx: &mut spawned_concurrency::ReplyReceiver<()>| match rx.try_recv()? { + Err(ActorError::ActorStopped(ExitReason::Normal)) => Some(()), + other => panic!("unexpected {other:?}"), + }; + eventually("the queued callers were not answered at death", || { + answered(&mut first) + }); + eventually( + "a later queued caller waited for an earlier destructor", + || answered(&mut second), + ); + release.send(()).unwrap(); + actor.join(); + } +} diff --git a/concurrency/tests/failpoints.rs b/concurrency/tests/failpoints.rs index 1700b87b..9fd8b056 100644 --- a/concurrency/tests/failpoints.rs +++ b/concurrency/tests/failpoints.rs @@ -930,6 +930,164 @@ async fn system_shutdown_coordinator_panic_is_reported_and_contained() { // ------------------------------------------------------------ actor.* (threads) +// ------------------------------------------------- actor.at_death (D fence) + +/// Holds deferred tokens for an outside replier. +#[derive(Default)] +struct Holder; +struct Hold(std::sync::mpsc::Sender>); +impl Message for Hold { + type Result = u32; +} +impl Actor for Holder {} +impl Handler for Holder { + async fn handle( + &mut self, + msg: Hold, + reply: spawned_concurrency::PendingReply<'_, u32>, + _: &Context, + ) { + msg.0.send(reply.defer().unwrap()).unwrap(); + } +} +impl Handler for Holder { + async fn handle( + &mut self, + _: Ping, + _: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + } +} + +/// Review P1 (Codex 4): once `NoProc` is observable (a send to the dead +/// owner fails), an external `ReplyTo::reply` must not succeed: the caller +/// gets the owner's final reason. Replies were admitted until the registry +/// closed the slot, after D: a reply could land after `NoProc`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn no_reply_succeeds_after_noproc_is_observable() { + let _scenario = FailScenario::setup(); + let holder = Holder.start(); + let (tokens, token) = std::sync::mpsc::channel(); + let caller = holder.request_raw(Hold(tokens)).unwrap(); + let token = token.recv_timeout(LIMIT).unwrap(); + failpoints::configure( + "actor.at_death", + FailPoint::new(Action::Delay(Duration::from_millis(300))).times(1), + ) + .unwrap(); + holder.context().stop(); + let began = Instant::now(); + while holder.send(Ping).is_ok() { + assert!(began.elapsed() < LIMIT, "the owner never died"); + tokio::time::sleep(Duration::from_millis(1)).await; + } + // `NoProc` is observable: the death point passed. + let delivery = token.reply(7); + assert_ne!( + delivery, + spawned_concurrency::Delivery::Delivered, + "a reply succeeded after NoProc was observable" + ); + assert!( + matches!( + caller.await, + Err(spawned_concurrency::ActorError::ActorStopped( + ExitReason::Normal + )) + ), + "the caller did not get the final reason" + ); + holder.join().await; +} + +// ---------------------------------------- actor.exit_before_deliver (fence) + +/// Traps exits and records them in arrival order. +struct Recorder(Arc>>); +impl Actor for Recorder { + async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); + } + async fn exit_received(&mut self, exit: spawned_concurrency::Exit, _: &Context) { + self.0.lock().unwrap().push((exit.from, exit.reason)); + } +} +impl Handler for Recorder { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } +} +struct SignalSender; +impl Actor for SignalSender {} +impl Handler for SignalSender { + async fn handle( + &mut self, + msg: LinkTo, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.link(&msg.0); + reply.reply(()); + } +} + +/// Review P1 (Codex 10): a signal a cloned context sends before its +/// sender's death reaches the target before the sender's death notice +/// (`ExitAfterMessages`, OTP's per-pair signal order). A helper passed the +/// sender-lifetime check, paused, and delivered after the death notice. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_exit_signal_is_not_overtaken_by_its_senders_death_notice() { + let _scenario = FailScenario::setup(); + let seen = Arc::new(Mutex::new(Vec::new())); + let recorder = Recorder(seen.clone()).start(); + recorder.request(Ping).await.unwrap(); // trapping now + let sender = SignalSender.start(); + sender + .request(LinkTo(recorder.child_handle())) + .await + .unwrap(); + failpoints::configure( + "actor.exit_before_deliver", + FailPoint::new(Action::Delay(Duration::from_millis(300))).times(1), + ) + .unwrap(); + let ctx = sender.context(); + let target = recorder.child_handle(); + let helper = std::thread::spawn(move || ctx.exit(&target, ExitReason::Shutdown)); + reached("actor.exit_before_deliver", 1).await; + // The sender dies while its helper's signal is in flight. + sender.context().stop(); + let began = Instant::now(); + while seen.lock().unwrap().len() < 2 { + assert!( + began.elapsed() < LIMIT, + "exits seen: {:?}", + seen.lock().unwrap() + ); + tokio::time::sleep(Duration::from_millis(2)).await; + } + helper.join().unwrap(); + let reasons: Vec<_> = seen + .lock() + .unwrap() + .iter() + .map(|(_, r)| r.clone()) + .collect(); + assert_eq!( + reasons, + [ExitReason::Shutdown, ExitReason::Normal], + "the death notice overtook the sender's own signal" + ); + recorder.context().stop(); + recorder.join().await; +} + mod native { use super::*; use spawned_concurrency::threads::{ diff --git a/docs/TESTING.md b/docs/TESTING.md index 1b1830ee..b4e38e81 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -991,6 +991,8 @@ and prints the seed when a test fails. Configuration is process-global. | `system.reserve` | yes | Placement failure (`SystemError::Capacity`) | | `registry.before_register` | yes | Shutdown racing managed name registration | | `registry.before_unregister` | no | Slow removal of an owned name before a replacement starts | +| `actor.at_death` | no | An external reply racing the death point after `NoProc` is observable (tasks and threads) | +| `actor.exit_before_deliver` | no | A cloned context's exit signal racing its sender's death (tasks and threads) | | `link.before_install` | no | Peer exiting while a link is installed (tasks and threads) | | `monitor.before_install` | no | Target exiting while a monitor is installed (tasks and threads) | diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index a69f8e9c..fee16c23 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -382,14 +382,27 @@ current turn's slot) and a job set. The order is: answered. 3. **Death.** This happens only when `stopped()` has returned. Rust cannot preempt `stopped(&mut self)` or a hanging destructor, and it cannot - destroy state that is still borrowed. In this order: - 1. the registry closes, and every open token it lists becomes `STOPPED`, + destroy state that is still borrowed. In this order, whether the run + loop or the runtime dropping the actor drives it: + 1. job admission closes and every job is cancelled; outgoing replies of + deferred tokens and exit signals sent through the actor's contexts + are fenced (those in flight finish, later ones are refused: a reply + returns `OwnerExited`, a signal is inert); + 2. the final reason is fixed, the link table closes and admission is + sealed (`NoProc`); + 3. the registry closes, and every open token it lists becomes `STOPPED`, wherever the token is (state, job, helper, leaked); - 2. the mailbox is disposed: queued envelopes are dropped as `STOPPED`, and - emit credits return; - 3. actor state is dropped; - 4. the deadline heap is dropped; - 5. the exit reason is published, and monitors and links fire. + 4. the mailbox closes and every queued request is answered `STOPPED`; + 5. only then are payloads destroyed: queued envelopes (emit credits + return), then actor state; + 6. the deadline heap is dropped; + 7. the exit reason is published, and monitors and links fire. + + Steps 1 to 4 run no user code. Steps 3 and 4 follow the seal instead of + being one atomic step with it, but nothing between them depends on the + requests they answer (no destructor, notice or publication), so the + models' atomic death is a sound abstraction of them (they commute left + to the seal). `ActorStopped` keeps one meaning, "the actor is gone". An observer that sees `Down` knows every caller of the instance has been answered. diff --git a/docs/design/semantics.md b/docs/design/semantics.md index 62c5f369..cb2675f2 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -152,11 +152,18 @@ takes one of the instance's `MAX_OUTSTANDING` credits (or answers from a later turn, a job or `stopped()`; dropping it answers `ReplyDropped` between turns, is committed at the turn's end inside a turn, and while the owner exits is held until its death point. At the owner's death every open -token becomes `ActorStopped` with the final reason, wherever it is held. The owner's deadline heap expires +token becomes `ActorStopped` with the final reason, wherever it is held; a +reply made once `NoProc` is observable for the owner is refused +(`Delivery::OwnerExited`) and the caller gets that reason. Queued requests +and deferred tokens are answered at the death point before any queued +payload or state destructor runs, also when the runtime drops the actor +(G1). The owner's deadline heap expires deferred tokens (returning their credits) even when no caller polls. Callers and protocols cannot tell a deferred reply from an immediate one. Checked by: `reply_slot_loom` (every concurrent pair of the slot word), -[`deferred_replies`][deferred_replies], `reply::registry::tests`. +[`deferred_replies`][deferred_replies], `reply::registry::tests`, +`death_boundary` (blocking destructors; both modes; runtime drop), +`failpoints::no_reply_succeeds_after_noproc_is_observable`. **R7. Pipe-to-self jobs.** `ctx.pipe*` runs work owned by the instance and delivers `done(outcome)` as one message in a later turn; a panic arrives as @@ -297,11 +304,16 @@ read when it arrives: Propagation is transitive through non-trapping actors. Links work across modes. Signals arriving during `started()` take effect when it returns, with -the trap flag read then. +the trap flag read then. `ctx.exit` works until the sender's death point, +also from a cloned context on another thread: the death point waits for a +signal in flight, so a signal sent before it reaches its target before the +sender's death notice (`Lifecycle!ExitAfterMessages`); one sent after it is +inert. Checked by: `Lifecycle!TrapRespected`, `Lifecycle!NormalSignalIgnored`, `Lifecycle!KilledFromKill`, `Lifecycle!SkipHook`, `Lifecycle!LinkedPeersStop`, `Lifecycle!ShutdownEndsChild` (TLC); [`otp_exit_signals`][otp_exit_signals]`::tasks_mode::{exit_signals_follow_exit_signal_2, kill_is_untrappable_and_propagates_killed, ctx_exit_sends_from_the_calling_actor, parent_exit_ends_a_trapping_child_normally, signals_during_started_are_resolved_with_the_flag_set_in_started}`, `threads_mode::kill_exit_and_noproc_in_threads_mode`; +`failpoints::an_exit_signal_is_not_overtaken_by_its_senders_death_notice`; `src/link.rs::tests::signal_rules_follow_exit_signal_2`; `src/tasks/actor.rs::link_propagates_panic_to_non_trapping_peer`, `link_delivers_exit_to_trapping_peer`, `link_normal_exit_not_propagated_to_non_trapping_peer`, diff --git a/verification/tla/Lifecycle.tla b/verification/tla/Lifecycle.tla index 1e5b9934..43cb143c 100644 --- a/verification/tla/Lifecycle.tla +++ b/verification/tla/Lifecycle.tla @@ -218,6 +218,11 @@ Fire(a, r) == (* The death point D of a with final reason r: admission closes, queued *) (* requests are answered, the table closes. Shared by Die and RuntimeDrop. *) +(* The code fences outgoing exit signals and replies first, then seals and *) +(* closes the table, then answers deferred and queued requests, and only *) +(* then destroys payloads and state (actor_cleanup.rs). The answers follow *) +(* the seal but run no user code and nothing in between depends on them, *) +(* so they commute left to it: this atomic step is a sound abstraction. *) Death(a, r) == /\ st' = [st EXCEPT ![a] = "gone"] /\ mbox' = [mbox EXCEPT ![a] = <<>>] From 1a5e4316878c7c649dc1c67d55c7d7a68c6b6a35 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:02:45 -0300 Subject: [PATCH 111/208] refactor!: DeferError and Delivery are non_exhaustive; doc and model notes agy 8: `DeferError` and `Delivery` are reply outcomes that will grow (for example new delivery outcomes); they are `#[non_exhaustive]`, like `JobOutcome`, `JobError` and `PipeErrorKind` already are. The UI fixture, the capability example and the guide/design examples gain a wildcard arm. Not marked: `JobOptions` (a struct with public fields: the ethlambda port builds it with struct literals, which `#[non_exhaustive]` forbids outside the crate), `InsertError` (matched exhaustively by the ethrex port; closed by design, as is `PipeReplyError`). agy 7: `Lifecycle.tla` and the TLA README now say that phase 0 (signals held during `started()`) is not modelled, only covered by integration tests. The other agy 7 items did not hold on this branch: the design doc already shows `with_deadline(&self, ..)`, and no doc mentions `EmitError` (7d is the restart-budget fix, done separately). --- CHANGELOG.md | 3 +++ concurrency/src/reply/mod.rs | 4 +++- concurrency/src/reply/slot.rs | 3 ++- concurrency/tests/ui-pass/deferred_handlers.rs | 2 ++ docs/GUIDE.md | 1 + docs/design/deferred-replies.md | 1 + examples/capabilities/src/bin/deferred_replies.rs | 6 ++++-- verification/tla/Lifecycle.tla | 4 ++++ verification/tla/README.md | 4 ++-- 9 files changed, 22 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75ac5fbc..2196c952 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +- **Breaking (0.8):** `DeferError` and `Delivery` are `#[non_exhaustive]` + (like `JobOutcome`, `JobError` and `PipeErrorKind`): match them with a + wildcard arm. - **Breaking (0.8):** the restart budget counts restart actions, as OTP's intensity does (ROADMAP rank 5f): a OneForAll or RestForOne recovery round takes one `max_restarts` grant when the coordinator selects it, before the diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 6dd301fe..0466a046 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -197,8 +197,10 @@ impl Drop for SlotHandle { // PendingReply // --------------------------------------------------------------------------- -/// Why [`PendingReply::defer`] did not return a token. +/// Why [`PendingReply::defer`] did not return a token. More reasons may be +/// added. #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] pub enum DeferError { /// The owner's `MAX_OUTSTANDING` credits are in use; the caller has /// already been answered `Overloaded`. diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 6b9e6f6e..b81ab285 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -115,8 +115,9 @@ impl Terminal { } } -/// Outcome of a reply, as the replier sees it. +/// Outcome of a reply, as the replier sees it. More outcomes may be added. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum Delivery { /// The value was published to the caller. Delivered, diff --git a/concurrency/tests/ui-pass/deferred_handlers.rs b/concurrency/tests/ui-pass/deferred_handlers.rs index 1888164f..719f5129 100644 --- a/concurrency/tests/ui-pass/deferred_handlers.rs +++ b/concurrency/tests/ui-pass/deferred_handlers.rs @@ -27,6 +27,8 @@ mod tasks_mode { match reply.defer() { Ok(token) => self.waiters.push(token), Err(DeferError::Overloaded | DeferError::Closed | DeferError::NoCaller) => {} + // `DeferError` is `#[non_exhaustive]`. + Err(_) => {} } } #[send_handler] diff --git a/docs/GUIDE.md b/docs/GUIDE.md index a55c1205..cb30f999 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -580,6 +580,7 @@ impl PeerTable { Err(DeferError::Overloaded) => {} // the caller was already answered Overloaded Err(DeferError::Closed) => {} // the caller timed out or left Err(DeferError::NoCaller) => {} // reached by `send` + Err(_) => {} // `DeferError` is #[non_exhaustive] } } } diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index fee16c23..f85e5009 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -304,6 +304,7 @@ impl PeerTable { Err(DeferError::Overloaded) => {} // caller already answered Overloaded Err(DeferError::Closed) => {} // caller already timed out or left Err(DeferError::NoCaller) => {} // reached by send + Err(_) => {} // #[non_exhaustive] } } } diff --git a/examples/capabilities/src/bin/deferred_replies.rs b/examples/capabilities/src/bin/deferred_replies.rs index 91623fcf..e2efd547 100644 --- a/examples/capabilities/src/bin/deferred_replies.rs +++ b/examples/capabilities/src/bin/deferred_replies.rs @@ -16,7 +16,7 @@ use spawned_concurrency::tasks::{ Actor, ActorRef, ActorStart, Context, Deadline, Handler, InsertError, JobError, JobOptions, JobOutcome, PendingReplies, PendingReply, ReplyTo, }; -use spawned_concurrency::{actor, protocol, ActorError, DeferError, Response}; +use spawned_concurrency::{actor, protocol, ActorError, Response}; use std::collections::VecDeque; use std::time::Duration; @@ -248,7 +248,9 @@ impl PeerTable { } match reply.defer() { Ok(token) => self.waiters.push_back(token), - Err(DeferError::Overloaded | DeferError::Closed | DeferError::NoCaller) => {} + // Overloaded, Closed or NoCaller: the caller was already answered + // (or there is none). `DeferError` is `#[non_exhaustive]`. + Err(_) => {} } } diff --git a/verification/tla/Lifecycle.tla b/verification/tla/Lifecycle.tla index 43cb143c..8966483f 100644 --- a/verification/tla/Lifecycle.tla +++ b/verification/tla/Lifecycle.tla @@ -27,6 +27,10 @@ EXTENDS Naturals, Sequences, FiniteSets (* termination when dispatched; a Normal signal is ignored by a *) (* non-trapping actor; any other reason begins exit with that reason. *) (* Administrative stops (ChildHandle::stop and friends) are untrappable. *) +(* Not modelled: phase 0 (signals arriving while started() runs are held *) +(* as envelopes in arrival order and resolved with the trap flag read when *) +(* it returns, link.rs ExitRequest::end_starting); actors start in "run". *) +(* Integration tests cover it (otp_exit_signals, signals during started). *) (* Handlers are atomic steps; "after the current handler" is the next *) (* step. Liveness assumes handlers, stopped() and destruction return *) (* (weak fairness on Dispatch, Die and Publish). *) diff --git a/verification/tla/README.md b/verification/tla/README.md index 36653e61..82d4ae05 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -480,8 +480,8 @@ steps; its header lists the mapping (actors `a1`, `a2` in spawn order, folded request panics, the late `NoProc` notice of a non-trapping caller, and `Hook` "skip" accepted after a kill that arrived before `stopped()` began, which the model's `Kill` leaves out). Phase 0 (signals held during -`started()`), managed scopes' own stops and actor-to-actor sends are not -mapped: they reject. `concurrency/tests/lifecycle_trace.rs` records 14 +`started()`; not in `Lifecycle.tla` either: actors start in `run`), managed +scopes' own stops and actor-to-actor sends are not mapped: they reject. `concurrency/tests/lifecycle_trace.rs` records 14 scenarios (`lifecycle_*.json`, schema `lifecycle-v1`): kill with a queued request answered `Killed` and a later `NoProc`, kill during `stopped()`, untrapped and `Normal` signals, trapped exits (also from an actor, then a From c151f5ed69411f3bd8411dce960f394b52448084 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:02:58 -0300 Subject: [PATCH 112/208] docs(changelog): user-visible fixes from the 0.8 pre-release review --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2196c952..38d026fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ ## Unreleased +- Deferred replies and jobs, fixes from the 0.8 pre-release review: a job + deadline still fires while the result waits for scheduled emits, and the + job keeps its `MAX_OUTSTANDING` credit until its terminal message; a job + admitted while its owner exits is refused in one step with the exit + (`pipe_reply*` returns `OwnerStopping` with the token); in `tasks` mode a + blocking job's thread (outside a runtime) is created at reservation + (`SpawnFailed` instead of a silently lost job) and a managed actor's + blocking job keeps its scope until its body ends (a supervisor does not + replace the child, and shutdown is not complete, while it runs); + shortening a token's deadline no longer cancels a `cancel_on_caller_gone` + blocking job and wakes the waiting caller to re-arm its timer; an + `emit_at` discarded by a concurrent cancel returns `JobGone`; a refused + emit returns its credit once; a panicking payload destructor no longer + stops the process-wide deadline service; a tasks handler that panics + while building its future is a handler panic (`stopped()` runs). - **Breaking (0.8):** `DeferError` and `Delivery` are `#[non_exhaustive]` (like `JobOutcome`, `JobError` and `PipeErrorKind`): match them with a wildcard arm. From edb3006ec5fd34803b71cd90c30d256c36b0c242 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:03:19 -0300 Subject: [PATCH 113/208] docs: in_actor runs after started() and cannot replace a bypass inside it MIGRATION said `spawned_test::{tasks,threads}::in_actor` replaces `cfg(test)` bypasses; it runs in a turn, so only after `started()` returned, and a test cannot reach state that `started()` never creates (for example `if handshake().await.is_err() { ctx.stop() }`). Rather than add a way to start an actor without `started()` (it would change the lifecycle contract that `started()` always runs first), MIGRATION and the guide's `in_actor` row now say so and recommend keeping such work injectable: pass the handshake (or its outcome) in the constructor, or move it to a message the test does not send. --- docs/GUIDE.md | 2 +- docs/MIGRATION.md | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/GUIDE.md b/docs/GUIDE.md index cb30f999..e15cce61 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -1614,7 +1614,7 @@ These helpers fail with a report instead of hanging the test: | `wait_for_generation(&child, after, deadline)` | a supervised child does not reach a new generation, printing its snapshot | | `shutdown_supervisor(&supervisor, deadline)` | the supervisor report is incomplete, listing its children | | `shutdown_system(&system, deadline)` / `TestSystem::shutdown` | the `ActorSystem` report is incomplete, listing unfinished groups, actors, exits and failures | -| `in_actor(&actor, \|state, ctx\| ..)` | the actor is gone or the closure panicked; otherwise runs the closure in one of the actor's turns (after earlier sends) and returns its result, so a test can reach `ctx` (start a job, defer a reply) or inspect state without an ad hoc message or a `cfg(test)` bypass | +| `in_actor(&actor, \|state, ctx\| ..)` | the actor is gone or the closure panicked; otherwise runs the closure in one of the actor's turns (after earlier sends) and returns its result, so a test can reach `ctx` (start a job, defer a reply) or inspect state without an ad hoc message or a `cfg(test)` bypass. It runs only after `started()` returned: work in `started()` a test must skip belongs in the constructor or a message | `TestSystem::new()` owns an `ActorSystem` with a two-worker async group named `test`, and starts actors and mocks on it. The `threads` module has the diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 9ce5fd00..831abaeb 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -227,8 +227,14 @@ reply gives a `request` caller `ReplyDropped`; with it, the caller gets `Ok(())`. **Tests.** Replace test-only "run this closure in the actor" messages and -`cfg(test)` bypasses with `spawned_test::tasks::in_actor(&actor, |state, -ctx| ..)` (`threads::in_actor` in threads mode). +`cfg(test)` bypasses of handlers with `spawned_test::tasks::in_actor(&actor, +|state, ctx| ..)` (`threads::in_actor` in threads mode). `in_actor` runs in +a turn, so only after `started()` returned: it cannot replace a bypass +inside `started()` itself. If `started()` does work a test must skip (for +example `if handshake().await.is_err() { ctx.stop() }`), keep that work +injectable instead: pass the handshake (or its outcome) in the actor's +constructor, or move it to a message the test does not send, so the test +builds the post-handshake state directly. **Lifecycle.** At death the registry closes (open deferred tokens answer `ActorStopped`), then the mailbox is disposed, then actor state is dropped: From 3a2f57d1e16cf6794a6b716a30a1e515448f1ef9 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:03:20 -0300 Subject: [PATCH 114/208] docs(jobs): an emit_at instant already reached is an immediate emit `JobCtx::emit_at` / `emit_at_blocking` take a monotonic `Instant` and did not say what one already reached (or in the past) does, so client ports branched to `emit` themselves. It is defined as what the code does: the message is enqueued at the call, ahead of later emits, fenced and credit-accounted like any emit (and an `at` past the job's deadline is accepted and fenced by it). Documented on both methods and in the guide. Tests: deferred_api tasks_mode and threads_mode emit_at_a_past_instant_is_delivered_immediately. --- concurrency/src/tasks/jobs.rs | 6 ++++ concurrency/src/threads/jobs.rs | 5 ++++ concurrency/tests/deferred_api.rs | 47 +++++++++++++++++++++++++++++++ docs/GUIDE.md | 4 ++- 4 files changed, 61 insertions(+), 1 deletion(-) diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index 823abc1e..9630ec14 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -349,6 +349,12 @@ impl JobCtx { /// it stays last even when the work finishes first. Fails with /// [`JobGone`] at the call once the job is cancelled or its notification /// was taken. + /// + /// `at` is an instant of the monotonic deadline clock. One already + /// reached (or in the past) is an immediate emit: the message is + /// enqueued at the call, ahead of later emits, so there is no need to + /// branch to [`emit`](Self::emit). An `at` past `JobOptions::deadline` + /// is accepted; the deadline fences it (the emit is dropped at `at`). pub async fn emit_at(&self, at: crate::Instant, msg: M) -> Result<(), JobGone> where A: Handler, diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index 44548d49..b489d3f4 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -277,6 +277,11 @@ impl JobCtx { /// like an emit: dropped if the job was cancelled or replaced, or its /// deadline passed. The job's terminal message waits for its scheduled /// emits, so it stays last even when the work finishes first. + /// + /// `at` is an instant of the monotonic deadline clock. One already + /// reached (or in the past) is an immediate emit: the message is + /// enqueued at the call, ahead of later emits, so there is no need to + /// branch to [`emit_blocking`](Self::emit_blocking). pub fn emit_at_blocking(&self, at: crate::Instant, msg: M) -> Result<(), JobGone> where A: Handler, diff --git a/concurrency/tests/deferred_api.rs b/concurrency/tests/deferred_api.rs index 6e61e7a9..d96ff4cd 100644 --- a/concurrency/tests/deferred_api.rs +++ b/concurrency/tests/deferred_api.rs @@ -337,6 +337,30 @@ mod tasks_mode { actor.join().await; } + /// `emit_at` with an instant already reached (or in the past) is an + /// immediate emit: enqueued at the call, ahead of later emits. + #[spawned_macros::sim_test] + async fn emit_at_a_past_instant_is_delivered_immediately() { + let (actor, log) = start(); + in_actor(&actor, |_, ctx| { + ctx.pipe_with_ctx( + JobOptions::default(), + |job| async move { + let past = Instant::now(); + spawned_rt::tasks::sleep(Duration::from_millis(5)).await; + job.emit_at(past, Note("past".into())).await?; + job.emit(Note("later".into())).await + }, + |o| describe("job", o), + ) + .unwrap(); + }) + .await; + assert_eq!(wait_for(&log, 3).await, ["past", "later", "job:ok:Ok(())"]); + actor.context().stop(); + actor.join().await; + } + #[spawned_macros::sim_test] async fn send_handler_impls_answer_requests_with_unit() { let (actor, log) = start(); @@ -514,6 +538,29 @@ mod threads_mode { actor.join(); } + /// `emit_at_blocking` with an instant already reached is immediate. + #[test] + fn emit_at_a_past_instant_is_delivered_immediately() { + let log = Arc::new(Log::default()); + let actor = Owner { log: log.clone() }.start(); + in_actor(&actor, |_, ctx| { + ctx.pipe_blocking_with( + JobOptions::default(), + |job| { + let past = Instant::now(); + std::thread::sleep(Duration::from_millis(5)); + job.emit_at_blocking(past, Note("past".into()))?; + job.emit_blocking(Note("later".into())) + }, + |o| label("job", o), + ) + .unwrap(); + }); + assert_eq!(wait_for(&log, 3), ["past", "later", "job:Ok(Ok(()))"]); + actor.context().stop(); + actor.join(); + } + #[test] fn send_handler_impls_answer_requests_with_unit() { let log = Arc::new(Log::default()); diff --git a/docs/GUIDE.md b/docs/GUIDE.md index e15cce61..ec53d967 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -682,7 +682,9 @@ ctx.pipe_reply(token, async move { downstream?.await }, |res, token| Delegated { `JobCtx::emit_at(at, msg)` (`emit_at_blocking` in blocking jobs) delivers at `at`, fenced and credit-accounted the same way: it never reaches a newer job's turn or a newer instance, it is dropped if the job is cancelled or - replaced first, and the job's terminal message waits for it. + replaced first, and the job's terminal message waits for it. An `at` + already reached, or in the past, is an immediate emit (enqueued at the + call, ahead of later emits): no need to branch to `emit` yourself. - **Job deadlines are hard.** When `JobOptions::deadline` (or `deadline_after(d)`) passes, `done` gets `DeadlineExceeded` at once, later emits fail with `JobGone` and the work's eventual result is dropped. The From 839e9e9146c045073efcae6538fb88877be3c969 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:03:28 -0300 Subject: [PATCH 115/208] docs(jobs): make the native-job budget hard to miss; log its first refusal Now that tasks-mode blocking jobs run on Tokio's blocking pool, the default `MAX_NATIVE_JOBS` (`min(4, available_parallelism)`) is easy to outgrow (the ethlambda port needed 352), and under load the only symptom was `PipeErrorKind::NativeBudget`. The guide (a callout), MIGRATION, the `MAX_NATIVE_JOBS` docs of both modes, `#[actor(max_native_jobs)]` and `ChildSpec::max_native_jobs` now say so. The first refusal per actor instance logs a `native_budget_exhausted` warning naming the limit; only the refusal path touches its flag, so a successful reservation costs nothing more. Test: reply::jobs::tests::the_first_native_refusal_is_flagged_once. --- concurrency/src/reply/job_core.rs | 4 ++++ concurrency/src/reply/jobs.rs | 33 +++++++++++++++++++++++++++++++ concurrency/src/supervision.rs | 3 ++- concurrency/src/tasks/actor.rs | 6 ++++++ concurrency/src/threads/actor.rs | 6 ++++++ docs/GUIDE.md | 11 ++++++++++- docs/MIGRATION.md | 6 +++++- macros/src/lib.rs | 6 ++++-- 8 files changed, 70 insertions(+), 5 deletions(-) diff --git a/concurrency/src/reply/job_core.rs b/concurrency/src/reply/job_core.rs index b79071da..d5c619fa 100644 --- a/concurrency/src/reply/job_core.rs +++ b/concurrency/src/reply/job_core.rs @@ -371,6 +371,10 @@ impl JobSetCore { debug_assert!(previous > 0, "native job budget underflow"); } + pub(crate) fn max_native(&self) -> usize { + self.max_native + } + #[allow(dead_code)] // used by the Loom model pub(crate) fn native(&self) -> usize { self.native.load(Ordering::Acquire) diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index a5afdd53..e4687424 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -575,6 +575,8 @@ pub(crate) struct JobSet { /// Whether the owner's exit was recorded (lifecycle traces). #[cfg(feature = "lifecycle-trace")] exited: std::sync::atomic::AtomicBool, + /// The first `NativeBudget` refusal was logged (only refusals touch it). + native_refusal_logged: std::sync::atomic::AtomicBool, #[cfg(feature = "async")] idle_async: Mutex>, } @@ -586,6 +588,7 @@ impl JobSet { core: JobSetCore::new(max_native), #[cfg(feature = "lifecycle-trace")] exited: std::sync::atomic::AtomicBool::new(false), + native_refusal_logged: std::sync::atomic::AtomicBool::new(false), #[cfg(feature = "async")] idle_async: Mutex::new(Vec::new()), } @@ -596,10 +599,26 @@ impl JobSet { } /// Reserve a native slot (`MAX_NATIVE_JOBS`), rejecting at the bound. + /// The first refusal per instance is logged (a `warn`): under load the + /// default budget is easy to miss, and `NativeBudget` is otherwise the + /// only symptom. The reserving path itself is unchanged. pub(crate) fn reserve_native(&self) -> bool { let reserved = self.core.reserve_native(); if !reserved { crate::instrumentation::hooks::job_rejected(); + if !self + .native_refusal_logged + .swap(true, std::sync::atomic::Ordering::Relaxed) + { + let max_native_jobs = self.core.max_native(); + crate::actor_cleanup::log_safely(|| { + tracing::warn!(event = "native_budget_exhausted", actor = %self.owner, + max_native_jobs, "blocking job refused (NativeBudget): all native \ + job slots are in use; raise MAX_NATIVE_JOBS \ + (#[actor(max_native_jobs = N)] or ChildSpec::max_native_jobs) if \ + this is expected load. Logged once per actor instance"); + }); + } } reserved } @@ -990,6 +1009,20 @@ mod tests { assert_eq!(registry.outstanding(), 0); } + /// The first `NativeBudget` refusal per instance is flagged (and + /// logged); a reservation that succeeds never touches the flag. + #[test] + fn the_first_native_refusal_is_flagged_once() { + use std::sync::atomic::Ordering; + let set = JobSet::new(ActorId::next(), 1); + assert!(set.reserve_native()); + assert!(!set.native_refusal_logged.load(Ordering::Relaxed)); + assert!(!set.reserve_native()); + assert!(set.native_refusal_logged.load(Ordering::Relaxed)); + assert!(!set.reserve_native()); + set.release_native(); + } + /// Review P1: a finished keyed job stays replaceable, so its queued /// messages are fenced when a newer job takes the key. #[test] diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs index 8aa2ce3b..bfa858b3 100644 --- a/concurrency/src/supervision.rs +++ b/concurrency/src/supervision.rs @@ -303,7 +303,8 @@ impl ChildSpec { self } /// Override `Actor::MAX_NATIVE_JOBS` (blocking jobs running at once) for - /// every generation of this child. + /// every generation of this child. The default (at most 4) is easy to + /// outgrow; its only symptom under load is `NativeBudget` refusals. pub fn max_native_jobs(mut self, max: usize) -> Self { self.bounds.max_native_jobs = Some(max); self diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 3b49a95c..a6dc8997 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -80,6 +80,12 @@ pub trait Actor: Send + Sized + 'static { /// Native threads this instance's blocking jobs may run at once /// (`pipe_blocking*`). `None`: `min(4, available_parallelism)`. + /// + /// The default is small: an actor that runs many blocking jobs at once + /// (I/O, proofs, signature checks) usually needs more, and under load the + /// only symptom is `PipeErrorKind::NativeBudget` refusals (the first one + /// per instance is logged as a `native_budget_exhausted` warning). Set it + /// with `#[actor(max_native_jobs = N)]` or `ChildSpec::max_native_jobs`. const MAX_NATIVE_JOBS: Option = None; fn started(&mut self, _ctx: &Context) -> impl Future + Send { diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index d4c363ae..ef5cc7bd 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -48,6 +48,12 @@ pub trait Actor: Send + Sized + 'static { /// Native threads this instance's jobs may run at once. `None`: /// `min(4, available_parallelism)`. + /// + /// The default is small: an actor that runs many blocking jobs at once + /// usually needs more, and under load the only symptom is + /// `PipeErrorKind::NativeBudget` refusals (the first one per instance is + /// logged as a `native_budget_exhausted` warning). Set it with + /// `#[actor(max_native_jobs = N)]`. const MAX_NATIVE_JOBS: Option = None; fn started(&mut self, _ctx: &Context) {} diff --git a/docs/GUIDE.md b/docs/GUIDE.md index ec53d967..afe2758d 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -712,7 +712,16 @@ ctx.pipe_reply(token, async move { downstream?.await }, |res, token| Delegated { `--cfg spawned_sim`, so code need not depend on `spawned-rt`. - **Bounds.** Each unlinked job holds one `MAX_OUTSTANDING` credit; blocking jobs also count against `Actor::MAX_NATIVE_JOBS` (default - `min(4, available_parallelism)`). Set both in the macro, + `min(4, available_parallelism)`). + + > **Size the native budget.** The default of at most 4 blocking jobs per + > actor is easy to outgrow now that blocking jobs run on Tokio's blocking + > pool: an actor that verifies, proves or does I/O in blocking jobs + > usually needs tens or hundreds (one client port needed 352). Under load + > the only symptom is `PipeErrorKind::NativeBudget` refusals; the first one + > per actor instance is logged as a `native_budget_exhausted` warning. + + Set both in the macro, `#[actor(max_native_jobs = 64, max_outstanding = 4096)]` (or as `Actor` constants in a manual impl), and override them per supervised child with `ChildSpec::max_native_jobs(n)` / `ChildSpec::max_outstanding(n)`. diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 831abaeb..1664435a 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -200,7 +200,11 @@ that builds a `Snapshot` with a struct literal must add them. Tokio's blocking pool. Remove a `pipe_with(.., async { spawn_blocking(f) .await })` workaround (and its `resume_unwind` of the join error): `pipe_blocking_with(opts, move |_job| f(), done)` reports a panic as - `JobError::Panicked`. Raise the native budget with + `JobError::Panicked`. **Size the native budget:** the default is + `min(4, available_parallelism)` blocking jobs per actor, easy to outgrow + (one client port needed 352), and under load its only symptom is + `PipeErrorKind::NativeBudget` refusals (the first one per actor is logged + as `native_budget_exhausted`). Raise it with `#[actor(max_native_jobs = N)]` (or `ChildSpec::max_native_jobs(N)` for a supervised child) instead of avoiding native jobs. A Tokio runtime being dropped now waits for blocking jobs still running, as for any diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 4850e4dd..246a3f14 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1120,8 +1120,10 @@ fn is_actor_method_attribute(attr: &Attribute) -> bool { /// /// `max_outstanding = N` sets `Actor::MAX_OUTSTANDING` (deferred credits per /// instance) and `max_native_jobs = N` sets `Actor::MAX_NATIVE_JOBS` (blocking -/// jobs running at once). Both take any constant expression and combine with -/// `protocol`; a supervised child can still override them per `ChildSpec`: +/// jobs running at once; the default, `min(4, available_parallelism)`, is +/// easy to outgrow: under load the only symptom is `NativeBudget` refusals). +/// Both take any constant expression and combine with `protocol`; a +/// supervised child can still override them per `ChildSpec`: /// /// ```ignore /// #[actor(protocol = P2PProtocol, max_native_jobs = 64, max_outstanding = 4096)] From 2f6690504ee8bd74395d265b8e85d3ec0c536a72 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:03:38 -0300 Subject: [PATCH 116/208] docs(error): keep ReplyDropped non-transient and say how to ask for a retry `ActorError::is_transient()` covers `RequestTimeout` and `Overloaded`; the ethrex port also retries `ReplyDropped`. Decided to keep it out, on semantic grounds: the actor is alive and its handler chose not to answer (a decision or a bug, not a load condition), so a retry at the same target usually gets the same result; OTP has no such outcome (an unanswered call times out), and the variant exists to report the drop at once without promising a later success. The docs now say so and point to the transient way to say "not now, try again": answer `reply.overloaded()` (or `token.overloaded()`); a protocol that treats a drop as retryable matches `ReplyDropped` explicitly. Test: deferred_replies tasks_mode::a_dropped_reply_is_not_transient_but_overloaded_is (end to end; the variant table is also covered by error::tests::actor_error_classification_covers_every_variant). --- concurrency/src/error.rs | 8 +++++++- concurrency/tests/deferred_replies.rs | 28 +++++++++++++++++++++++++++ docs/GUIDE.md | 2 +- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/concurrency/src/error.rs b/concurrency/src/error.rs index 5a012aa0..0de1ceeb 100644 --- a/concurrency/src/error.rs +++ b/concurrency/src/error.rs @@ -148,7 +148,13 @@ impl ActorError { /// bound (`Overloaded`). `false` for a gone target, and for /// `ReplyDropped`, where the actor is alive but its handler chose not to /// answer (a decision or a bug, not a load condition): a retry would - /// usually get the same result. + /// usually get the same result. OTP has no such outcome (a `call` the + /// server never answers times out); the variant reports a drop at once + /// so a caller does not wait for its deadline, without claiming that the + /// same request will succeed later. A handler that means "not now, try + /// again" answers `reply.overloaded()` (or `token.overloaded()`), which is + /// transient; a caller whose protocol treats a drop as retryable matches + /// `ReplyDropped` explicitly. pub fn is_transient(&self) -> bool { matches!(self, ActorError::RequestTimeout | ActorError::Overloaded) } diff --git a/concurrency/tests/deferred_replies.rs b/concurrency/tests/deferred_replies.rs index bfedc5ed..621dec17 100644 --- a/concurrency/tests/deferred_replies.rs +++ b/concurrency/tests/deferred_replies.rs @@ -480,6 +480,34 @@ mod tasks_mode { actor.join().await; } + /// `ReplyDropped` (the handler chose not to answer) is not transient; + /// `overloaded()`, a handler's "not now, try again", is. + #[spawned_macros::sim_test] + async fn a_dropped_reply_is_not_transient_but_overloaded_is() { + struct Picky; + impl Actor for Picky {} + struct Ask(bool); + impl Message for Ask { + type Result = (); + } + impl Handler for Picky { + async fn handle(&mut self, msg: Ask, reply: PendingReply<'_, ()>, _: &Context) { + if msg.0 { + reply.overloaded(); + } // otherwise dropped unanswered + } + } + let actor = Picky.start(); + let dropped = actor.request(Ask(false)).await.unwrap_err(); + assert!(matches!(dropped, ActorError::ReplyDropped)); + assert!(!dropped.is_transient() && !dropped.target_gone()); + let busy = actor.request(Ask(true)).await.unwrap_err(); + assert!(matches!(busy, ActorError::Overloaded)); + assert!(busy.is_transient()); + actor.context().stop(); + actor.join().await; + } + #[spawned_macros::sim_test] async fn request_timeout_starts_at_send() { let (actor, _) = start(); diff --git a/docs/GUIDE.md b/docs/GUIDE.md index afe2758d..07dad401 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -795,7 +795,7 @@ pub enum ActorError { | `ReplyDropped` | `false` | `false` | `false` | `None` | | `Overloaded` | `false` | `false` | `true` | `None` | -`target_gone()`: retrying the same reference cannot succeed (look the actor up again; a supervisor may have restarted it). `is_transient()`: retrying the same target may succeed. `ReplyDropped` is neither: the actor is alive but its handler chose not to answer. `ExitReason::is_abnormal()` is `false` only for `Normal` and `Shutdown` (`Killed` and `NoProc` are abnormal). +`target_gone()`: retrying the same reference cannot succeed (look the actor up again; a supervisor may have restarted it). `is_transient()`: retrying the same target may succeed. `ReplyDropped` is neither: the actor is alive but its handler chose not to answer (a handler that means "not now, try again" answers `reply.overloaded()`, which is transient; a caller whose protocol retries dropped requests matches `ReplyDropped` explicitly). `ExitReason::is_abnormal()` is `false` only for `Normal` and `Shutdown` (`Killed` and `NoProc` are abnormal). --- From 710095d1e0ea1b81f0b54d26e3688113e24e3e1c Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:03:38 -0300 Subject: [PATCH 117/208] fix(macros): #[send_handler] works when spawned-concurrency is renamed `#[send_handler]` hard-coded `spawned_concurrency::reply::PendingReply` in the generated `handle`, which does not resolve in a crate that renames the dependency. The reply type is now named through the `Handler` path when it is qualified (`impl my_spawned::tasks::Handler` uses `my_spawned::tasks::PendingReply`, re-exported by both modes); with a bare `Handler` a renamed crate passes it explicitly, `#[send_handler(crate = my_spawned)]` (serde-style). The default is unchanged. `#[actor]` and `#[protocol]` hard-code `spawned_concurrency::` too (no macro resolves renames today); that is left as a follow-up. Test: trybuild ui-pass/send_handler_renamed.rs (both forms, with a local `spawned_concurrency` module shadowing the original name so a hard-coded path fails to resolve). --- .../tests/ui-pass/send_handler_renamed.rs | 37 ++++++++++++ macros/src/lib.rs | 56 ++++++++++++++++--- 2 files changed, 84 insertions(+), 9 deletions(-) create mode 100644 concurrency/tests/ui-pass/send_handler_renamed.rs diff --git a/concurrency/tests/ui-pass/send_handler_renamed.rs b/concurrency/tests/ui-pass/send_handler_renamed.rs new file mode 100644 index 00000000..478572cf --- /dev/null +++ b/concurrency/tests/ui-pass/send_handler_renamed.rs @@ -0,0 +1,37 @@ +// `#[send_handler]` in a crate that renames `spawned-concurrency`: the +// local module shadows the original name, so a hard-coded +// `spawned_concurrency::` path in the expansion fails to resolve. +extern crate spawned_concurrency as my_spawned; + +#[allow(dead_code)] +mod spawned_concurrency {} + +use my_spawned::message::Message; +use my_spawned::send_handler; +use my_spawned::tasks::{Actor, Context, Handler}; + +struct Poller; +impl Actor for Poller {} + +struct Tick; +impl Message for Tick { + type Result = (); +} +struct Tock; +impl Message for Tock { + type Result = (); +} + +// A qualified `Handler` path names the reply type through its module. +#[send_handler] +impl my_spawned::tasks::Handler for Poller { + async fn handle(&mut self, _msg: Tick, _ctx: &Context) {} +} + +// A bare `Handler` takes the renamed crate explicitly. +#[send_handler(crate = my_spawned)] +impl Handler for Poller { + async fn handle(&mut self, _msg: Tock, _ctx: &Context) {} +} + +fn main() {} diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 246a3f14..9f10af67 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1732,16 +1732,39 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { /// `Ok(())`, not `ReplyDropped`, and a panic in the body answers /// `ActorStopped` as for any handler. The message's `Result` must be `()`. /// Works for `tasks` (`async fn`) and `threads` (`fn`) handlers. +/// +/// The reply type is named through the `Handler` path when it is qualified +/// (`impl my_spawned::tasks::Handler for A` uses +/// `my_spawned::tasks::PendingReply`), so a renamed dependency works. +/// With a bare `Handler`, it is `spawned_concurrency::reply::PendingReply`; +/// a crate that renames `spawned-concurrency` names it: +/// `#[send_handler(crate = my_spawned)]`. #[proc_macro_attribute] pub fn send_handler(attr: TokenStream, item: TokenStream) -> TokenStream { - if !attr.is_empty() { - return syn::Error::new( - proc_macro2::Span::call_site(), - "#[send_handler] takes no arguments", - ) - .to_compile_error() - .into(); - } + let krate: Option = if attr.is_empty() { + None + } else { + let parser = |input: syn::parse::ParseStream| { + input.parse::()?; + input.parse::()?; + let path: syn::Path = input.parse()?; + if !input.is_empty() { + return Err(input.error("expected only `crate = path`")); + } + Ok(path) + }; + match syn::parse::Parser::parse(parser, attr) { + Ok(path) => Some(path), + Err(error) => { + return syn::Error::new( + error.span(), + format!("#[send_handler] takes no arguments or `crate = path`: {error}"), + ) + .to_compile_error() + .into() + } + } + }; let mut impl_block = parse_macro_input!(item as ItemImpl); let is_handler = impl_block .trait_ @@ -1784,8 +1807,23 @@ pub fn send_handler(attr: TokenStream, item: TokenStream) -> TokenStream { } } let reply = format_ident!("__spawned_reply"); + // The `Handler` path's module (`tasks`/`threads`) re-exports + // `PendingReply`; a bare `Handler` falls back to the crate path. + let handler_module = impl_block.trait_.as_ref().and_then(|(_, path, _)| { + (path.segments.len() > 1).then(|| { + let mut module = path.clone(); + module.segments.pop(); + module.segments.pop_punct(); + module + }) + }); + let reply_ty = match (krate, handler_module) { + (Some(krate), _) => quote! { #krate::reply::PendingReply<'_, ()> }, + (None, Some(module)) => quote! { #module::PendingReply<'_, ()> }, + (None, None) => quote! { spawned_concurrency::reply::PendingReply<'_, ()> }, + }; let reply_param: FnArg = syn::parse_quote! { - #reply: spawned_concurrency::reply::PendingReply<'_, ()> + #reply: #reply_ty }; method.sig.inputs.insert(2, reply_param); method.sig.output = ReturnType::Default; From 24de8323463a663dd58d33128c23bffe39dc9ec3 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:07:39 -0300 Subject: [PATCH 118/208] fix(examples): if-let for the deferred_replies example's defer (clippy) --- examples/capabilities/src/bin/deferred_replies.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/examples/capabilities/src/bin/deferred_replies.rs b/examples/capabilities/src/bin/deferred_replies.rs index e2efd547..445be929 100644 --- a/examples/capabilities/src/bin/deferred_replies.rs +++ b/examples/capabilities/src/bin/deferred_replies.rs @@ -246,11 +246,10 @@ impl PeerTable { if self.waiters.len() >= MAX_WAITERS { return reply.overloaded(); } - match reply.defer() { - Ok(token) => self.waiters.push_back(token), - // Overloaded, Closed or NoCaller: the caller was already answered - // (or there is none). `DeferError` is `#[non_exhaustive]`. - Err(_) => {} + // On `Err` (Overloaded, Closed or NoCaller) the caller was already + // answered, or there is none. + if let Ok(token) = reply.defer() { + self.waiters.push_back(token); } } From ca6b6d606998f73cb774ef48e254811f4a126e6a Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:10:48 -0300 Subject: [PATCH 119/208] test(death): keep the tasks-only Park message in the tasks module (no-default-features clippy) --- concurrency/tests/death_boundary.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/concurrency/tests/death_boundary.rs b/concurrency/tests/death_boundary.rs index fdbdaa64..079398b4 100644 --- a/concurrency/tests/death_boundary.rs +++ b/concurrency/tests/death_boundary.rs @@ -55,11 +55,6 @@ struct Block(mpsc::Receiver<()>, mpsc::Sender<()>); impl Message for Block { type Result = (); } -/// Defers and keeps the token in the actor's state. -struct Park; -impl Message for Park { - type Result = (); -} /// Wait until `poll` yields, at most `LIMIT`. fn eventually(what: &str, mut poll: impl FnMut() -> Option) -> T { @@ -79,6 +74,12 @@ mod tasks_mode { use spawned_concurrency::tasks::{Actor, ActorStart, Context, Handler}; use spawned_concurrency::{PendingReply, ReplyTo}; + /// Defers and keeps the token in the actor's state. + struct Park; + impl Message for Park { + type Result = (); + } + struct Owner { _state: Gated, parked: Vec>, From 609f00a75e84cdd3bfb65f816a9b5cacf2ea5686 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:10:48 -0300 Subject: [PATCH 120/208] test(sim): compile stack_budget out under spawned_sim Same change as main's #243 (the sim CI job): the test measures real OS thread stacks in child processes, and the simulator runs every task on one thread, so the budgets do not apply there. Without it the sim job fails on this branch once it reaches main. --- concurrency/tests/stack_budget.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/concurrency/tests/stack_budget.rs b/concurrency/tests/stack_budget.rs index 48dedf71..0a34620b 100644 --- a/concurrency/tests/stack_budget.rs +++ b/concurrency/tests/stack_budget.rs @@ -34,6 +34,9 @@ //! aarch64-linux glibc's 128 KiB thread minimum hides every scenario, so the //! larger x86_64 budgets apply there and to other targets. `--features //! metrics` measures the same debug minimums on macOS. +// Measures real OS thread stacks in child processes; the simulator runs +// every task on one thread, so the budgets do not apply there. +#![cfg(not(spawned_sim))] #[path = "support/drop_subprocess.rs"] mod subprocess; From 46cbef62fa19565f6dcc841a8350f361a5a24fb1 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:42:34 -0300 Subject: [PATCH 121/208] test: port #246's semantics_gaps and sim_ordering to the 0.8 API and semantics #246 was written against main's 0.7 API. Mechanical port: manual handlers take their `PendingReply` and answer with the value they used to return; `ActorStopped` carries the final reason. Assertions that encoded 0.7 behaviour, updated to the 0.8 contract (the code is right, each is a documented 0.8 change): - A send or request after the death point is `NoProc`, not `ActorStopped` (D3; NoProc past D, exit-signals design). The R1 outcome checks accept `NoProc` for requests that were never admitted. - A queued request's reply is "released before join()" by being answered `ActorStopped(reason)` at the death point (D4/L7, the death boundary), not by a dropped oneshot sender. - System shutdown is the parent's exit signal (E3): a non-trapping actor ends `Shutdown` without `stopped()`. The two tests whose probe runs in `stopped()` now trap exits, as an actor whose cleanup must run does. - At the system grace the coordinator signals the earlier actor with `Shutdown` and then kills what remains (E3 escalation): the earlier actor ends `Shutdown` or `Killed` (whichever wins), the later one, still in `stopped()`, finishes and may end `Killed` (a kill supersedes the reason until D, X4). - Under the simulator, `timeout` needs a `Future`: the `ReplyReceiver` is turned into one with `IntoFuture`. Both suites pass in the normal build (three runs) and under the simulator (full sim run: 236 passed). --- concurrency/tests/semantics_gaps.rs | 554 +++++++++++++++++++++------- concurrency/tests/sim_ordering.rs | 204 +++++++--- 2 files changed, 569 insertions(+), 189 deletions(-) diff --git a/concurrency/tests/semantics_gaps.rs b/concurrency/tests/semantics_gaps.rs index 8698cf08..34665c6e 100644 --- a/concurrency/tests/semantics_gaps.rs +++ b/concurrency/tests/semantics_gaps.rs @@ -678,9 +678,13 @@ fn check_turns(events: &[Turn], counter: usize, outcomes: &[(usize, Result { + Err(ActorError::ActorStopped(_)) => { assert!(!handled.contains(id), "R1: {id} ran but got ActorStopped") } + // Sent after the death point (0.8: `NoProc`, never admitted). + Err(ActorError::NoProc) => { + assert!(!handled.contains(id), "R1: {id} ran but got NoProc") + } Err(other) => panic!("R1: unexpected {other:?} for {id}"), } } @@ -715,21 +719,45 @@ mod native_contract { } } impl Handler for Holder { - fn handle(&mut self, hold: Hold, _: &Context) { - self.seen.lock().unwrap().push("hold"); - hold.entered.send(()).unwrap(); - let _ = hold.release.recv_timeout(LIMIT); + fn handle( + &mut self, + hold: Hold, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = { + self.seen.lock().unwrap().push("hold"); + hold.entered.send(()).unwrap(); + let _ = hold.release.recv_timeout(LIMIT); + }; + __reply.reply(__value); } } impl Handler for Holder { - fn handle(&mut self, _: Queued, _: &Context) { - self.seen.lock().unwrap().push("queued"); + fn handle( + &mut self, + _: Queued, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = { + self.seen.lock().unwrap().push("queued"); + }; + __reply.reply(__value); } } impl Handler for Holder { - fn handle(&mut self, _: Asked, _: &Context) -> u64 { - self.seen.lock().unwrap().push("asked"); - 1 + fn handle( + &mut self, + _: Asked, + __reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __value: u64 = { + self.seen.lock().unwrap().push("asked"); + 1 + }; + __reply.reply(__value); } } @@ -773,10 +801,9 @@ mod native_contract { WINDOW + 1, "D4/L7: a queued payload outlived join()" ); - for reply in replies { - assert_eq!( - reply.try_recv(), - Err(mpsc::TryRecvError::Disconnected), + for mut reply in replies { + assert!( + matches!(reply.try_recv(), Some(Err(ActorError::ActorStopped(_)))), "D4/L7: a queued reply sender outlived join()" ); } @@ -789,7 +816,7 @@ mod native_contract { assert_eq!(actor.wait_exit(), ExitReason::Normal); assert!(matches!( actor.send(Queued(Payload(drops.clone()))), - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) )); } @@ -817,13 +844,21 @@ mod native_contract { } } impl Handler for Turns { - fn handle(&mut self, Work(id): Work, _: &Context) -> usize { - self.push(Turn::Begin(id)); - let seen = self.counter; - std::thread::yield_now(); - self.counter = seen + 1; - self.push(Turn::End(id)); - id + fn handle( + &mut self, + Work(id): Work, + __reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + let __value: usize = { + self.push(Turn::Begin(id)); + let seen = self.counter; + std::thread::yield_now(); + self.counter = seen + 1; + self.push(Turn::End(id)); + id + }; + __reply.reply(__value); } } @@ -877,19 +912,41 @@ mod native_contract { struct Backlog(Vec); impl Actor for Backlog {} impl Handler for Backlog { - fn handle(&mut self, hold: Hold, _: &Context) { - hold.entered.send(()).unwrap(); - let _ = hold.release.recv_timeout(LIMIT * 6); + fn handle( + &mut self, + hold: Hold, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = { + hold.entered.send(()).unwrap(); + let _ = hold.release.recv_timeout(LIMIT * 6); + }; + __reply.reply(__value); } } impl Handler for Backlog { - fn handle(&mut self, Seq(i): Seq, _: &Context) { - self.0.push(i); + fn handle( + &mut self, + Seq(i): Seq, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = { + self.0.push(i); + }; + __reply.reply(__value); } } impl Handler for Backlog { - fn handle(&mut self, _: Count, _: &Context) -> Vec { - std::mem::take(&mut self.0) + fn handle( + &mut self, + _: Count, + __reply: spawned_concurrency::PendingReply<'_, Vec>, + _: &Context, + ) { + let __value: Vec = std::mem::take(&mut self.0); + __reply.reply(__value); } } let actor = Backlog(Vec::new()).start(); @@ -927,12 +984,20 @@ mod native_contract { } impl Actor for Looper {} impl Handler for Looper { - fn handle(&mut self, _: Loop, ctx: &Context) { - self.count += 1; - let _ = self.reports.send(self.count); - if self.count < 100 { - ctx.send(Loop).unwrap(); - } + fn handle( + &mut self, + _: Loop, + __reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + let __value: () = { + self.count += 1; + let _ = self.reports.send(self.count); + if self.count < 100 { + ctx.send(Loop).unwrap(); + } + }; + __reply.reply(__value); } } @@ -1053,21 +1118,48 @@ mod tasks_contract { } } impl Handler for Holder { - async fn handle(&mut self, hold: Hold, _: &Context) { - self.seen.lock().unwrap().push("hold"); - hold.entered.send(()).unwrap(); - let _ = timeout(LIMIT, hold.release).await; + async fn handle( + &mut self, + hold: Hold, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + self.seen.lock().unwrap().push("hold"); + hold.entered.send(()).unwrap(); + let _ = timeout(LIMIT, hold.release).await; + } + .await; + __reply.reply(__value); } } impl Handler for Holder { - async fn handle(&mut self, _: Queued, _: &Context) { - self.seen.lock().unwrap().push("queued"); + async fn handle( + &mut self, + _: Queued, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + self.seen.lock().unwrap().push("queued"); + } + .await; + __reply.reply(__value); } } impl Handler for Holder { - async fn handle(&mut self, _: Asked, _: &Context) -> u64 { - self.seen.lock().unwrap().push("asked"); - 1 + async fn handle( + &mut self, + _: Asked, + __reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __value: u64 = async { + self.seen.lock().unwrap().push("asked"); + 1 + } + .await; + __reply.reply(__value); } } @@ -1115,7 +1207,7 @@ mod tasks_contract { ); for mut reply in replies { assert!( - matches!(reply.try_recv(), Err(oneshot::error::TryRecvError::Closed)), + matches!(reply.try_recv(), Some(Err(ActorError::ActorStopped(_)))), "D4/L7 {place:?}: a queued reply sender outlived join()" ); } @@ -1128,7 +1220,7 @@ mod tasks_contract { assert_eq!(actor.wait_exit().await, ExitReason::Normal, "{place:?}"); assert!(matches!( actor.send(Queued(Payload(drops.clone()))), - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) )); } placement.finish().await; @@ -1160,14 +1252,23 @@ mod tasks_contract { impl Handler for Turns { /// A read-modify-write that spans an await: it loses updates unless /// the handler keeps the turn while it awaits (M1). - async fn handle(&mut self, Work(id): Work, _: &Context) -> usize { - self.push(Turn::Begin(id)); - let seen = self.counter; - spawned_rt::tasks::yield_now().await; - spawned_rt::tasks::yield_now().await; - self.counter = seen + 1; - self.push(Turn::End(id)); - id + async fn handle( + &mut self, + Work(id): Work, + __reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + let __value: usize = async { + self.push(Turn::Begin(id)); + let seen = self.counter; + spawned_rt::tasks::yield_now().await; + spawned_rt::tasks::yield_now().await; + self.counter = seen + 1; + self.push(Turn::End(id)); + id + } + .await; + __reply.reply(__value); } } @@ -1229,19 +1330,43 @@ mod tasks_contract { struct Backlog(Vec); impl Actor for Backlog {} impl Handler for Backlog { - async fn handle(&mut self, hold: Hold, _: &Context) { - hold.entered.send(()).unwrap(); - let _ = timeout(LIMIT * 6, hold.release).await; + async fn handle( + &mut self, + hold: Hold, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + hold.entered.send(()).unwrap(); + let _ = timeout(LIMIT * 6, hold.release).await; + } + .await; + __reply.reply(__value); } } impl Handler for Backlog { - async fn handle(&mut self, Seq(i): Seq, _: &Context) { - self.0.push(i); + async fn handle( + &mut self, + Seq(i): Seq, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + self.0.push(i); + } + .await; + __reply.reply(__value); } } impl Handler for Backlog { - async fn handle(&mut self, _: Count, _: &Context) -> Vec { - std::mem::take(&mut self.0) + async fn handle( + &mut self, + _: Count, + __reply: spawned_concurrency::PendingReply<'_, Vec>, + _: &Context, + ) { + let __value: Vec = async { std::mem::take(&mut self.0) }.await; + __reply.reply(__value); } } @@ -1292,12 +1417,21 @@ mod tasks_contract { } impl Actor for Looper {} impl Handler for Looper { - async fn handle(&mut self, _: Loop, ctx: &Context) { - self.count += 1; - let _ = self.reports.send(self.count); - if self.count < 100 { - ctx.send(Loop).unwrap(); + async fn handle( + &mut self, + _: Loop, + __reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + let __value: () = async { + self.count += 1; + let _ = self.reports.send(self.count); + if self.count < 100 { + ctx.send(Loop).unwrap(); + } } + .await; + __reply.reply(__value); } } @@ -1376,12 +1510,21 @@ mod tasks_contract { } } impl Handler for Victim { - async fn handle(&mut self, Work(id): Work, _: &Context) -> usize { - self.handled.fetch_add(1, Ordering::SeqCst); - if self.fault == Fault::Handler { - panic!("victim handler"); + async fn handle( + &mut self, + Work(id): Work, + __reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + let __value: usize = async { + self.handled.fetch_add(1, Ordering::SeqCst); + if self.fault == Fault::Handler { + panic!("victim handler"); + } + id } - id + .await; + __reply.reply(__value); } } @@ -1410,7 +1553,7 @@ mod tasks_contract { actor.child_handle().stop(); } else { assert!( - matches!(first, Err(ActorError::ActorStopped)), + matches!(first, Err(ActorError::ActorStopped(_))), "{place:?} {fault:?}: {first:?}" ); } @@ -1453,9 +1596,18 @@ mod tasks_contract { } } impl Handler for Gate { - async fn handle(&mut self, Work(id): Work, _: &Context) -> usize { - self.handled.fetch_add(1, Ordering::SeqCst); - id + async fn handle( + &mut self, + Work(id): Work, + __reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + let __value: usize = async { + self.handled.fetch_add(1, Ordering::SeqCst); + id + } + .await; + __reply.reply(__value); } } @@ -1518,17 +1670,35 @@ mod tasks_contract { struct Timed; impl Actor for Timed {} impl Handler for Timed { - async fn handle(&mut self, _: Late, _: &Context) { - panic!("T1: a timer fired after its actor stopped"); + async fn handle( + &mut self, + _: Late, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + panic!("T1: a timer fired after its actor stopped"); + } + .await; + __reply.reply(__value); } } impl Handler for Timed { - async fn handle(&mut self, Arm(slot): Arm, ctx: &Context) { - spawned_concurrency::tasks::send_after( - Duration::from_secs(3600), - ctx.clone(), - Late(WhereDropped(slot)), - ); + async fn handle( + &mut self, + Arm(slot): Arm, + __reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + let __value: () = async { + spawned_concurrency::tasks::send_after( + Duration::from_secs(3600), + ctx.clone(), + Late(WhereDropped(slot)), + ); + } + .await; + __reply.reply(__value); } } @@ -1574,8 +1744,17 @@ mod tasks_contract { type Result = (); } impl Handler for Napper { - async fn handle(&mut self, Nap(d): Nap, _: &Context) { - tokio::time::sleep(d).await; + async fn handle( + &mut self, + Nap(d): Nap, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + tokio::time::sleep(d).await; + } + .await; + __reply.reply(__value); } } @@ -1623,18 +1802,27 @@ mod tasks_contract { struct Spawner; impl Actor for Spawner {} impl Handler for Spawner { - async fn handle(&mut self, spawn: Spawn, _: &Context) { - let probe = Payload(spawn.task_drops); - drop(tokio::spawn(async move { - let _probe = probe; - std::future::pending::<()>().await; - })); - let release = spawn.thread_release; - let done = spawn.thread_done; - drop(std::thread::spawn(move || { - let _ = release.recv(); - done.fetch_add(1, Ordering::SeqCst); - })); + async fn handle( + &mut self, + spawn: Spawn, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + let probe = Payload(spawn.task_drops); + drop(tokio::spawn(async move { + let _probe = probe; + std::future::pending::<()>().await; + })); + let release = spawn.thread_release; + let done = spawn.thread_done; + drop(std::thread::spawn(move || { + let _ = release.recv(); + done.fetch_add(1, Ordering::SeqCst); + })); + } + .await; + __reply.reply(__value); } } let system = ActorSystem::new().unwrap(); @@ -1739,33 +1927,78 @@ mod supervision_contract { } impl Actor for Slot {} impl Handler for Slot { - async fn handle(&mut self, hold: Hold, _: &Context) { - let _ = hold.entered.send(()); - let _ = hold.release.await; - panic!("crash with a queue behind it"); + async fn handle( + &mut self, + hold: Hold, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + let _ = hold.entered.send(()); + let _ = hold.release.await; + panic!("crash with a queue behind it"); + } + .await; + __reply.reply(__value); } } impl Handler for Slot { - async fn handle(&mut self, _: Incr, _: &Context) { - self.n += 1; - self.handled.fetch_add(1, Ordering::SeqCst); + async fn handle( + &mut self, + _: Incr, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + self.n += 1; + self.handled.fetch_add(1, Ordering::SeqCst); + } + .await; + __reply.reply(__value); } } impl Handler for Slot { - async fn handle(&mut self, _: Next, _: &Context) -> usize { - self.n += 1; - self.n + async fn handle( + &mut self, + _: Next, + __reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + let __value: usize = async { + self.n += 1; + self.n + } + .await; + __reply.reply(__value); } } impl Handler for Slot { - async fn handle(&mut self, _: Crash, _: &Context) { - panic!("injected crash"); + async fn handle( + &mut self, + _: Crash, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + panic!("injected crash"); + } + .await; + __reply.reply(__value); } } impl Handler for Slot { - async fn handle(&mut self, _: Slow, _: &Context) { - self.handled.fetch_add(1, Ordering::SeqCst); - sleep(Duration::from_millis(10)).await; + async fn handle( + &mut self, + _: Slow, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + self.handled.fetch_add(1, Ordering::SeqCst); + sleep(Duration::from_millis(10)).await; + } + .await; + __reply.reply(__value); } } @@ -1829,7 +2062,10 @@ mod supervision_contract { let new = next_generation(&child, generation).await; for reply in replies { assert!( - timeout(LIMIT, reply).await.unwrap().is_err(), + timeout(LIMIT, std::future::IntoFuture::into_future(reply)) + .await + .unwrap() + .is_err(), "V4: a queued request was answered" ); } @@ -1839,7 +2075,7 @@ mod supervision_contract { "V4: state or queue replayed" ); assert_eq!(handled.load(Ordering::SeqCst), 0, "V4: a queued Incr ran"); - assert!(matches!(old.send(Incr), Err(ActorError::ActorStopped))); + assert!(matches!(old.send(Incr), Err(ActorError::NoProc))); assert_eq!(calls.load(Ordering::SeqCst), 2); assert!(supervisor.shutdown(LIMIT).await.complete); assert!(system.shutdown(LIMIT).await.complete); @@ -1995,29 +2231,56 @@ mod supervision_contract { } impl Actor for Late {} impl Handler for Late { - async fn handle(&mut self, _: Fired, _: &Context) { - self.fired.fetch_add(1, Ordering::SeqCst); + async fn handle( + &mut self, + _: Fired, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + self.fired.fetch_add(1, Ordering::SeqCst); + } + .await; + __reply.reply(__value); } } impl Handler for Late { - async fn handle(&mut self, _: Down, _: &Context) { - self.downs.fetch_add(1, Ordering::SeqCst); + async fn handle( + &mut self, + _: Down, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + self.downs.fetch_add(1, Ordering::SeqCst); + } + .await; + __reply.reply(__value); } } impl Handler for Late { /// Called while system shutdown is in progress but this actor is /// still running: timers, monitors and registrations are inert. - async fn handle(&mut self, probe: Probe, ctx: &Context) -> (bool, bool) { - send_after(Duration::from_millis(1), ctx.clone(), Fired); - ctx.monitor(&probe.target); - let registered = ctx.register_name("semantics-gaps-late-name").is_ok(); - let started = Late { - fired: self.fired.clone(), - downs: self.downs.clone(), + async fn handle( + &mut self, + probe: Probe, + __reply: spawned_concurrency::PendingReply<'_, (bool, bool)>, + ctx: &Context, + ) { + let __value: (bool, bool) = async { + send_after(Duration::from_millis(1), ctx.clone(), Fired); + ctx.monitor(&probe.target); + let registered = ctx.register_name("semantics-gaps-late-name").is_ok(); + let started = Late { + fired: self.fired.clone(), + downs: self.downs.clone(), + } + .start_on(&ctx.execution_group().unwrap()) + .is_ok(); + (registered, started) } - .start_on(&ctx.execution_group().unwrap()) - .is_ok(); - (registered, started) + .await; + __reply.reply(__value); } } /// Stops first (admitted last); probes `late` from its `stopped()` and @@ -2028,6 +2291,11 @@ mod supervision_contract { result: Arc>>, } impl Actor for Prober { + // 0.8 (E3): the system's shutdown is its parent's exit signal; a + // trapping actor ends through `stopped()`, where the probe runs. + async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); + } async fn stopped(&mut self, ctx: &Context) { let probe = Probe { target: ctx.actor_ref().child_handle(), @@ -2085,6 +2353,10 @@ mod supervision_contract { release: Option>, } impl Actor for Gated { + // 0.8 (E3): shutdown reaches a trapping actor through `stopped()`. + async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &Context) { if let Some(stopping) = self.stopping.take() { let _ = stopping.send(()); @@ -2096,7 +2368,8 @@ mod supervision_contract { } /// S3, S2: dropping an `ActorSystem` does not block, stops the latest - /// admitted actor first, and signals earlier ones only at the 5 s grace. + /// admitted actor first, and signals earlier ones only at the 5 s grace + /// (with the parent's kill, E3). #[spawned_macros::sim_test] async fn dropping_a_system_shuts_down_in_order_with_a_five_second_grace() { let system = ActorSystem::new().unwrap(); @@ -2132,17 +2405,24 @@ mod supervision_contract { "S2/S3: earlier actor signalled before the later one finished or the grace" ); } - assert_eq!( - timeout(Duration::from_secs(3), earlier.wait_exit()) - .await - .expect("S3: earlier actor not signalled at the 5 s grace"), - ExitReason::Shutdown + let earlier_reason = timeout(Duration::from_secs(3), earlier.wait_exit()) + .await + .expect("S3: earlier actor not signalled at the 5 s grace"); + // 0.8 (E3): at the grace the earlier actor gets the parent's + // `Shutdown`, then every remaining actor the parent's kill; a + // trapping actor that finishes its parent exit first ends `Shutdown`. + assert!( + matches!(earlier_reason, ExitReason::Shutdown | ExitReason::Killed), + "{earlier_reason:?}" ); assert!(later.child_handle().is_alive()); release.send(()).unwrap(); - assert_eq!( - timeout(LIMIT, later.wait_exit()).await.unwrap(), - ExitReason::Shutdown + // `later` is inside `stopped()`: it finishes; a kill that reached it + // before its death point supersedes its reason (X4). + let later_reason = timeout(LIMIT, later.wait_exit()).await.unwrap(); + assert!( + matches!(later_reason, ExitReason::Shutdown | ExitReason::Killed), + "{later_reason:?}" ); } } diff --git a/concurrency/tests/sim_ordering.rs b/concurrency/tests/sim_ordering.rs index 13ccef97..c08115e1 100644 --- a/concurrency/tests/sim_ordering.rs +++ b/concurrency/tests/sim_ordering.rs @@ -25,7 +25,6 @@ use std::sync::{ Arc, Mutex, }; use std::time::Duration; -use tokio::sync::oneshot::error::TryRecvError; const LIMIT: Duration = Duration::from_secs(5); @@ -67,17 +66,35 @@ impl Message for Echo { type Result = (usize, usize); } impl Handler for Sink { - async fn handle(&mut self, Put(sender, seq): Put, _: &Context) { - push(&self.0, (sender, seq)); + async fn handle( + &mut self, + Put(sender, seq): Put, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + push(&self.0, (sender, seq)); + } + .await; + __reply.reply(__value); } } impl Handler for Sink { - async fn handle(&mut self, Echo(sender, seq): Echo, _: &Context) -> (usize, usize) { - push(&self.0, (sender, seq)); - // Hold the turn across an await so a reply could be misrouted only - // if replies were not bound to their own request. - yield_now().await; - (sender, seq) + async fn handle( + &mut self, + Echo(sender, seq): Echo, + __reply: spawned_concurrency::PendingReply<'_, (usize, usize)>, + _: &Context, + ) { + let __value: (usize, usize) = async { + push(&self.0, (sender, seq)); + // Hold the turn across an await so a reply could be misrouted only + // if replies were not bound to their own request. + yield_now().await; + (sender, seq) + } + .await; + __reply.reply(__value); } } @@ -192,11 +209,20 @@ impl Message for Work { type Result = usize; } impl Handler for Turns { - async fn handle(&mut self, Work(id): Work, _: &Context) -> usize { - push(&self.0, Turn::Begin(id)); - yields(2).await; - push(&self.0, Turn::End(id)); - id + async fn handle( + &mut self, + Work(id): Work, + __reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + let __value: usize = async { + push(&self.0, Turn::Begin(id)); + yields(2).await; + push(&self.0, Turn::End(id)); + id + } + .await; + __reply.reply(__value); } } @@ -240,13 +266,11 @@ async fn turns_body() { // L7: join() returned, so stopped() has completed and new work is refused. let events = snapshot(&events); assert_eq!(events.last(), Some(&Turn::StoppedEnd), "L7: {events:?}"); - assert!(matches!( - actor.send(Work(99)), - Err(ActorError::ActorStopped) - )); + // 0.8: past the death point a send or request is `NoProc`. + assert!(matches!(actor.send(Work(99)), Err(ActorError::NoProc))); assert!(matches!( actor.request_with_timeout(Work(99), LIMIT).await, - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) )); // L1/M1: started, then whole non-overlapping turns, then stopped. @@ -281,12 +305,16 @@ async fn turns_body() { assert!(handled.contains(&id), "R1: Ok without a turn for {id}"); } Ok(None) => {} - Err(ActorError::ActorStopped) => { + Err(ActorError::ActorStopped(_)) => { assert!( !handled.contains(&id), "R1: ran but caller got ActorStopped" ); } + // Sent after the death point (0.8): never admitted. + Err(ActorError::NoProc) => { + assert!(!handled.contains(&id), "R1: ran but caller got NoProc"); + } Err(other) => panic!("R1: unexpected outcome {other:?} for {id}"), } } @@ -347,21 +375,48 @@ impl Actor for Holder { } } impl Handler for Holder { - async fn handle(&mut self, hold: Hold, _: &Context) { - push(&self.0, Seen::Hold); - let _ = hold.entered.send(()); - let _ = hold.release.await; + async fn handle( + &mut self, + hold: Hold, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + push(&self.0, Seen::Hold); + let _ = hold.entered.send(()); + let _ = hold.release.await; + } + .await; + __reply.reply(__value); } } impl Handler for Holder { - async fn handle(&mut self, _: Queued, _: &Context) { - push(&self.0, Seen::Queued); + async fn handle( + &mut self, + _: Queued, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + push(&self.0, Seen::Queued); + } + .await; + __reply.reply(__value); } } impl Handler for Holder { - async fn handle(&mut self, _: Asked, _: &Context) -> u64 { - push(&self.0, Seen::Asked); - 1 + async fn handle( + &mut self, + _: Asked, + __reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __value: u64 = async { + push(&self.0, Seen::Asked); + 1 + } + .await; + __reply.reply(__value); } } @@ -427,7 +482,7 @@ async fn stop_discards_body() { ); for mut reply in replies { assert!( - matches!(reply.try_recv(), Err(TryRecvError::Closed)), + matches!(reply.try_recv(), Some(Err(ActorError::ActorStopped(_)))), "D4/L7: a queued reply sender outlived the exit" ); } @@ -440,7 +495,7 @@ async fn stop_discards_body() { assert!( matches!( actor.send(Queued(Payload(drops.clone()))), - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) ), "L7: send after exit" ); @@ -484,10 +539,19 @@ impl Message for Watch { type Result = (); } impl Handler for Observer { - async fn handle(&mut self, Watch(target): Watch, ctx: &Context) { - ctx.trap_exit(true); - ctx.link(&target); - ctx.monitor(&target); + async fn handle( + &mut self, + Watch(target): Watch, + __reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + let __value: () = async { + ctx.trap_exit(true); + ctx.link(&target); + ctx.monitor(&target); + } + .await; + __reply.reply(__value); } } struct Ping(usize); @@ -495,8 +559,17 @@ impl Message for Ping { type Result = (); } impl Handler for Observer { - async fn handle(&mut self, Ping(i): Ping, _: &Context) { - push(&self.0, Notice::Ping(i)); + async fn handle( + &mut self, + Ping(i): Ping, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + push(&self.0, Notice::Ping(i)); + } + .await; + __reply.reply(__value); } } struct Noise; @@ -504,14 +577,32 @@ impl Message for Noise { type Result = (); } impl Handler for Observer { - async fn handle(&mut self, _: Noise, _: &Context) { - push(&self.0, Notice::Noise); + async fn handle( + &mut self, + _: Noise, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + push(&self.0, Notice::Noise); + } + .await; + __reply.reply(__value); } } impl Handler for Observer { - async fn handle(&mut self, down: Down, _: &Context) { - push(&self.0, Notice::Down(down.reason)); - let _ = self.1.send(()); + async fn handle( + &mut self, + down: Down, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + push(&self.0, Notice::Down(down.reason)); + let _ = self.1.send(()); + } + .await; + __reply.reply(__value); } } @@ -536,17 +627,26 @@ impl Message for Burst { type Result = (); } impl Handler for Source { - async fn handle(&mut self, burst: Burst, ctx: &Context) { - let _ = burst.entered.send(()); - for i in 0..PINGS { - burst.to.send(Ping(i)).unwrap(); - yield_now().await; - } - match burst.end { - End::SelfStop => ctx.stop_with_reason(ExitReason::Shutdown), - End::Panic => panic!("source panicked"), - End::External => {} + async fn handle( + &mut self, + burst: Burst, + __reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + let __value: () = async { + let _ = burst.entered.send(()); + for i in 0..PINGS { + burst.to.send(Ping(i)).unwrap(); + yield_now().await; + } + match burst.end { + End::SelfStop => ctx.stop_with_reason(ExitReason::Shutdown), + End::Panic => panic!("source panicked"), + End::External => {} + } } + .await; + __reply.reply(__value); } } From 1144b8707a6af704a7c2ae96f4125046bd3377a1 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:43:57 -0300 Subject: [PATCH 122/208] ci: accept RUSTSEC-2026-0441 (pin-utils unmaintained) for the Loom dev-dependency `pin-utils` is reached only through loom 0.7's `futures` feature, which the reply-slot Loom model uses for `loom::future::block_on`, and loom is a dev-dependency only under `--cfg spawned_scheduler_loom`: never in a shipped build. The advisory is "unmaintained", with no vulnerability. Accepted in deny.toml and .cargo/audit.toml with that reason, like the other non-shipping advisories. `main` does not depend on loom's futures feature, so it is unaffected. --- .cargo/audit.toml | 1 + deny.toml | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/.cargo/audit.toml b/.cargo/audit.toml index 4dddb265..1be28fdc 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -5,6 +5,7 @@ ignore = [ "RUSTSEC-2026-0194", # quick-xml via pprof/inferno (private profiling-experiments feature) "RUSTSEC-2026-0195", # quick-xml via pprof/inferno (private profiling-experiments feature) "RUSTSEC-2025-0057", # fxhash via monoio (private io-experiments feature) + "RUSTSEC-2026-0441", # pin-utils via loom's futures feature (Loom dev-dependency only) ] informational_warnings = ["unmaintained", "unsound"] diff --git a/deny.toml b/deny.toml index e6a8076e..44baa3da 100644 --- a/deny.toml +++ b/deny.toml @@ -19,6 +19,11 @@ ignore = [ # fxhash (unmaintained) via monoio 0.2, only in the private `io-experiments` # benchmark feature. No replacement release of monoio drops it yet. { id = "RUSTSEC-2025-0057", reason = "fxhash via monoio; private io-experiments feature only" }, + # pin-utils (unmaintained, no vulnerability) via loom 0.7's `futures` + # feature, a dev-dependency only under `--cfg spawned_scheduler_loom` (the + # reply-slot Loom model drives a future with `loom::future::block_on`). + # Never part of a shipped build. Drop when loom stops depending on it. + { id = "RUSTSEC-2026-0441", reason = "pin-utils via loom's futures feature; Loom model-check dev-dependency only" }, ] [licenses] From e9350388cb7a59fdef5d23c478c1edf334c1692a Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:52:39 -0300 Subject: [PATCH 123/208] test(fuzz): port the scenario harness to the 0.8 API and contracts #238's harness was written against main's 0.7 API. Manual handlers take their `PendingReply`; and two oracles encoded 0.7 rules: - R1 / L7: a request that reaches a dead actor is `NoProc` (0.8, sent after the death point), as well as `ActorStopped(reason)` (answered at it). - V5: the budget charges restart actions; a OneForAll/RestForOne round rebuilds several children on one grant, so the factory log bounds a cohort supervisor's rebuilds by `max_restarts` times its children (OneForOne keeps the per-retry bound). - X1: a monitor installed after the target's death point gets `NoProc`. Seed corpus, regressions and generated inputs pass under the simulator. --- fuzz/src/driver.rs | 31 ++++++++++++++++++++++++++----- fuzz/src/world.rs | 36 +++++++++++++++++++++++++++++++++--- 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/fuzz/src/driver.rs b/fuzz/src/driver.rs index e31ee9df..f6df8bee 100644 --- a/fuzz/src/driver.rs +++ b/fuzz/src/driver.rs @@ -279,6 +279,7 @@ fn new_sup_inst(world: &World, root: usize, cfg: &SupervisorConfig) -> usize { root, max_restarts: cfg.max_restarts, window: cfg.restart_window, + cohort: !matches!(cfg.strategy, SupervisorStrategy::OneForOne), }); s.sups.len() - 1 } @@ -493,7 +494,8 @@ impl Driver { ); } } - Err(ActorError::ActorStopped) => {} + // Answered at the death point, or sent after it (0.8: NoProc). + Err(ActorError::ActorStopped(_) | ActorError::NoProc) => {} #[allow(unreachable_patterns)] Err(other) => self.violate("R1: unexpected request error", format!("{other}")), } @@ -1433,7 +1435,7 @@ impl Driver { } true } - Err(ActorError::ActorStopped) => { + Err(ActorError::ActorStopped(_) | ActorError::NoProc) => { if rt::timeout(LONG, aref.wait_exit()).await.is_err() { self.violate( "L7: a stopping actor never published its exit", @@ -1536,7 +1538,9 @@ impl Driver { .collect() }; for r in received(self) { - if r != published { + // 0.8 (X1): a monitor installed after the target's death + // point gets `NoProc`; one installed before, its reason. + if r != published && r != ExitReason::NoProc { self.violate( "X1: Down carried a different reason than the target published", format!("{r:?} vs {published:?}"), @@ -1781,14 +1785,31 @@ fn budget_checks(world: &World) { let SupInst { max_restarts, window, + cohort, .. } = s.sups[inst].clone(); + // V5 charges restart actions: a OneForOne retry is one, a cohort + // round is one however many children it rebuilds (at most all of + // them), so the factory log bounds a cohort supervisor's rebuilds by + // `max_restarts` times its children. + let children = calls + .iter() + .filter(|c| c.inst == inst) + .map(|c| &c.slot) + .collect::>() + .len() + .max(1); + let bound = if cohort { + max_restarts * children + } else { + max_restarts + }; for (i, t) in times.iter().enumerate() { let in_window = times[..=i].iter().filter(|u| *t - **u < window).count(); - if in_window > max_restarts { + if in_window > bound { found.push(( "V5: more restarts than the budget within one window", - format!("instance {inst}: {in_window} > {max_restarts} in {window:?}"), + format!("instance {inst}: {in_window} > {bound} in {window:?}"), )); break; } diff --git a/fuzz/src/world.rs b/fuzz/src/world.rs index 710fdba1..7984c5b2 100644 --- a/fuzz/src/world.rs +++ b/fuzz/src/world.rs @@ -67,6 +67,9 @@ pub struct SupInst { pub root: usize, pub max_restarts: usize, pub window: Duration, + /// OneForAll or RestForOne: a recovery round rebuilds several children + /// on one restart grant (0.8, V5). + pub cohort: bool, } /// One factory invocation (worker factory or nested `configure` callback). @@ -322,7 +325,13 @@ impl Worker { } impl Handler for Worker { - async fn handle(&mut self, msg: Do, ctx: &Context) -> u64 { + async fn handle( + &mut self, + msg: Do, + __reply: spawned_concurrency::PendingReply<'_, u64>, + ctx: &Context, + ) { + let __value: u64 = async { { let mut s = self.world.lock(); self.turn(&mut s, "Do"); @@ -393,6 +402,9 @@ impl Handler for Worker { } } msg.nonce + } + .await; + __reply.reply(__value); } } @@ -409,7 +421,13 @@ impl Worker { } impl Handler for Worker { - async fn handle(&mut self, down: Down, ctx: &Context) { + async fn handle( + &mut self, + down: Down, + __reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + let __value: () = async { let mut s = self.world.lock(); self.turn(&mut s, "Down"); let me = ctx.id(); @@ -437,17 +455,29 @@ impl Handler for Worker { if duplicate { s.violate("X1: Down delivered twice", format!("w{}", self.instance)); } + } + .await; + __reply.reply(__value); } } impl Handler for Worker { - async fn handle(&mut self, note: Note, _ctx: &Context) { + async fn handle( + &mut self, + note: Note, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _ctx: &Context, + ) { + let __value: () = async { let mut s = self.world.lock(); self.turn(&mut s, "Note"); if s.insts[self.instance].ended.contains(¬e.from) { let detail = format!("from {} at w{}", s.name(note.from), self.instance); s.violate("X7: a message arrived after the sender's Down/Exit", detail); } + } + .await; + __reply.reply(__value); } } From 9bf6f050cb99c5db795eaf75f8d31b3b72a49a63 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:52:39 -0300 Subject: [PATCH 124/208] fix(verification): copy specs writable in negative_checks (nix store sources are read-only) --- verification/tla/negative_checks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 6fce2b1d..07db8f11 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -82,7 +82,7 @@ def model_fault(module, fault, tlc, temporal=False, invariant="Safety", drop=()) with tempfile.TemporaryDirectory(prefix="bad-model-") as directory: work = Path(directory) for source in HERE.glob("*.tla"): - shutil.copy(source, work) + shutil.copyfile(source, work / source.name) # writable copy (sources may be read-only) cfg = (HERE / f"{module}.cfg").read_text().replace('Fault = "None"', f'Fault = "{fault}"') for name in drop: cfg = cfg.replace(f"INVARIANT {name}\n", "") @@ -103,7 +103,7 @@ def named_fault(module, fault, invariant, tlc): with tempfile.TemporaryDirectory(prefix="bad-model-") as directory: work = Path(directory) for source in HERE.glob("*.tla"): - shutil.copy(source, work) + shutil.copyfile(source, work / source.name) # writable copy (sources may be read-only) lines = [] for line in (HERE / f"{module}.cfg").read_text().splitlines(): if line.startswith("PROPERTY"): @@ -130,7 +130,7 @@ def spec_mutation(module, spec, label, pattern, replacement, prop, tlc): with tempfile.TemporaryDirectory(prefix="bad-spec-") as directory: work = Path(directory) for source in HERE.glob("*.tla"): - shutil.copy(source, work) + shutil.copyfile(source, work / source.name) # writable copy (sources may be read-only) text = (work / spec).read_text() mutated, count = re.subn(pattern, replacement, text, flags=re.S) if count != 1: @@ -177,7 +177,7 @@ def named_temporal(module, fault, prop, tlc, constants=None): with tempfile.TemporaryDirectory(prefix="bad-model-") as directory: work = Path(directory) for source in HERE.glob("*.tla"): - shutil.copy(source, work) + shutil.copyfile(source, work / source.name) # writable copy (sources may be read-only) lines = [] for line in (HERE / f"{module}.cfg").read_text().splitlines(): if line.startswith(("PROPERTY", "INVARIANT")): From a0b37e4a4aaae7646f8433181adad61927f41bf2 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:52:39 -0300 Subject: [PATCH 125/208] docs: fix intra-doc links that break the thread-only rustdoc build --- concurrency/src/error.rs | 2 +- concurrency/src/reply/jobs.rs | 2 +- concurrency/src/reply/mod.rs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/concurrency/src/error.rs b/concurrency/src/error.rs index 0de1ceeb..61b62253 100644 --- a/concurrency/src/error.rs +++ b/concurrency/src/error.rs @@ -63,7 +63,7 @@ impl std::fmt::Display for ExitReason { } } -/// The target of [`try_link`](crate::tasks::Context::try_link) no longer +/// The target of `Context::try_link` no longer /// existed (it was past its death point), as OTP's `noproc`. #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] #[error("noproc")] diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index e4687424..72f10164 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -1,7 +1,7 @@ //! Pipe-to-self jobs: the mode-independent core //! (docs/design/deferred-replies.md §8, §9). //! -//! Each job has one shared [`JobState`] with three parts: +//! Each job has one shared `JobState` with three parts: //! //! - **execution** (running or finished), physical: cancelled or timed-out //! work may keep running, and keeps its credit (or native slot) until it diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 0466a046..798a1a76 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -1,7 +1,7 @@ //! Reply transport for requests: reply slots, tokens and receivers //! (docs/design/deferred-replies.md). //! -//! Every request has a reply slot ([`slot::Slot`]) shared by the envelope, +//! Every request has a reply slot (`slot::Slot`) shared by the envelope, //! then by the handler's [`PendingReply`] (and, after [`PendingReply::defer`], //! a storable [`ReplyTo`]), and by the caller's [`ReplyReceiver`]. Each slot //! gets exactly one terminal discharge. @@ -498,7 +498,7 @@ impl Drop for ReplyTo { /// with the owner's final exit reason when it died first. /// /// One pointer wide: the deadline timer lives in the awaiting future -/// ([`ReplyFuture`]), not in the receiver, so receivers are cheap to store +/// (`ReplyFuture`), not in the receiver, so receivers are cheap to store /// and move. Dropping it abandons the request: a later reply is dropped and /// any deferred credit returns at once. pub struct ReplyReceiver { From 1dc692b5f0d7f18058c90f914d578626229d1532 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:52:39 -0300 Subject: [PATCH 126/208] fix(examples,tests): adapt to 0.8 shutdown semantics and one-CPU native budget - coordinated_shutdown: a supervisor's shutdown is its exit signal (E3): the actors whose cleanup must run in `stopped()` trap exits. - deferred_api caller-gone tests run two blocking jobs at once; the default `MAX_NATIVE_JOBS` is 1 on a one-CPU runner (single-cpu job), so the test actor sets 2. --- concurrency/tests/deferred_api.rs | 10 ++++++++-- examples/capabilities/src/bin/coordinated_shutdown.rs | 10 ++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/concurrency/tests/deferred_api.rs b/concurrency/tests/deferred_api.rs index d96ff4cd..650dcf37 100644 --- a/concurrency/tests/deferred_api.rs +++ b/concurrency/tests/deferred_api.rs @@ -591,7 +591,10 @@ mod caller_gone_vs_shortening_tasks { struct Delegator { job: Option, } - impl Actor for Delegator {} + impl Actor for Delegator { + // Two blocking jobs at once (the default is 1 on one CPU). + const MAX_NATIVE_JOBS: Option = Some(2); + } struct Ask; impl Message for Ask { type Result = (bool, bool); @@ -721,7 +724,10 @@ mod caller_gone_vs_shortening_threads { struct Delegator { job: Option, } - impl Actor for Delegator {} + impl Actor for Delegator { + // Two blocking jobs at once (the default is 1 on one CPU). + const MAX_NATIVE_JOBS: Option = Some(2); + } struct Ask; impl Message for Ask { type Result = (bool, bool); diff --git a/examples/capabilities/src/bin/coordinated_shutdown.rs b/examples/capabilities/src/bin/coordinated_shutdown.rs index b8aec835..97be3919 100644 --- a/examples/capabilities/src/bin/coordinated_shutdown.rs +++ b/examples/capabilities/src/bin/coordinated_shutdown.rs @@ -31,6 +31,11 @@ struct Peers { stops: Stops, } impl Actor for Peers { + // The supervisor's shutdown is its exit signal (as in OTP): trapping exits + // makes it end through `stopped()`, where this cleanup runs. + async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &Context) { self.stops.lock().unwrap().push("peers"); } @@ -61,6 +66,11 @@ struct Importer { stops: Stops, } impl Actor for Importer { + // The supervisor's shutdown is its exit signal (as in OTP): trapping exits + // makes it end through `stopped()`, where this cleanup runs. + async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &Context) { self.stops.lock().unwrap().push("importer"); } From e11386ee79ba0b0a4d36988a22d2d2d209b840a3 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:53:56 -0300 Subject: [PATCH 127/208] fix(reply): overloaded() releases its death-fence permit before waking Round-two review (Codex 3), introduced by the death-boundary fix: `ReplyTo::overloaded()` held its reply permit across `Slot::terminate`, including the caller's waker. A waker that waits for the owner's published exit then deadlocked with the owner's death, which waits for in-flight reply permits. `Slot::terminate_admitted` holds the permit across the transition's CAS only, as `reply_admitted` does; the credit return and the wakes run after it is released. Test: death_boundary tasks_mode::overloaded_wakes_the_caller_after_releasing_the_death_fence (lifecycle tracing off; failed before: the owner could not die while the helper's waker waited). --- concurrency/src/reply/mod.rs | 5 +- concurrency/src/reply/slot.rs | 21 ++++++++- concurrency/tests/death_boundary.rs | 73 +++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 5 deletions(-) diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 798a1a76..87bf7b99 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -399,9 +399,8 @@ impl ReplyTo { /// Past the owner's death point the caller gets its final reason. pub fn overloaded(mut self) { if let Some(slot) = self.slot.take() { - if let Some(_permit) = self.registry.reply_permit() { - slot.terminate(Terminal::Overloaded); - } + // The permit covers the transition only, never its wakes. + slot.terminate_admitted(Terminal::Overloaded, || self.registry.reply_permit()); } } diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index b81ab285..1017fd8d 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -365,15 +365,32 @@ impl Slot { /// `OPEN -> to` (drop, overloaded, expiry, abandonment, registry close). /// Returns whether this call won the transition. pub(crate) fn terminate(&self, to: Terminal) -> bool { + self.terminate_admitted(to, || Some(())) + } + + /// [`terminate`](Self::terminate) admitted by the owner: `admit`'s + /// permit covers only the transition's CAS; the credit return and the + /// wakes (executor code) run after it is released. Refused (`None`), the + /// slot is left to the owner's close. + pub(crate) fn terminate_admitted

( + &self, + to: Terminal, + admit: impl FnOnce() -> Option

, + ) -> bool { debug_assert!(to != Terminal::Replied); + let Some(permit) = admit() else { + return false; + }; let target = to.state(); - match self.cas::<()>(None, |w| { + let transition = self.cas::<()>(None, |w| { if state(w) == OPEN { Ok(with_state(w, target)) } else { Err(()) } - }) { + }); + drop(permit); + match transition { Ok((old, new)) => { self.after(old, new).run(&self.parked); true diff --git a/concurrency/tests/death_boundary.rs b/concurrency/tests/death_boundary.rs index 079398b4..8cb13ff9 100644 --- a/concurrency/tests/death_boundary.rs +++ b/concurrency/tests/death_boundary.rs @@ -73,6 +73,8 @@ mod tasks_mode { use super::*; use spawned_concurrency::tasks::{Actor, ActorStart, Context, Handler}; use spawned_concurrency::{PendingReply, ReplyTo}; + use std::future::Future; + use std::sync::Arc; /// Defers and keeps the token in the actor's state. struct Park; @@ -103,6 +105,77 @@ mod tasks_mode { } } + /// Defers and hands the token out. + struct HandOut(mpsc::Sender>); + impl Message for HandOut { + type Result = (); + } + impl Handler for Owner { + async fn handle(&mut self, msg: HandOut, reply: PendingReply<'_, ()>, _: &Context) { + msg.0.send(reply.defer().unwrap()).unwrap(); + } + } + + /// Review round 2 (Codex 3): `ReplyTo::overloaded()` holds its + /// death-fence permit only across the transition. Its caller's waker + /// runs after: here it stops the owner and waits for the published + /// exit. The permit used to cover the wake, so the owner's death waited + /// for the waker and the waker for the death. + #[test] + fn overloaded_wakes_the_caller_after_releasing_the_death_fence() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::task::{Wake, Waker}; + struct StopAndWait { + actor: spawned_concurrency::tasks::ActorRef, + exited: Arc, + } + impl Wake for StopAndWait { + fn wake(self: Arc) { + self.actor.context().stop(); + let began = Instant::now(); + while began.elapsed() < Duration::from_secs(3) { + if self.actor.exit_reason().is_some() { + self.exited.store(true, Ordering::SeqCst); + return; + } + std::thread::sleep(Duration::from_millis(1)); + } + } + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let _guard = runtime.enter(); + let actor = Owner { + _state: inert(), + parked: Vec::new(), + } + .start(); + let (tokens, token) = mpsc::channel(); + let receiver = actor + .request_raw_until(HandOut(tokens), spawned_concurrency::Deadline::Never) + .unwrap(); + let token = token.recv_timeout(LIMIT).unwrap(); + let exited = Arc::new(AtomicBool::new(false)); + let waker = Waker::from(Arc::new(StopAndWait { + actor: actor.clone(), + exited: exited.clone(), + })); + let mut awaiting = Box::pin(std::future::IntoFuture::into_future(receiver)); + let mut cx = std::task::Context::from_waker(&waker); + assert!(awaiting.as_mut().poll(&mut cx).is_pending()); + // The wake runs on this helper, inside `overloaded()`. + std::thread::spawn(move || token.overloaded()) + .join() + .unwrap(); + assert!( + exited.load(Ordering::SeqCst), + "the owner could not die while overloaded() held its fence permit" + ); + } + fn stopped(result: Option>, reason: &ExitReason) -> Option<()> { match result? { Err(ActorError::ActorStopped(r)) if r == *reason => Some(()), From fb77cf8e3e0bc0bc0697b04f83ab68c4e0e25836 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:58:27 -0300 Subject: [PATCH 128/208] fix: answer every caller at the death point before any wake-up or log runs Round-two review (Codex 5). Discharging the death point's obligations still ran user code between answers: - closing a slot ran its caller's waker at once, before later slots were closed: a waker waiting for another queued caller's answer blocked the discharge that would give it; - a runtime drop's "stopped() skipped" log (a tracing subscriber) ran after the seal but before any deferred or queued request was answered. The registry now batches the death point: from its close until `end_discharge`, closes make only the terminal transition (credits return with it) and collect the wake-ups (`Slot::close_collect`). The modes answer the queued requests without destroying them (`answer_mailbox`), then `ActorCleanup::discharged` runs the wake-ups and the deferred log, and only then are payloads and state destroyed. A never-polled actor's startup drop does the same. `Lifecycle.tla`'s abstraction note now says what makes the answers commute to the seal (terminal transitions only). Tests (death_boundary; both failed before): - tasks_mode::every_queued_caller_is_answered_before_any_wake_runs (the first caller's waker waits for the second caller's answer); - subscriber_after_discharge::the_skipped_stopped_log_runs_after_every_caller_is_answered (subprocess with a global subscriber that waits for a queued caller). reply_slot_loom and registry_turn_loom pass. --- concurrency/src/actor_cleanup.rs | 51 +++++++-- concurrency/src/reply/registry.rs | 60 ++++++++++- concurrency/src/reply/slot.rs | 38 ++++++- concurrency/src/tasks/actor.rs | 25 +++-- concurrency/src/threads/actor.rs | 22 ++-- concurrency/tests/death_boundary.rs | 159 ++++++++++++++++++++++++++++ verification/tla/Lifecycle.tla | 9 +- 7 files changed, 329 insertions(+), 35 deletions(-) diff --git a/concurrency/src/actor_cleanup.rs b/concurrency/src/actor_cleanup.rs index b4bc2adb..a5f9de60 100644 --- a/concurrency/src/actor_cleanup.rs +++ b/concurrency/src/actor_cleanup.rs @@ -59,6 +59,11 @@ pub(crate) struct ActorCleanup { replies: Option>, /// The owner's jobs: admission closes and they are cancelled at D. jobs: Option>, + /// A runtime drop skipped `stopped()`: logged once every obligation is + /// answered (a subscriber is user code). + skip_log: Option, + /// [`discharged`](Self::discharged) ran. + discharged: bool, } impl ActorCleanup { pub(crate) fn new( @@ -82,6 +87,8 @@ impl ActorCleanup { failures: None, replies: None, jobs: None, + skip_log: None, + discharged: false, } } /// Seal `replies` with the final reason at D, inside the exit-table @@ -155,16 +162,36 @@ impl ActorCleanup { self.close(reason) }); if skipped { - log_stopped_skipped(self.id, "", "runtime_drop", Some(&reason)); + self.skip_log = Some(reason.clone()); } fail_point!("actor.at_death"); // Every open deferred reply (and every exit-pending one) is answered - // with the final reason, before any destructor runs. + // with the final reason, before any destructor runs. Their wake-ups + // wait until the mailbox's queued requests are answered too + // ([`discharged`](Self::discharged)): no waker runs between two + // answers. if let Some(replies) = &self.replies { + replies.begin_discharge(); replies.close(); } reason } + + /// Every obligation of the death point is answered (the mode answered + /// its queued requests after [`die`](Self::die)): now run the deferred + /// wake-ups, then log a skipped `stopped()`. Call it before destroying + /// any payload or state. Idempotent. + pub(crate) fn discharged(&mut self) { + if std::mem::replace(&mut self.discharged, true) { + return; + } + if let Some(replies) = &self.replies { + replies.end_discharge(); + } + if let Some(reason) = self.skip_log.take() { + log_stopped_skipped(self.id, "", "runtime_drop", Some(&reason)); + } + } fn close(&mut self, reason: ExitReason) -> ExitReason { let replies = self.replies.as_deref(); self.notices = Some(crate::link::close_links_at_death( @@ -193,6 +220,7 @@ impl Drop for ActorCleanup { // Normally D was reached already; otherwise the runtime dropped the // actor (G1) and nothing of it is left to destroy. let reason = self.die_dropped(); + self.discharged(); if let Some(message) = self.failures.as_ref().and_then(|f| f.first()) { // Subscriber code must not unwind through a receiver/actor teardown. log_safely(|| { @@ -243,10 +271,16 @@ pub(crate) fn finish_actor_state(actor: A) -> Option { pub(crate) struct ActorStartup { payload: Option

, cleanup: Option>, - discharge: fn(&mut P), + /// Answers the queued requests and returns what it drained, destroyed + /// with the payload once the wake-ups ran. + discharge: fn(&mut P) -> Box, } impl ActorStartup { - pub(crate) fn new(payload: P, cleanup: ActorCleanup, discharge: fn(&mut P)) -> Self { + pub(crate) fn new( + payload: P, + cleanup: ActorCleanup, + discharge: fn(&mut P) -> Box, + ) -> Self { Self { payload: Some(payload), cleanup: Some(cleanup), @@ -263,9 +297,14 @@ impl Drop for ActorStartup { cleanup.die_dropped(); } if let Some(mut payload) = self.payload.take() { - (self.discharge)(&mut payload); + // Answers the queued requests; their payloads are destroyed with + // the rest of the startup payload, after the wake-ups. + let queued = (self.discharge)(&mut payload); + if let Some(cleanup) = &mut self.cleanup { + cleanup.discharged(); + } if let Err(panic) = - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| drop(payload))) + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| drop((queued, payload)))) { let message = crate::error::panic_message(&*panic); crate::mailbox::dispose_panic(panic); diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index f8fc0d38..ef6912cc 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -25,7 +25,7 @@ //! lands on it (`IN_TURN_PENDING`). use super::deadline::{self, DeadlineHeap, Expirable}; -use super::slot::{CreditPool, Slot, Terminal, NEVER}; +use super::slot::{CreditPool, Slot, Terminal, Wake, NEVER}; use crate::{ActorId, ExitReason}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, OnceLock, Weak}; @@ -38,6 +38,9 @@ use super::turn::{Outcome, TurnPhase, DEAD, EXITING}; /// A slot seen without its reply type. pub(crate) trait Closeable: Send + Sync { fn close(&self) -> bool; + /// Close without waking; the returned work runs with `run_wake`. + fn close_collect(&self) -> Option; + fn run_wake(&self, wake: Wake); fn terminate(&self, to: Terminal) -> bool; fn is_terminal(&self) -> bool; } @@ -56,6 +59,12 @@ impl Closeable for Slot { fn close(&self) -> bool { Slot::close(self) } + fn close_collect(&self) -> Option { + Slot::close_collect(self) + } + fn run_wake(&self, wake: Wake) { + Slot::run_wake(self, wake) + } fn terminate(&self, to: Terminal) -> bool { Slot::terminate(self, to) } @@ -93,6 +102,11 @@ pub(crate) struct Registry { /// at the death point, before the death notices are taken, so a signal /// sent before death is delivered before them. signals: crate::mailbox::SendGate, + /// While the death point discharges (from its close until + /// [`end_discharge`](Self::end_discharge)): the wake-ups of the slots it + /// closed, run only once every obligation is answered, so no waker runs + /// between two closes. + discharge: Mutex, Wake)>>>, #[cfg(all(spawned_sim, feature = "async"))] sim: deadline::SimService, } @@ -118,6 +132,7 @@ impl Registry { final_reason: OnceLock::new(), replies: crate::mailbox::SendGate::default(), signals: crate::mailbox::SendGate::default(), + discharge: Mutex::new(None), #[cfg(all(spawned_sim, feature = "async"))] sim: deadline::SimService::default(), }) @@ -276,7 +291,7 @@ impl Registry { // slot listed here is either taken by that close or closed below. if self.is_dead() { drop(lists); - slot.close(); + self.close_slot(slot.clone()); } else { lists.tokens.push(Arc::downgrade(&slot)); drop(lists); @@ -373,6 +388,43 @@ impl Registry { self.turn.load() >= EXITING } + /// The death point starts discharging: closes from here until + /// [`end_discharge`](Self::end_discharge) defer their wake-ups. + pub(crate) fn begin_discharge(&self) { + let mut batch = self.discharge.lock().unwrap_or_else(|p| p.into_inner()); + batch.get_or_insert_with(Vec::new); + } + + /// Every obligation of the death point is answered: run the collected + /// wake-ups (executor or user code), outside every lock. Later closes + /// wake at once. Idempotent. + pub(crate) fn end_discharge(&self) { + let batch = self + .discharge + .lock() + .unwrap_or_else(|p| p.into_inner()) + .take(); + for (slot, wake) in batch.into_iter().flatten() { + slot.run_wake(wake); + } + } + + /// Close one slot as `STOPPED`, deferring its wake-up while discharging. + fn close_slot(&self, slot: Arc) -> bool { + let Some(wake) = slot.close_collect() else { + return false; + }; + let mut batch = self.discharge.lock().unwrap_or_else(|p| p.into_inner()); + match batch.as_mut() { + Some(batch) => batch.push((slot, wake)), + None => { + drop(batch); + slot.run_wake(wake); + } + } + true + } + /// Death: every open token listed, and every exit-pending slot, becomes /// `STOPPED` with the final reason, wherever it is. Returns how many were /// closed. @@ -384,13 +436,13 @@ impl Registry { self.seal(&ExitReason::Killed); crate::lifecycle_trace::reply(self.owner.raw(), "Death", 0, ""); for slot in self.turn.close() { - slot.close(); + self.close_slot(slot); } let tokens = std::mem::take(&mut lock(&self.lists).tokens); let closed = tokens .into_iter() .filter_map(|t| t.upgrade()) - .filter(|s| s.close()) + .filter(|s| self.close_slot(s.clone())) .count(); // The heap is dropped after the close (§5 step 3.4), and the service // forgets this owner, so dead actors leave nothing behind. diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 1017fd8d..d09f523c 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -208,7 +208,7 @@ fn lock(mutex: &Mutex) -> super::sync::MutexGuard<'_, T> { } /// What a winning transition must do after its CAS: return credits, then wake. -struct Wake { +pub(crate) struct Wake { pool: Option>, release: u64, waker: Option, @@ -403,17 +403,47 @@ impl Slot { /// `STOPPED`. A reply in `WRITING` is a value write with no user code in /// it, so the close waits for it to resolve. pub(crate) fn close(&self) -> bool { + match self.close_collect() { + Some(wake) => { + wake.run(&self.parked); + true + } + None => false, + } + } + + /// [`close`](Self::close) without the wake-up: the transition (and its + /// credit return) happens now and the caller runs the returned work with + /// [`run_wake`](Self::run_wake) once a whole batch is closed, so no + /// waker (executor or user code) runs between two closes. + pub(crate) fn close_collect(&self) -> Option { loop { - if self.terminate(Terminal::Stopped) { - return true; + if let Ok((old, new)) = self.cas::<()>(None, |w| { + if state(w) == OPEN { + Ok(with_state(w, STOPPED)) + } else { + Err(()) + } + }) { + let mut wake = self.after(old, new); + // The credit returns with the transition; only wakes wait. + if let Some(pool) = wake.pool.take() { + pool.release(wake.release); + } + return Some(wake); } if state(self.word()) != WRITING { - return false; + return None; } yield_now(); } } + /// Run the wake-up work [`close_collect`](Self::close_collect) returned. + pub(crate) fn run_wake(&self, wake: Wake) { + wake.run(&self.parked); + } + /// `[deadline]`: expire the slot if `now >= deadline`. Returns whether the /// deadline has passed (whoever won the transition). pub(crate) fn check_deadline(&self, now: &impl Fn() -> u64) -> bool { diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index a6dc8997..302d4216 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -174,15 +174,18 @@ trait Envelope: Send { ) -> Pin + Send + 'a>>; } -/// Answer every request queued in a dead owner's mailbox, then destroy the -/// queued payloads (a destructor panic is a cleanup failure). Admission is -/// closed first, so nothing is queued after. -fn discharge_mailbox(rx: &mut mpsc::Receiver + Send>>) { +/// Answer every request queued in a dead owner's mailbox, destroying +/// nothing: the drained envelopes come back, to be destroyed once the death +/// point's wake-ups ran. Admission is closed first, so nothing is queued +/// after. +fn answer_mailbox( + rx: &mut mpsc::Receiver + Send>>, +) -> Vec + Send>> { let mut queued = rx.drain(); for envelope in &mut queued { envelope.discharge(); } - rx.discard_all(queued); + queued } /// A never-polled actor's startup payload, dropped with its runtime. @@ -192,8 +195,8 @@ type StartupPayload = ( mpsc::Receiver + Send>>, CancellationToken, ); -fn discharge_startup(payload: &mut StartupPayload) { - discharge_mailbox(&mut payload.2); +fn discharge_startup(payload: &mut StartupPayload) -> Box { + Box::new(answer_mailbox(&mut payload.2)) } struct MessageEnvelope { @@ -1688,8 +1691,12 @@ impl ExitOnDrop<'_, A, C> { } else { self.cleanup.die(); } - discharge_mailbox(&mut self.rx); - // Destruction: a destructor panic is a cleanup failure, never the reason. + let queued = answer_mailbox(&mut self.rx); + // Every caller is answered: wake them, then destroy (queued + // payloads, then state). A destructor panic is a cleanup failure, + // never the reason. + self.cleanup.discharged(); + self.rx.discard_all(queued); if let Some(failure) = crate::actor_cleanup::finish_actor_state(actor) { self.cleanup.cleanup_failed(failure); } diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index ef5cc7bd..f20d0fe5 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -106,13 +106,13 @@ trait Envelope: Send { } /// Answer every request queued in a dead owner's mailbox (and `extra`, an -/// envelope received after the stop), then destroy the queued payloads (a -/// destructor panic is a cleanup failure). Admission is closed first -/// (waiting for in-flight sends), so nothing is queued after. -fn discharge_mailbox( +/// envelope received after the stop), destroying nothing: the items come +/// back, to be destroyed once the death point's wake-ups ran. Admission is +/// closed first (waiting for in-flight sends), so nothing is queued after. +fn answer_mailbox( rx: &mut mpsc::Receiver>, extra: Option + Send>>, -) { +) -> Vec> { let mut queued = rx.drain(); queued.extend(extra.map(MailboxItem::Message)); for item in &mut queued { @@ -120,7 +120,7 @@ fn discharge_mailbox( envelope.discharge(); } } - rx.discard_all(queued); + queued } /// A never-polled actor's startup payload (its thread was never started). @@ -130,8 +130,8 @@ type StartupPayload = ( mpsc::Receiver>, CancellationToken, ); -fn discharge_startup(payload: &mut StartupPayload) { - discharge_mailbox(&mut payload.2, None); +fn discharge_startup(payload: &mut StartupPayload) -> Box { + Box::new(answer_mailbox(&mut payload.2, None)) } struct MessageEnvelope { @@ -1238,7 +1238,11 @@ impl ActorRef { // requests (and one received after the stop) are answered, all // before any destructor. cleanup.die(); - discharge_mailbox(&mut rx, discarded); + let queued = answer_mailbox(&mut rx, discarded); + // Every caller is answered: wake them, then destroy (queued + // payloads, then state). + cleanup.discharged(); + rx.discard_all(queued); drop(rx); // Destruction: a destructor panic is a cleanup failure only. if let Some(failure) = crate::actor_cleanup::finish_actor_state(actor) { diff --git a/concurrency/tests/death_boundary.rs b/concurrency/tests/death_boundary.rs index 8cb13ff9..d94c9ea4 100644 --- a/concurrency/tests/death_boundary.rs +++ b/concurrency/tests/death_boundary.rs @@ -116,6 +116,69 @@ mod tasks_mode { } } + /// Review round 2 (Codex 5): the death point answers every caller before + /// any wake-up runs. The first queued caller's waker waits for the second + /// caller's answer; closing the first slot used to run that waker before + /// the second slot was closed, so the second answer had to wait. + #[test] + fn every_queued_caller_is_answered_before_any_wake_runs() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Mutex; + use std::task::{Wake, Waker}; + type Receiver = spawned_concurrency::ReplyReceiver<()>; + struct WaitForSecond { + second: Mutex, + answered: AtomicBool, + } + impl Wake for WaitForSecond { + fn wake(self: Arc) { + let began = Instant::now(); + while began.elapsed() < Duration::from_secs(3) { + if self.second.lock().unwrap().try_recv().is_some() { + self.answered.store(true, Ordering::SeqCst); + return; + } + std::thread::sleep(Duration::from_millis(1)); + } + } + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let _guard = runtime.enter(); + let actor = Owner { + _state: inert(), + parked: Vec::new(), + } + .start(); + let (unblock, blocked) = mpsc::channel(); + let (entered_tx, entered) = mpsc::channel(); + actor.send(Block(blocked, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + let never = spawned_concurrency::Deadline::Never; + let first = actor.request_raw_until(Queued(inert()), never).unwrap(); + let second = actor.request_raw_until(Queued(inert()), never).unwrap(); + let waiter = Arc::new(WaitForSecond { + second: Mutex::new(second), + answered: AtomicBool::new(false), + }); + let waker = Waker::from(waiter.clone()); + let mut awaiting = Box::pin(std::future::IntoFuture::into_future(first)); + assert!(awaiting + .as_mut() + .poll(&mut std::task::Context::from_waker(&waker)) + .is_pending()); + actor.context().stop(); + unblock.send(()).unwrap(); + runtime.block_on(actor.join()); + assert!( + waiter.answered.load(Ordering::SeqCst), + "a wake-up ran before every queued caller was answered" + ); + } + /// Review round 2 (Codex 3): `ReplyTo::overloaded()` holds its /// death-fence permit only across the transition. Its caller's waker /// runs after: here it stops the owner and waits for the published @@ -336,3 +399,99 @@ mod threads_mode { actor.join(); } } + +#[cfg(feature = "async")] +#[path = "support/drop_subprocess.rs"] +mod subprocess; + +/// Review round 2 (Codex 5): a runtime drop's "stopped() skipped" log (a +/// tracing subscriber is user code) runs after every caller is answered. It +/// used to run between the seal and the answers, so a subscriber waiting for +/// a queued caller blocked the death point. Global subscriber: own process. +#[cfg(feature = "async")] +mod subscriber_after_discharge { + use super::subprocess; + use super::*; + use spawned_concurrency::tasks::{Actor, ActorStart, Context, Handler}; + use spawned_concurrency::{PendingReply, ReplyReceiver}; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Mutex; + + static QUEUED: Mutex>> = Mutex::new(None); + static ANSWERED_WHEN_LOGGED: AtomicBool = AtomicBool::new(false); + static LOGGED: AtomicBool = AtomicBool::new(false); + + /// Waits (bounded) in every event for the queued caller's answer. + struct WaitingSubscriber; + impl tracing::Subscriber for WaitingSubscriber { + fn enabled(&self, _: &tracing::Metadata<'_>) -> bool { + true + } + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id { + tracing::span::Id::from_u64(1) + } + fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {} + fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {} + fn event(&self, _: &tracing::Event<'_>) { + let Some(mut queued) = QUEUED.lock().unwrap().take() else { + return; + }; + LOGGED.store(true, Ordering::SeqCst); + let began = Instant::now(); + while began.elapsed() < Duration::from_secs(3) { + if queued.try_recv().is_some() { + ANSWERED_WHEN_LOGGED.store(true, Ordering::SeqCst); + return; + } + std::thread::sleep(Duration::from_millis(1)); + } + } + fn enter(&self, _: &tracing::span::Id) {} + fn exit(&self, _: &tracing::span::Id) {} + } + + struct Idle; + impl Actor for Idle {} + impl Handler for Idle { + async fn handle(&mut self, _: Queued, reply: PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } + } + + #[test] + fn the_skipped_stopped_log_runs_after_every_caller_is_answered() { + subprocess::run( + "subscriber_after_discharge::child", + "skipped-stopped log ran after the answers", + ); + } + + #[test] + #[ignore = "subprocess helper exercised by the enabled parent"] + fn child() { + if std::env::var_os("SPAWNED_MAILBOX_DROP_CHILD").is_none() { + return; + } + tracing::subscriber::set_global_default(WaitingSubscriber).unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + // Never polled: dropping the runtime drops it before `stopped()`. + let actor = { + let _guard = runtime.enter(); + Idle.start() + }; + let queued = actor + .request_raw_until(Queued(inert()), spawned_concurrency::Deadline::Never) + .unwrap(); + *QUEUED.lock().unwrap() = Some(queued); + drop(runtime); + assert!(LOGGED.load(Ordering::SeqCst), "nothing was logged"); + assert!( + ANSWERED_WHEN_LOGGED.load(Ordering::SeqCst), + "the log ran before the queued caller was answered" + ); + println!("skipped-stopped log ran after the answers"); + } +} diff --git a/verification/tla/Lifecycle.tla b/verification/tla/Lifecycle.tla index 8966483f..65225678 100644 --- a/verification/tla/Lifecycle.tla +++ b/verification/tla/Lifecycle.tla @@ -224,9 +224,12 @@ Fire(a, r) == (* requests are answered, the table closes. Shared by Die and RuntimeDrop. *) (* The code fences outgoing exit signals and replies first, then seals and *) (* closes the table, then answers deferred and queued requests, and only *) -(* then destroys payloads and state (actor_cleanup.rs). The answers follow *) -(* the seal but run no user code and nothing in between depends on them, *) -(* so they commute left to it: this atomic step is a sound abstraction. *) +(* then runs the answers' wake-ups and logging, and destroys payloads and *) +(* state (actor_cleanup.rs). Between the seal and the last answer the code *) +(* runs only terminal transitions of slots (no waker, subscriber, callback *) +(* or destructor: wake-ups are collected and run after the batch), and no *) +(* other step depends on those answers, so they commute left to the seal: *) +(* this atomic step is a sound abstraction. *) Death(a, r) == /\ st' = [st EXCEPT ![a] = "gone"] /\ mbox' = [mbox EXCEPT ![a] = <<>>] From a62dc8256165a7dfaed57eeabe7d21eae39e0cbf Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:01:23 -0300 Subject: [PATCH 129/208] fix(jobs): the owner's exit cancels jobs without destroying their payloads before death MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-two review (Codex 1). The owner's exit (when it begins, and again at the death point) cancelled every job through the full cancel path, which destroys user payloads at once: a suppressed held result (`done` and the outcome) and every scheduled emit's message. A destructor that waited for the owner's queued caller then blocked the very death point that would answer it (both modes; the runtime-drop paths too). `JobSetCore::cancel_all` now cancels with `Job::cancel_for_exit`: `JobState` marks the job cancelled (fence, emitter and async-work wake-up) and keeps the held result and the scheduled emits; the job set keeps the cancelled jobs. `JobSet::dispose_cancelled`, called by `ActorCleanup::discharged` after every caller is answered and woken, destroys them through the jobs' failure sinks. Handle cancellation and keyed replacement still destroy at once (no death point is involved). deferred-replies.md §5 describes the order. Tests: death_boundary {tasks,threads}_mode::job_payloads_are_destroyed_after_every_caller_is_answered (a scheduled message, and in tasks mode also a held result, whose destructor waits for a queued caller; both failed before). job_core_loom passes. --- concurrency/src/actor_cleanup.rs | 9 +- concurrency/src/reply/job_core.rs | 33 ++++-- concurrency/src/reply/jobs.rs | 53 ++++++++- concurrency/tests/death_boundary.rs | 161 ++++++++++++++++++++++++++++ docs/design/deferred-replies.md | 14 ++- 5 files changed, 253 insertions(+), 17 deletions(-) diff --git a/concurrency/src/actor_cleanup.rs b/concurrency/src/actor_cleanup.rs index a5f9de60..6da9e7c4 100644 --- a/concurrency/src/actor_cleanup.rs +++ b/concurrency/src/actor_cleanup.rs @@ -179,8 +179,9 @@ impl ActorCleanup { /// Every obligation of the death point is answered (the mode answered /// its queued requests after [`die`](Self::die)): now run the deferred - /// wake-ups, then log a skipped `stopped()`. Call it before destroying - /// any payload or state. Idempotent. + /// wake-ups, then log a skipped `stopped()`, then destroy what the jobs' + /// cancellation released. Call it before destroying the mailbox's + /// payloads and the state. Idempotent. pub(crate) fn discharged(&mut self) { if std::mem::replace(&mut self.discharged, true) { return; @@ -191,6 +192,10 @@ impl ActorCleanup { if let Some(reason) = self.skip_log.take() { log_stopped_skipped(self.id, "", "runtime_drop", Some(&reason)); } + // Payloads the exit's job cancellation released: destroyed only now. + if let Some(jobs) = &self.jobs { + jobs.dispose_cancelled(); + } } fn close(&mut self, reason: ExitReason) -> ExitReason { let replies = self.replies.as_deref(); diff --git a/concurrency/src/reply/job_core.rs b/concurrency/src/reply/job_core.rs index d5c619fa..22915583 100644 --- a/concurrency/src/reply/job_core.rs +++ b/concurrency/src/reply/job_core.rs @@ -108,6 +108,16 @@ impl JobCore { /// wakes blocked emitters. Never undoes effects. Returns whether this /// call set the flag. The caller wakes async emitters afterwards. pub(crate) fn cancel(&self) -> bool { + let (newly, held) = self.cancel_keeping_held(); + // A held result is suppressed; it drops outside the lock. + drop(held); + newly + } + + /// [`cancel`](Self::cancel) that hands the suppressed held result back + /// instead of dropping it (user payload): the owner's exit destroys it + /// only after its death point answered every caller. + pub(crate) fn cancel_keeping_held(&self) -> (bool, Option) { let (newly, held) = { let mut inner = lock(&self.inner); let newly = !inner.cancelled; @@ -120,10 +130,8 @@ impl JobCore { self.cancelled_fast.store(true, Ordering::Release); (newly, inner.held.take()) }; - // A held result is suppressed; it drops outside the lock. - drop(held); self.emitters.notify_all(); - newly + (newly, held) } /// Take the terminal notification and run `enqueue` under the job's @@ -310,6 +318,12 @@ impl JobCore { /// What the job set needs from a job. pub(crate) trait Job { fn cancel(&self); + /// The owner's exit cancels the job: like [`cancel`](Self::cancel), but + /// the payloads it releases are kept for the owner to destroy after its + /// death point. + fn cancel_for_exit(&self) { + self.cancel(); + } fn is_running(&self) -> bool; fn finished(&self); /// Kept in the job list at compaction (so `cancel_all` reaches it): @@ -443,9 +457,10 @@ impl JobSetCore { } /// Owner exit: close admission and cancel every job (flag set, blocked - /// emitters woken). The key map's references and the upgraded jobs drop - /// outside the locks. - pub(crate) fn cancel_all(&self) { + /// emitters woken; [`Job::cancel_for_exit`]). Returns the cancelled jobs + /// (the caller drops them, outside the locks); the key map's references + /// drop outside the locks. + pub(crate) fn cancel_all(&self) -> Vec> { let (jobs, keys) = { let mut list = lock(&self.jobs); list.closed = true; @@ -454,10 +469,12 @@ impl JobSetCore { std::mem::take(&mut *lock(&self.keys)), ) }; - for job in jobs.iter().filter_map(Weak::upgrade) { - job.cancel(); + let cancelled: Vec> = jobs.iter().filter_map(Weak::upgrade).collect(); + for job in &cancelled { + job.cancel_for_exit(); } drop(keys); + cancelled } /// Wait until no job is running. `wait` parks on the idle condition diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 72f10164..a2e1dfc9 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -218,8 +218,15 @@ pub(crate) struct JobState { /// Delayed emits (`JobCtx::emit_at`) not yet fired: their strong /// references (the deadline heap holds `Weak`s). Cleared at cancellation. scheduled: Mutex>>, + /// What the owner's exit released (a held result, scheduled emits): + /// user payload, destroyed by [`JobSet::dispose_cancelled`] once the + /// owner's death point answered and woke every caller. + exit_released: Mutex>, } +/// Payloads an owner's exit took from a job, destroyed after its death point. +type ExitReleased = (Option, Vec>); + fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { mutex.lock().unwrap_or_else(|p| p.into_inner()) } @@ -236,6 +243,7 @@ impl JobState { #[cfg(feature = "async")] emit_wakers: Mutex::new(Vec::new()), scheduled: Mutex::new(Vec::new()), + exit_released: Mutex::new(None), }) } @@ -517,6 +525,23 @@ impl Job for JobState { fn cancel(&self) { self.cancel_quietly(); } + /// Mark the job cancelled (fences its messages, wakes its emitters, + /// stops its async work) without destroying anything: the suppressed + /// held result and the scheduled emits are kept for + /// [`JobSet::dispose_cancelled`]. + fn cancel_for_exit(&self) { + let (newly, held) = self.core.cancel_keeping_held(); + if newly { + self.count_cancel(); + } + let scheduled = std::mem::take(&mut *lock(&self.scheduled)); + *lock(&self.exit_released) = Some((held, scheduled)); + #[cfg(feature = "async")] + { + self.cancel.cancel(); + self.wake_emitters(); + } + } fn is_running(&self) -> bool { self.core.is_running() } @@ -575,6 +600,9 @@ pub(crate) struct JobSet { /// Whether the owner's exit was recorded (lifecycle traces). #[cfg(feature = "lifecycle-trace")] exited: std::sync::atomic::AtomicBool, + /// Jobs the owner's exit cancelled, whose released payloads wait for + /// [`dispose_cancelled`](Self::dispose_cancelled). + exit_cancelled: Mutex>>, /// The first `NativeBudget` refusal was logged (only refusals touch it). native_refusal_logged: std::sync::atomic::AtomicBool, #[cfg(feature = "async")] @@ -588,6 +616,7 @@ impl JobSet { core: JobSetCore::new(max_native), #[cfg(feature = "lifecycle-trace")] exited: std::sync::atomic::AtomicBool::new(false), + exit_cancelled: Mutex::new(Vec::new()), native_refusal_logged: std::sync::atomic::AtomicBool::new(false), #[cfg(feature = "async")] idle_async: Mutex::new(Vec::new()), @@ -663,10 +692,13 @@ impl JobSet { self.core.running() } - /// Owner exit: cancel every job (flag set, blocked emitters woken). + /// Owner exit: close admission and cancel every job (flag set, blocked + /// emitters woken). Nothing is destroyed here: what the jobs release + /// waits for [`dispose_cancelled`](Self::dispose_cancelled). pub(crate) fn cancel_all(&self) { trace::atomic(|| { - self.core.cancel_all(); + let cancelled = self.core.cancel_all(); + lock(&self.exit_cancelled).extend(cancelled); #[cfg(feature = "lifecycle-trace")] if !self.exited.swap(true, std::sync::atomic::Ordering::AcqRel) { trace::job(self.owner.raw(), || trace::JobEvent::new("Exit", 0)); @@ -674,6 +706,23 @@ impl JobSet { }); } + /// After the owner's death point answered and woke every caller: destroy + /// what the exit's cancellation released (held results, scheduled + /// emits; contained through each job's failure sink). Idempotent. + pub(crate) fn dispose_cancelled(&self) { + let cancelled = std::mem::take(&mut *lock(&self.exit_cancelled)); + for job in &cancelled { + let released = lock(&job.exit_released).take(); + if let Some((held, scheduled)) = released { + drop(held); + for emit in scheduled { + emit.discard(job); + } + } + } + drop(cancelled); + } + /// Join the cancelled jobs, bounded by `deadline`; returns how many are /// still running (unfinished at stop). pub(crate) fn join_blocking(&self, deadline: std::time::Instant) -> usize { diff --git a/concurrency/tests/death_boundary.rs b/concurrency/tests/death_boundary.rs index d94c9ea4..8976dc2c 100644 --- a/concurrency/tests/death_boundary.rs +++ b/concurrency/tests/death_boundary.rs @@ -56,6 +56,36 @@ impl Message for Block { type Result = (); } +/// A payload whose destructor waits (at most 3 s) for a queued caller's +/// answer, and records whether it arrived: destroyed before the death point +/// answers that caller, it waits in vain. +struct WaitsForAnswer { + caller: std::sync::Arc>>>, + answered: std::sync::Arc, +} +impl Drop for WaitsForAnswer { + fn drop(&mut self) { + let began = Instant::now(); + while began.elapsed() < Duration::from_secs(3) { + let mut caller = self.caller.lock().unwrap(); + if let Some(receiver) = caller.as_mut() { + if receiver.try_recv().is_some() { + self.answered + .store(true, std::sync::atomic::Ordering::SeqCst); + return; + } + } + drop(caller); + std::thread::sleep(Duration::from_millis(1)); + } + } +} +/// A job's message carrying such a payload. +struct Waiting(#[allow(dead_code)] WaitsForAnswer); +impl Message for Waiting { + type Result = (); +} + /// Wait until `poll` yields, at most `LIMIT`. fn eventually(what: &str, mut poll: impl FnMut() -> Option) -> T { let began = Instant::now(); @@ -116,6 +146,85 @@ mod tasks_mode { } } + impl Handler for Owner { + async fn handle(&mut self, _: Waiting, _: PendingReply<'_, ()>, _: &Context) {} + } + + /// Review round 2 (Codex 1): the owner's exit cancels its jobs without + /// destroying their payloads; a scheduled emit's message and a held + /// result are destroyed after the death point answered every caller. + /// Cancellation used to destroy them first: a destructor waiting for a + /// queued caller's answer waited for an answer that could not come. + #[test] + fn job_payloads_are_destroyed_after_every_caller_is_answered() { + use spawned_concurrency::tasks::JobOptions; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Mutex; + for held in [false, true] { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let _guard = runtime.enter(); + let actor = Owner { + _state: inert(), + parked: Vec::new(), + } + .start(); + let caller = Arc::new(Mutex::new(None)); + let answered = Arc::new(AtomicBool::new(false)); + let payload = WaitsForAnswer { + caller: caller.clone(), + answered: answered.clone(), + }; + let (scheduled_tx, scheduled) = mpsc::channel(); + runtime.block_on(async { + actor + .__in_turn(move |_, ctx| { + ctx.pipe_with_ctx( + JobOptions::default(), + move |job| async move { + let far = spawned_concurrency::Instant::now() + LIMIT * 10; + if held { + // The result waits for this emit: held. + job.emit_at(far, Queued(inert())).await.unwrap(); + scheduled_tx.send(()).unwrap(); + Some(payload) + } else { + job.emit_at(far, Waiting(payload)).await.unwrap(); + scheduled_tx.send(()).unwrap(); + job.cancelled().await; + None + } + }, + |_| Queued(inert()), + ) + .unwrap(); + }) + .unwrap() + .await + .unwrap(); + }); + scheduled.recv_timeout(LIMIT).unwrap(); + let (unblock, blocked) = mpsc::channel(); + let (entered_tx, entered) = mpsc::channel(); + actor.send(Block(blocked, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + let queued = actor + .request_raw_until(Queued(inert()), spawned_concurrency::Deadline::Never) + .unwrap(); + *caller.lock().unwrap() = Some(queued); + actor.context().stop(); + unblock.send(()).unwrap(); + runtime.block_on(actor.join()); + assert!( + answered.load(Ordering::SeqCst), + "a job payload (held: {held}) was destroyed before the queued caller was answered" + ); + } + } + /// Review round 2 (Codex 5): the death point answers every caller before /// any wake-up runs. The first queued caller's waker waits for the second /// caller's answer; closing the first slot used to run that waker before @@ -370,6 +479,58 @@ mod threads_mode { } } + impl Handler for Owner { + fn handle(&mut self, _: Waiting, _: PendingReply<'_, ()>, _: &Context) {} + } + + /// Review round 2 (Codex 1), threads mode: a scheduled emit's message + /// is destroyed after the death point answered every caller. + #[test] + fn job_payloads_are_destroyed_after_every_caller_is_answered() { + use spawned_concurrency::threads::JobOptions; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + let actor = Owner.start(); + let caller = Arc::new(Mutex::new(None)); + let answered = Arc::new(AtomicBool::new(false)); + let payload = WaitsForAnswer { + caller: caller.clone(), + answered: answered.clone(), + }; + let (scheduled_tx, scheduled) = mpsc::channel(); + spawned_test::threads::in_actor(&actor, move |_, ctx| { + ctx.pipe_blocking_with( + JobOptions::default(), + move |job| { + let far = spawned_concurrency::Instant::now() + LIMIT * 10; + job.emit_at_blocking(far, Waiting(payload)).unwrap(); + scheduled_tx.send(()).unwrap(); + while !job.is_cancelled() { + std::thread::sleep(Duration::from_millis(1)); + } + }, + |_| Queued(inert()), + ) + .unwrap(); + }); + scheduled.recv_timeout(LIMIT).unwrap(); + let (unblock, blocked) = mpsc::channel(); + let (entered_tx, entered) = mpsc::channel(); + actor.send(Block(blocked, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + let queued = actor + .request_raw_until(Queued(inert()), spawned_concurrency::Deadline::Never) + .unwrap(); + *caller.lock().unwrap() = Some(queued); + actor.context().stop(); + unblock.send(()).unwrap(); + actor.join(); + assert!( + answered.load(Ordering::SeqCst), + "a job payload was destroyed before the queued caller was answered" + ); + } + /// Review P0 (Codex 1), threads mode. #[test] fn queued_requests_are_answered_before_payload_destructors() { diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index f85e5009..44faf9fd 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -399,11 +399,15 @@ current turn's slot) and a job set. The order is: 6. the deadline heap is dropped; 7. the exit reason is published, and monitors and links fire. - Steps 1 to 4 run no user code. Steps 3 and 4 follow the seal instead of - being one atomic step with it, but nothing between them depends on the - requests they answer (no destructor, notice or publication), so the - models' atomic death is a sound abstraction of them (they commute left - to the seal). + Steps 1 to 4 run no user code: steps 3 and 4 only make terminal + transitions; the callers' wake-ups (wakers are executor or user code), + and a runtime drop's "stopped() skipped" log, run after the last answer + and before step 5. Cancellation in step 1 only marks the jobs: the + payloads it releases (held results, scheduled emits) are destroyed in + step 5. Steps 3 and 4 follow the seal instead of being one atomic step + with it, but nothing between them depends on the requests they answer, + so the models' atomic death is a sound abstraction of them (they commute + left to the seal). `ActorStopped` keeps one meaning, "the actor is gone". An observer that sees `Down` knows every caller of the instance has been answered. From 28d66d5016e967ead49522f271a9567fff4d3539 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:01:08 +0000 Subject: [PATCH 130/208] fix(verification): restore SupervisionInd induction after merging main into #240 The merge combined main's #233 Supervision.tla changes with #240's S1 changes (no retry of a failed first start; Transient no longer restarts after a startup failure in DesignAllowsRestart; deferral of ready-child exits during the initial start), and Apalache had never checked the result (CI does not run on PRs into feature branches). Induction failed on two unreachable pre-states: - step: a controller at "attempt" with gen 0 but initial TRUE. initial is set only when a generation becomes ready and gen never decreases, so H_first now states initial => gen > 0. - actions (RestartFollowsPolicy): a ready generation restarting with exitR "none". A generation under construction is never ready (Attempt clears readyGen), and a ready generation leaves "running" only through SelfExit or StopObserved, which record a self-exit reason; new helper H_exit states both. No change to Supervision.tla or to the properties. Apalache: Supervision init/step/actions hold at N=5/R=4 and N=10/R=10; every probe and negative control (Supervision, Cohorts, Nested, Coordinator) is still violated. --- verification/tla/SupervisionInd.tla | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/verification/tla/SupervisionInd.tla b/verification/tla/SupervisionInd.tla index 81a879f2..0903aec7 100644 --- a/verification/tla/SupervisionInd.tla +++ b/verification/tla/SupervisionInd.tla @@ -92,11 +92,22 @@ H_retry == \A c \in Children : \* so Decide's deferral guard (open, ~sready, readyGen) leaves no restart. \* A deferred controller holds a ready generation's exit. H_first == \A c \in Children : + /\ initial[c] => gen[c] > 0 /\ gen[c] <= 1 /\ initial[c] => readyGen[c] /\ gen[c] > 1 => sready /\ pc[c] \in {"top", "backoff", "attempt"} /\ gen[c] > 0 => sready /\ pc[c] = "deferred" => readyGen[c] +\* A generation under construction is not ready yet (Attempt cleared +\* readyGen). A ready generation leaves "running" only through SelfExit or StopObserved, +\* which record a self-exit reason; "none" and "timeout" are startup failures +\* of a generation that never became ready (S1 removed them from the +\* Transient restart table, so RestartFollowsPolicy needs this). +H_exit == \A c \in Children : + /\ pc[c] = "constructing" => ~readyGen[c] \* Attempt cleared it + /\ pc[c] \in {"cleanup", "decide", "deferred", "top", "backoff", "attempt"} + /\ gen[c] > 0 /\ readyGen[c] => exitR[c] \in SelfExits + \* Reverse-order shutdown: while stopping, every child after sidx has \* finished or was abandoned on timeout (cdone and timedOut only grow). H_shutdown == @@ -126,7 +137,7 @@ SafetyInd == Helpers == /\ H_gen /\ H_launch /\ H_initial - /\ H_retry /\ H_first /\ H_shutdown /\ H_done /\ H_budget + /\ H_retry /\ H_first /\ H_exit /\ H_shutdown /\ H_done /\ H_budget IndInv == TypeInd /\ H_alive /\ H_pub /\ Helpers /\ SafetyInd From 6267477ae86c70a0d684ace84042f058a98d6007 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:15:14 -0300 Subject: [PATCH 131/208] fix(tasks): a runtime drop reaches the death point before destroying a pending turn Round-two review (Codex 2, agy 1). When the runtime dropped a polled actor while `started()`, a turn (handler, `exit_received`, a job's `done`) or `stopped()` was pending, the pending lifecycle future, with its captures (the message, the handler's locals, the turn's `PendingReply`), was destroyed before `ExitOnDrop` reached D: a blocking destructor there left the turn's caller, the deferred tokens and the queued requests unanswered, a reply made from such a destructor still succeeded, and a destructor panic escaped into the runtime's teardown. Each pending user future now runs inside `DeathFirst`. If it is dropped while pending, its `Drop` reaches D (`die_dropped`), answers the turn's own caller (its slot, taken from the envelope before dispatch) and the mailbox, runs the wake-ups (`discharged`), destroys the queued payloads, and only then drops the future, inside `catch_unwind` (a panic is a cleanup failure). A panic while polling is still a turn panic: the future is dropped before the panic continues. The never-polled path now destroys its drained envelopes one at a time, so one panicking destructor cannot unwind into the next. Also: the death_boundary gates wait 6 x LIMIT, so no test can pass because a destructor released itself inside its assertion window. Test: death_boundary tasks_mode::a_dropped_pending_turn_is_destroyed_after_every_caller_is_answered (the turn's caller and a `Deadline::Never` queued caller; failed before). Full concurrency suite, failpoints, sim (236) and the 75 lifecycle traces pass. --- concurrency/src/actor_cleanup.rs | 35 +++++++-- concurrency/src/reply/mod.rs | 8 ++ concurrency/src/reply/registry.rs | 16 ++-- concurrency/src/tasks/actor.rs | 113 ++++++++++++++++++++++++++-- concurrency/src/threads/actor.rs | 10 ++- concurrency/tests/death_boundary.rs | 65 +++++++++++++++- 6 files changed, 224 insertions(+), 23 deletions(-) diff --git a/concurrency/src/actor_cleanup.rs b/concurrency/src/actor_cleanup.rs index 6da9e7c4..38b17dfe 100644 --- a/concurrency/src/actor_cleanup.rs +++ b/concurrency/src/actor_cleanup.rs @@ -177,6 +177,18 @@ impl ActorCleanup { reason } + /// Answer a destroyed turn's own caller at the death point (its slot is + /// held by the turn's `PendingReply`, not listed in the registry). + #[cfg_attr(not(feature = "async"), allow(dead_code))] + pub(crate) fn answer_turn( + &self, + caller: std::sync::Arc, + ) { + if let Some(replies) = &self.replies { + replies.answer_turn(caller); + } + } + /// Every obligation of the death point is answered (the mode answered /// its queued requests after [`die`](Self::die)): now run the deferred /// wake-ups, then log a skipped `stopped()`, then destroy what the jobs' @@ -276,15 +288,15 @@ pub(crate) fn finish_actor_state(actor: A) -> Option { pub(crate) struct ActorStartup { payload: Option

, cleanup: Option>, - /// Answers the queued requests and returns what it drained, destroyed - /// with the payload once the wake-ups ran. - discharge: fn(&mut P) -> Box, + /// Answers the queued requests and returns what it drained, each item + /// destroyed on its own (contained) once the wake-ups ran. + discharge: fn(&mut P) -> Vec>, } impl ActorStartup { pub(crate) fn new( payload: P, cleanup: ActorCleanup, - discharge: fn(&mut P) -> Box, + discharge: fn(&mut P) -> Vec>, ) -> Self { Self { payload: Some(payload), @@ -308,8 +320,21 @@ impl Drop for ActorStartup { if let Some(cleanup) = &mut self.cleanup { cleanup.discharged(); } + // One at a time: a panicking destructor must not unwind into the + // next one (that would abort). + for item in queued { + if let Err(panic) = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| drop(item))) + { + let message = crate::error::panic_message(&*panic); + crate::mailbox::dispose_panic(panic); + if let Some(cleanup) = &mut self.cleanup { + cleanup.cleanup_failed(message); + } + } + } if let Err(panic) = - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| drop((queued, payload)))) + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| drop(payload))) { let message = crate::error::panic_message(&*panic); crate::mailbox::dispose_panic(panic); diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 87bf7b99..8f9094bd 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -178,6 +178,14 @@ impl SlotHandle { pub(crate) fn take(&mut self) -> Option>> { self.0.take() } + /// The request's slot, type-erased (the turn's caller, for the death + /// point to answer if the turn is destroyed unfinished). + #[cfg_attr(not(feature = "async"), allow(dead_code))] + pub(crate) fn caller(&self) -> Option> { + self.0 + .as_ref() + .map(|slot| slot.clone() as Arc) + } } impl Drop for SlotHandle { diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index ef6912cc..d7c0a501 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -37,7 +37,6 @@ use super::turn::{Outcome, TurnPhase, DEAD, EXITING}; /// A slot seen without its reply type. pub(crate) trait Closeable: Send + Sync { - fn close(&self) -> bool; /// Close without waking; the returned work runs with `run_wake`. fn close_collect(&self) -> Option; fn run_wake(&self, wake: Wake); @@ -45,6 +44,9 @@ pub(crate) trait Closeable: Send + Sync { fn is_terminal(&self) -> bool; } +/// A closed slot and its deferred wake-up. +type PendingWake = (Arc, Wake); + impl Expirable for Slot { fn pending_deadline(&self) -> Option { let deadline = self.deadline(); @@ -56,9 +58,6 @@ impl Expirable for Slot { } impl Closeable for Slot { - fn close(&self) -> bool { - Slot::close(self) - } fn close_collect(&self) -> Option { Slot::close_collect(self) } @@ -106,7 +105,7 @@ pub(crate) struct Registry { /// [`end_discharge`](Self::end_discharge)): the wake-ups of the slots it /// closed, run only once every obligation is answered, so no waker runs /// between two closes. - discharge: Mutex, Wake)>>>, + discharge: Mutex>>, #[cfg(all(spawned_sim, feature = "async"))] sim: deadline::SimService, } @@ -409,6 +408,13 @@ impl Registry { } } + /// A turn destroyed unfinished (its runtime dropped the actor): its + /// caller is answered at the death point like every other one. + #[cfg_attr(not(feature = "async"), allow(dead_code))] + pub(crate) fn answer_turn(&self, caller: Arc) { + self.close_slot(caller); + } + /// Close one slot as `STOPPED`, deferring its wake-up while discharging. fn close_slot(&self, slot: Arc) -> bool { let Some(wake) = slot.close_collect() else { diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 302d4216..af7811c4 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -167,6 +167,12 @@ trait Envelope: Send { /// mailbox is destroyed. No user code runs. fn discharge(&mut self) {} + /// The request's caller, if any (answered at the death point if the + /// runtime destroys the turn before it finishes). + fn caller(&self) -> Option> { + None + } + fn handle<'a>( self: Box, actor: &'a mut A, @@ -195,8 +201,14 @@ type StartupPayload = ( mpsc::Receiver + Send>>, CancellationToken, ); -fn discharge_startup(payload: &mut StartupPayload) -> Box { - Box::new(answer_mailbox(&mut payload.2)) +fn discharge_startup(payload: &mut StartupPayload) -> Vec> { + erase(answer_mailbox(&mut payload.2)) +} +fn erase(items: Vec) -> Vec> { + items + .into_iter() + .map(|item| Box::new(item) as Box) + .collect() } struct MessageEnvelope { @@ -234,6 +246,9 @@ where fn discharge(&mut self) { drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); } + fn caller(&self) -> Option> { + self.slot.caller() + } fn handle<'a>( self: Box, actor: &'a mut A, @@ -309,6 +324,9 @@ impl Envelope for InTurnEnvelope { fn discharge(&mut self) { drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); } + fn caller(&self) -> Option> { + self.slot.caller() + } fn handle<'a>( self: Box, actor: &'a mut A, @@ -1669,6 +1687,79 @@ impl MessageTurn { } } +/// A pending user future of the lifecycle: `started()`, a turn (handler, +/// `exit_received`, a job's `done`) or `stopped()`. If the runtime drops the +/// lifecycle while it is pending (G1), its `Drop` reaches the death point +/// first: the turn's own caller, every deferred token and every queued +/// request are answered and woken before this future and its captures (the +/// message, the handler's locals) are destroyed. A destructor panic of the +/// future is a cleanup failure. +struct DeathFirst<'a, A: Actor, C: Fn()> { + inner: Option + Send + 'a>>>, + rx: &'a mut mpsc::Receiver + Send>>, + cleanup: &'a mut crate::actor_cleanup::ActorCleanup, + caller: Option>, +} +impl<'a, A: Actor, C: Fn()> DeathFirst<'a, A, C> { + fn new( + inner: Pin + Send + 'a>>, + rx: &'a mut mpsc::Receiver + Send>>, + cleanup: &'a mut crate::actor_cleanup::ActorCleanup, + caller: Option>, + ) -> Self { + Self { + inner: Some(inner), + rx, + cleanup, + caller, + } + } +} +impl Future for DeathFirst<'_, A, C> { + type Output = (); + fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<()> { + let this = self.get_mut(); + let inner = this.inner.as_mut().expect("polled after completion"); + match std::panic::catch_unwind(AssertUnwindSafe(|| inner.as_mut().poll(cx))) { + Ok(std::task::Poll::Pending) => std::task::Poll::Pending, + Ok(std::task::Poll::Ready(())) => { + this.inner = None; + std::task::Poll::Ready(()) + } + // A panic is the turn's (or hook's) panic, not a runtime drop: + // the future is gone before the panic continues. + Err(panic) => { + let inner = this.inner.take(); + if let Err(secondary) = std::panic::catch_unwind(AssertUnwindSafe(|| drop(inner))) { + crate::mailbox::dispose_panic(secondary); + } + std::panic::resume_unwind(panic) + } + } + } +} +impl Drop for DeathFirst<'_, A, C> { + fn drop(&mut self) { + let Some(inner) = self.inner.take() else { + return; + }; + // Destroyed while pending: the runtime dropped the actor. + self.cleanup.die_dropped(); + if let Some(caller) = self.caller.take() { + self.cleanup.answer_turn(caller); + } + let queued = answer_mailbox(self.rx); + self.cleanup.discharged(); + self.rx.discard_all(queued); + if let Err(panic) = std::panic::catch_unwind(AssertUnwindSafe(|| drop(inner))) { + let message = panic_message(&*panic); + crate::mailbox::dispose_panic(panic); + self.cleanup + .cleanup_failed(format!("dropped turn destructor: {message}")); + } + } +} + /// The actor's state and mailbox while its lifecycle runs. They are /// destroyed only after the death point D: normally at the end of /// [`run_actor`]; if the lifecycle future is dropped mid-way (its runtime was @@ -1726,6 +1817,7 @@ async fn run_actor( owned.actor.as_mut().expect("the state lives until death"), ctx, &mut owned.rx, + &mut *owned.cleanup, cancellation_token, #[cfg(feature = "backend-experiments")] turn, @@ -1749,10 +1841,11 @@ async fn run_actor( owned.finish(false); } -async fn run_actor_lifecycle( +async fn run_actor_lifecycle( actor: &mut A, ctx: Context, rx: &mut mpsc::Receiver + Send>>, + cleanup: &mut crate::actor_cleanup::ActorCleanup, cancellation_token: CancellationToken, #[cfg(feature = "backend-experiments")] mut turn: Option, ) { @@ -1767,7 +1860,7 @@ async fn run_actor_lifecycle( } let start_result = AssertUnwindSafe(async { fail_point!("actor.before_started", async); - actor.started(&ctx).await + DeathFirst::new(Box::pin(actor.started(&ctx)), &mut *rx, &mut *cleanup, None).await }) .catch_unwind() .await; @@ -1858,9 +1951,13 @@ async fn run_actor_lifecycle( // manual `Handler` may panic before it returns one, and that // is a handler panic like any other (threads mode catches the // same call). - let result = AssertUnwindSafe(async { envelope.handle(actor, &ctx).await }) - .catch_unwind() - .await; + let caller = envelope.caller(); + let result = AssertUnwindSafe(async { + let turn = envelope.handle(actor, &ctx); + DeathFirst::new(turn, &mut *rx, &mut *cleanup, caller).await + }) + .catch_unwind() + .await; ctx.shared.replies.end_turn(result.is_err()); #[cfg(feature = "diagnostics")] drop(_turn); @@ -1933,7 +2030,7 @@ async fn run_actor_lifecycle( } let stop_result = AssertUnwindSafe(async { fail_point!("actor.before_stopped", async); - actor.stopped(&ctx).await + DeathFirst::new(Box::pin(actor.stopped(&ctx)), &mut *rx, &mut *cleanup, None).await }) .catch_unwind() .await; diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index f20d0fe5..a6c2674c 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -130,8 +130,14 @@ type StartupPayload = ( mpsc::Receiver>, CancellationToken, ); -fn discharge_startup(payload: &mut StartupPayload) -> Box { - Box::new(answer_mailbox(&mut payload.2, None)) +fn discharge_startup(payload: &mut StartupPayload) -> Vec> { + erase(answer_mailbox(&mut payload.2, None)) +} +fn erase(items: Vec) -> Vec> { + items + .into_iter() + .map(|item| Box::new(item) as Box) + .collect() } struct MessageEnvelope { diff --git a/concurrency/tests/death_boundary.rs b/concurrency/tests/death_boundary.rs index 8976dc2c..da711ef4 100644 --- a/concurrency/tests/death_boundary.rs +++ b/concurrency/tests/death_boundary.rs @@ -11,8 +11,9 @@ use std::time::{Duration, Instant}; const LIMIT: Duration = Duration::from_secs(5); -/// A payload whose destructor blocks until released (or `LIMIT`), and -/// reports that it started. +/// A payload whose destructor blocks until released, and reports that it +/// started. It never releases itself within an assertion window (`LIMIT`): +/// a test that answers only after destruction fails, never passes late. struct Gated { started: Option>, release: Option>, @@ -21,7 +22,7 @@ impl Drop for Gated { fn drop(&mut self) { if let (Some(started), Some(release)) = (self.started.take(), self.release.take()) { let _ = started.send(()); - let _ = release.recv_timeout(LIMIT); + let _ = release.recv_timeout(LIMIT * 6); } } } @@ -225,6 +226,64 @@ mod tasks_mode { } } + /// Holds its message (and its destructor) across a pending await. + struct Hang(#[allow(dead_code)] Gated, mpsc::Sender<()>); + impl Message for Hang { + type Result = (); + } + impl Handler for Owner { + async fn handle(&mut self, msg: Hang, _: PendingReply<'_, ()>, _: &Context) { + let msg = msg; + let _ = msg.1.send(()); + std::future::pending::<()>().await; + drop(msg); + } + } + + /// Review round 2 (Codex 2, agy 1): a runtime that drops an actor while + /// a turn is pending destroys the turn (its message, the handler's + /// locals) only after the death point answered the turn's own caller and + /// every queued one. The pending lifecycle future used to be destroyed + /// first, so a blocking destructor in it left every caller waiting. + #[test] + fn a_dropped_pending_turn_is_destroyed_after_every_caller_is_answered() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (payload, destroying, release) = gated(); + let (entered_tx, entered) = mpsc::channel(); + let never = spawned_concurrency::Deadline::Never; + let (actor, mut turn_caller, mut queued) = runtime.block_on(async { + let actor = Owner { + _state: inert(), + parked: Vec::new(), + } + .start(); + let turn_caller = actor + .request_raw_until(Hang(payload, entered_tx), never) + .unwrap(); + let queued = actor.request_raw_until(Queued(inert()), never).unwrap(); + while entered.try_recv().is_err() { + tokio::time::sleep(Duration::from_millis(1)).await; + } + (actor, turn_caller, queued) + }); + let dropper = std::thread::spawn(move || drop(runtime)); + // The pending turn's message is being destroyed (and blocks). + destroying.recv_timeout(LIMIT).unwrap(); + eventually( + "the dropped turn's caller waited for its destruction", + || stopped(turn_caller.try_recv(), &ExitReason::Killed), + ); + eventually("a queued caller waited for the turn's destruction", || { + stopped(queued.try_recv(), &ExitReason::Killed) + }); + release.send(()).unwrap(); + dropper.join().unwrap(); + assert_eq!(actor.exit_reason(), Some(ExitReason::Killed)); + } + /// Review round 2 (Codex 5): the death point answers every caller before /// any wake-up runs. The first queued caller's waker waits for the second /// caller's answer; closing the first slot used to run that waker before From a36bff67268a13c7ae3cc10fe9eeea53547c1e17 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:46:38 -0300 Subject: [PATCH 132/208] fix(tasks): take a blocking job's scope lease at admission, not reservation Round-two review (Codex 4): the managed scope lease that keeps a dead generation owned while its blocking job runs was taken in `reserve_native` and stored in the permit. An unused `BlockingJobPermit` held outside the actor (it is a public, storable value) is not in the job set, so the owner's exit could neither cancel nor reclaim it: the stopped actor's scope never joined, a supervisor reported stuck cleanup instead of restarting, and system shutdown stayed incomplete until the permit was dropped. The lease is now taken when the job is admitted (`admit_native`: lease, then `admit_job`), refused once the scope is closing (the owner exits), and handed to the native runner, which still releases it only after the body ends (Codex 7's guarantee). A late permit gets the existing refusal: a cancelled handle, or `OwnerStopping` with the token for `pipe_reply_blocking_with`. Tests (native_jobs): an_unused_permit_does_not_hold_the_scope_owned_runtime and ..._borrowed_runtime (both failed before: shutdown incomplete); a_supervisor_does_not_replace_a_child_whose_blocking_job_still_runs still passes. --- concurrency/src/tasks/jobs.rs | 58 +++++++++++++++++++++----------- concurrency/tests/native_jobs.rs | 47 ++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 19 deletions(-) diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index 9630ec14..780f1df7 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -243,9 +243,11 @@ impl Drop for NativeSlot { /// so that launching it cannot fail (threads mode reserves its thread the /// same way). struct NativeRunner { - /// A managed actor's scope lease, held until the body physically ends: - /// the actor's cleanup is not complete (no supervisor replacement, no - /// complete system shutdown) while its native work still runs. + /// A managed actor's scope lease, taken when the job is admitted (never + /// by an unused permit, which must not keep a dead actor's scope owned) + /// and held until the body physically ends: the actor's cleanup is not + /// complete (no supervisor replacement, no complete system shutdown) + /// while its native work still runs. lease: Option, pool: RunnerPool, } @@ -553,12 +555,12 @@ impl BlockingJobPermit { let BlockingJobPermit { ctx, credit, - native, + mut native, } = self; let Some(state) = ctx.start_job(&opts) else { return ctx.refused_job(&opts, (work, done, native)); }; - if !ctx.admit_job(&state, &opts) { + if !ctx.admit_native(&state, &opts, &mut native) { return ctx.refuse(state, (work, done, native, credit)); } ctx.launch_blocking::( @@ -615,9 +617,9 @@ impl Context { /// Reserve a native slot (`MAX_NATIVE_JOBS`) and where the job will run: /// the blocking pool, or an owned thread spawned now (`SpawnFailed` - /// here, never at launch), plus a managed actor's scope lease - /// (`OwnerStopping` once its scope is closing). A failure releases the - /// slot. + /// here, never at launch). A failure releases the slot. A managed + /// actor's scope lease is taken later, at admission + /// ([`admit_native`](Self::admit_native)). fn reserve_native(&self) -> Result, PipeErrorKind> { if !self.shared.jobs.reserve_native() { return Err(PipeErrorKind::NativeBudget); @@ -626,14 +628,6 @@ impl Context { ctx: self.clone(), runner: None, }; - let lease = match &self.shared.managed { - Some(scope) => Some( - scope - .job_lease() - .map_err(|_| PipeErrorKind::OwnerStopping)?, - ), - None => None, - }; #[cfg(not(spawned_sim))] let handle = match &self.shared.managed { Some(scope) => Some(scope.handle.clone()), @@ -658,10 +652,36 @@ impl Context { RunnerPool::Thread(start) } }; - slot.runner = Some(NativeRunner { lease, pool }); + slot.runner = Some(NativeRunner { lease: None, pool }); Ok(slot) } + /// Admit a blocking job: a managed actor's scope lease (refused once + /// its scope is closing, as the owner exits), then + /// [`admit_job`](Self::admit_job). The lease goes to the native runner + /// and lives until the body ends; refused, it is released here. + fn admit_native( + &self, + state: &Arc, + opts: &JobOptions, + native: &mut NativeSlot, + ) -> bool { + let lease = match &self.shared.managed { + Some(scope) => match scope.job_lease() { + Ok(lease) => Some(lease), + Err(_) => return false, + }, + None => None, + }; + if !self.admit_job(state, opts) { + return false; + } + if let Some(runner) = &mut native.runner { + runner.lease = lease; + } + true + } + /// Reserve an unlinked job's admission (the owner is running, one /// `MAX_OUTSTANDING` credit) before building `work` and `done`: a /// refusal leaves the caller everything it would have moved into them, @@ -993,7 +1013,7 @@ impl Context { { let (state, slot, token) = self.link_reply(&opts, token)?; let hold = Hold::Linked(LinkedJob(slot.clone())); - let native = match self.reserve_native() { + let mut native = match self.reserve_native() { Ok(native) => native, // Dropping the hold detaches the linked job from the slot. Err(kind) => { @@ -1001,7 +1021,7 @@ impl Context { return Err(PipeReplyError::Failed { token, kind }); } }; - if !self.admit_job(&state, &opts) { + if !self.admit_native(&state, &opts, &mut native) { drop((hold, native)); return Err(PipeReplyError::Failed { token, diff --git a/concurrency/tests/native_jobs.rs b/concurrency/tests/native_jobs.rs index b895508d..ac0b120c 100644 --- a/concurrency/tests/native_jobs.rs +++ b/concurrency/tests/native_jobs.rs @@ -239,6 +239,53 @@ async fn a_job_admitted_after_its_owner_exited_never_runs() { actor.join().await; } +/// Review round 2 (Codex 4): an unused blocking permit held outside the +/// actor does not keep a dead generation's scope +/// owned. Its scope lease used to be taken at reservation, so the stopped +/// actor's scope could not join, and a system shutdown stayed incomplete, +/// until the external permit was dropped. The lease is now taken when the +/// job is admitted; an executing job still holds it until its body ends. +async fn an_unused_permit_does_not_hold_the_scope(borrowed: bool) { + use spawned_concurrency::tasks::{ActorStart, BlockingJobPermit}; + struct Owner; + impl Actor for Owner {} + impl Handler for Owner { + async fn handle(&mut self, _: Done, _: PendingReply<'_, ()>, _: &Context) {} + } + let system = ActorSystem::new().unwrap(); + let group = if borrowed { + system + .async_group_on("unused-permit-borrowed", &tokio::runtime::Handle::current()) + .unwrap() + } else { + system.async_group("unused-permit-owned", 1).unwrap() + }; + let actor = Owner.start_on(&group).unwrap(); + actor.ready().await.unwrap(); + // Reserved through a cloned context and kept outside the actor, unused. + let permit: BlockingJobPermit = actor.context().reserve_blocking_job().unwrap(); + actor.context().stop(); + assert_eq!(actor.wait_exit().await, spawned_concurrency::ExitReason::Normal); + let report = system.shutdown(Duration::from_secs(2)).await; + assert!( + report.complete, + "an unused blocking permit kept the dead actor's scope owned" + ); + // Using it now is refused: nothing runs. + let handle = permit.pipe_blocking_with(JobOptions::default(), |_| (), |_| Done); + assert!(handle.is_cancelled()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_unused_permit_does_not_hold_the_scope_owned_runtime() { + an_unused_permit_does_not_hold_the_scope(false).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_unused_permit_does_not_hold_the_scope_borrowed_runtime() { + an_unused_permit_does_not_hold_the_scope(true).await; +} + /// Review P1 (Codex 12, agy 6): outside a Tokio runtime a tasks actor's /// blocking job runs on an owned thread. Its creation is part of the /// reservation, so a failure is `SpawnFailed` (and `pipe_reply*` hands the From f41912ba80772898ef571fd1d26c00d0984b9d3f Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:48:00 -0300 Subject: [PATCH 133/208] fix(threads): an unused blocking permit returns its native slot on drop Round-two review (agy 2): `spawn_worker` moved the `MAX_NATIVE_JOBS` slot into the parked worker thread at reservation. Dropping an unused `BlockingJobPermit` (or a refused one) only dropped the start sender; the slot came back when the parked thread woke and exited, so an immediate reservation could fail with a spurious `NativeBudget`, contrary to "dropping an unused permit returns everything". The slot now stays in the `NativeWorker` and moves into the task at launch, as in tasks mode, so dropping the worker returns it synchronously on the dropping thread. Test: threads::actor::jobs::tests::a_dropped_unused_permit_returns_its_native_slot_at_once (max_native_jobs = 1, reserve and drop 2000 times; failed before on the second reservation). --- concurrency/src/threads/jobs.rs | 47 +++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index b489d3f4..81bce755 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -327,15 +327,17 @@ impl JobCtx { // Context API // --------------------------------------------------------------------------- -/// The task a native worker thread runs once its job is spawned; it gets -/// the thread's native slot so it can release it before notifying. -type NativeTask = Box) + Send>; +/// The task a native worker thread runs once its job is spawned. +type NativeTask = Box; /// A native thread spawned at reservation and parked until its job is -/// spawned, so spawning from a permit cannot fail. Dropped unused, the -/// thread exits and releases its native slot. +/// spawned, so spawning from a permit cannot fail, with its native slot. +/// The slot stays here until the job is launched (it then moves into the +/// task): dropped unused, the worker returns the slot at once, on the +/// dropping thread, and the parked thread exits. struct NativeWorker { - start: std::sync::mpsc::Sender>, + start: std::sync::mpsc::Sender, + native: NativeSlot, } /// A blocking job's admission reserved ahead of building the work: @@ -419,16 +421,16 @@ impl Context { return Err(PipeErrorKind::NativeBudget); } let native = NativeSlot(self.clone()); - let (start, started) = std::sync::mpsc::channel::>(); + let (start, started) = std::sync::mpsc::channel::(); std::thread::Builder::new() .name("spawned-job".into()) .spawn(move || { if let Ok(task) = started.recv() { - task(native); + task(); } }) .map_err(PipeErrorKind::SpawnFailed)?; - Ok(NativeWorker { start }) + Ok(NativeWorker { start, native }) } /// Reserve a blocking job's admission (owner running, one credit, a @@ -510,7 +512,8 @@ impl Context { state: state.clone(), ctx: self.clone(), }; - let task: NativeTask = Box::new(move |native| { + let NativeWorker { start, native } = worker; + let task: NativeTask = Box::new(move || { let _ended = ended; let result = std::panic::catch_unwind(AssertUnwindSafe(|| work(&job))); // Execution ended: release the native slot now; the run returns @@ -520,7 +523,7 @@ impl Context { }); // The worker thread is parked on `recv` until this send (or its // sender's drop); it cannot have exited. - let _ = worker.start.send(task); + let _ = start.send(task); JobHandle::new(state) } @@ -640,6 +643,28 @@ mod tests { fn handle(&mut self, _: Note, _: PendingReply<'_, ()>, _: &Context) {} } + /// Review round 2 (agy 2): dropping an unused permit returns its + /// `MAX_NATIVE_JOBS` slot synchronously, so the next reservation + /// succeeds at once. The slot used to live on the parked thread and + /// return only when that thread woke up and exited. + #[test] + fn a_dropped_unused_permit_returns_its_native_slot_at_once() { + struct One; + impl Actor for One { + const MAX_NATIVE_JOBS: Option = Some(1); + } + let owner = One.start(); + let ctx = owner.context(); + for i in 0..2000 { + let permit = ctx + .reserve_blocking_job() + .unwrap_or_else(|e| panic!("reservation {i} refused: {e}")); + drop(permit); + } + ctx.stop(); + owner.join(); + } + /// Review P1 (Codex 13), threads mode: see the tasks-mode test. #[test] fn a_refused_emit_returns_its_credit_once() { From 4435ae773c0f194c2e4e7d6c454a9a5e77b6768c Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:48:10 -0300 Subject: [PATCH 134/208] docs(semantics): sections 11 and 12 describe the per-round restart budget as implemented Round-two review (agy 3): the Differences-from-OTP row still said the budget counts every child factory call (planned for v0.8), and the planned-changes list still listed per-round charging. V5 and the code charge one grant per OneForAll/RestForOne recovery round (#240); the row now says it is aligned and the planned bullet is folded into the implemented v0.8 one. --- docs/design/semantics.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/design/semantics.md b/docs/design/semantics.md index 7f41d8b4..0751909c 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -735,7 +735,7 @@ remaining differences of the exit-signal design are listed in | v0.7 propagated a kill over a link as `Kill` | It arrives as `killed` | Aligned in v0.8 (rank 5e, #240): `Killed` (X4) | | v0.7 linking or monitoring an already-exited actor delivered its real exit reason | `noproc` | Aligned in v0.8 (rank 5e, #240): `NoProc` (X1, X5) | | A kill or late signal does not interrupt a running `stopped()` (L3) | The process dies at once | Rust-forced: the reason still becomes `Killed` or the signal's reason | -| The restart budget counts every child factory call, collateral ones included (V5) | Intensity counts recovery rounds | Planned v0.8, rank 5f | +| v0.7 charged the restart budget for every child factory call, collateral ones included | Intensity counts restart actions: one per `one_for_all`/`rest_for_one` round | Aligned in v0.8 (rank 5f, #240): a OneForAll or RestForOne recovery round takes one grant however many children it rebuilds (V5) | | Initial startup retried a Permanent or Transient child whose factory failed, panicked or timed out (V7), within the budget | `start_link` aborts on any child start error | Aligned in v0.8 (rank 5f, #240): any first-start failure fails `start()` | | v0.7 reported a runtime-dropped actor as `Kill` (tasks) or `Panic("unexpected framework panic")` (threads) | No counterpart | v0.8 (rank 5g, #240): `Killed` in both modes (X6) | | Stop is cooperative: a running handler or blocking call is never interrupted (L3) | `kill` ends a process at once | Rust-forced: Rust cannot preempt or kill a thread | @@ -750,8 +750,8 @@ remaining differences of the exit-signal design are listed in - v0.8, ranks 5e, 5f (startup) and 5g, exit signals, startup and runtime drop as in OTP ([otp-exit-signals.md](otp-exit-signals.md)): implemented on the exit-signals branch (#240, on top of rank 11's #239) and described in - L3-L6, X1-X6, V1, V3, V7 and V9 above. -- v0.8, rank 5f: the restart budget counts recovery rounds (changes V5). + L3-L6, X1-X6, V1, V3, V5 (the restart budget counts recovery rounds), V7 + and V9 above. - Rank 11, deferred typed replies, forwarding and pipe-to-self: a reply may leave the handler's turn; adds exactly-one completion on panic and timeout. From 2d5aed4b964bb9a280e87a099b9253cd27e3326e Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 12:48:40 -0300 Subject: [PATCH 135/208] docs: state the emit_at-past-deadline and raw-receiver clock contracts Round-two review (A2, A4): two behaviours the first round kept on purpose are now stated as contract, with the reason, in semantics.md (R3, R7) and the guide: - `emit_at` with `at` past `JobOptions::deadline` is accepted. The hard deadline takes the notification when it passes and fences the emit, which is dropped at its instant; if the job finishes first, the emit and then the result arrive. No error at the call: whether the deadline wins is decided when it passes, and jobs follow one rule. - A `ReplyReceiver` awaited directly is expired by the owner's deadline heap on the real (or simulated) clock, not paused Tokio time, to avoid a runtime timer per request; `request` and `Response::from` follow paused time. A shortened deadline wakes the caller either way. --- docs/GUIDE.md | 13 +++++++++++-- docs/design/semantics.md | 16 ++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 07dad401..23d8cadb 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -684,7 +684,11 @@ ctx.pipe_reply(token, async move { downstream?.await }, |res, token| Delegated { job's turn or a newer instance, it is dropped if the job is cancelled or replaced first, and the job's terminal message waits for it. An `at` already reached, or in the past, is an immediate emit (enqueued at the - call, ahead of later emits): no need to branch to `emit` yourself. + call, ahead of later emits): no need to branch to `emit` yourself. An + `at` past `JobOptions::deadline` is accepted, not an error: if the + deadline passes first it takes the notification and the emit is dropped + at its instant (if the job finishes first, the emit and then the result + arrive as usual). - **Job deadlines are hard.** When `JobOptions::deadline` (or `deadline_after(d)`) passes, `done` gets `DeadlineExceeded` at once, later emits fail with `JobGone` and the work's eventual result is dropped. The @@ -1491,7 +1495,12 @@ requests are observed by their blocking request; `Response::ready` does not coun that result a second time. Manually constructed ready results are not requests. `request_raw` returns a `ReplyReceiver`; it does not observe how the caller -waits on it, so it is not a caller-observed request. Such requests +waits on it, so it is not a caller-observed request. Its deadline clock is +part of the contract: awaited directly, a raw receiver is expired by the +owner's deadline heap on the real clock (the simulated one under +`spawned_sim`), not by a runtime timer, so it does not follow paused Tokio +time; that saves a timer per request on fan-out paths. Wrap it in +`Response::from(rx)`, or use `request`, when a test pauses Tokio time. Such requests still contribute message, handler, queue and reply-delivery metrics, but only a framework request/Response wrapper contributes caller-outcome/latency metrics. Reply delivery means the reply was stored in the request's slot, not proof the diff --git a/docs/design/semantics.md b/docs/design/semantics.md index 0751909c..dd51ccda 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -143,6 +143,15 @@ a server one. tasks: `request`-family futures enqueue on first poll and protocol methods enqueue when called; either way the deadline starts at the enqueue. threads: the call blocks. Expired requests are still dispatched (OTP); a late reply is dropped (`Delivery::Expired`). +Which clock expires a waiting caller is part of the contract: `request`, +`request_with_timeout` and `Response::from(rx)` arm the runtime's timer, so +they follow paused Tokio time; a `ReplyReceiver` awaited directly +(`request_raw(..)?.await`) puts its deadline in the owner's deadline heap +instead, serviced on the real clock (the simulated one under `spawned_sim`), +not paused Tokio time. The reason is cost: a raw receiver is the building +block for fan-out and needs no runtime timer per request. Tests that pause +Tokio time wait through `request` or `Response::from`. Either way a +shortened deadline (`ReplyTo::with_deadline`) wakes the waiting caller. Checked by: [`protocol_timeouts`][protocol_timeouts], [`cancellation_safety::protocol_calls_are_cancel_safe_and_send_eagerly`][cancellation_safety], [`protocol_timeouts`][ex_protocol_timeouts] example (both modes), @@ -194,6 +203,13 @@ work keeps its credit. Messages of cancelled or replaced (keyed) jobs are fenced at dispatch; a job's emits stay FIFO with the terminal message last. Owner exit cancels every job; they are joined after death, bounded by the cleanup deadline, and reported as `unfinished_at_stop`. +`JobCtx::emit_at(at, ..)` with `at` past `JobOptions::deadline` is accepted: +the hard deadline takes the terminal notification when it passes and fences +everything after it, so the emit is dropped at its instant (its credit +returns then). It is not an error at the call: whether the deadline wins is +decided when it passes (the job may finish first, delivering the emit and +then its result), and a job checks one rule, the deadline, rather than two. +An `at` already reached is an immediate emit. Checked by: [`deferred_jobs`][deferred_jobs]. ## 4. Lifecycle From 4ebc3bb2657b6f340dfe245257cf6ed3d3d0707d Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:02:47 -0300 Subject: [PATCH 136/208] verify(tla): model delayed emits and held results in DeferredJobs Round-two review (Codex 6): the job model's `Finish` notified at once, `JobDeadline` required running work and `Exit` cancelled only running jobs, so it could not represent a result held for scheduled emits, which the code now keeps (deadline armed, credit kept). Completion, the held notification, scheduled emits and credit ownership are now separate: - `ScheduleEmit` takes an emit credit; `FireScheduled` enqueues the emit if the notification is pending, and the last one releases a held result (terminal message, then the job's credit). - `Finish` with emits scheduled holds the result and keeps the credit; otherwise it notifies (or not) and the credit returns. - `JobDeadline` can take a held result's notification (dropping it and its credit); running work keeps its credit as before. - Cancellation and owner exit drop scheduled emits and a held result; exit also reaches jobs that are no longer running but hold either. - Admission counts credits (`Permits`), and `EmitCreditsBounded` counts scheduled emits. New invariants `HeldCounted` (running work and held results stay within the bound) and `DeadlineArmedWhileHeld`, with negative controls `EarlyCredit` and `HeldDeadlineLost` on the new `MCDeferredJobsHeld` configuration (per-PR `deferred` leg; 1,946,047 states). TLC: no error on MCDeferredJobs, MCDeferredJobsEmits, MCDeferredJobsHeld and MCDeferredJobsDeep; both controls violate exactly their invariant; 75/75 recorded traces (including the job traces, re-recorded) accepted and every negative control passes (`check.sh --traces-only --negative-controls`). --- docs/design/deferred-replies.md | 7 +- verification/tla/DeferredJobs.tla | 152 +++++++++++++++++++----- verification/tla/MCDeferredJobsHeld.cfg | 14 +++ verification/tla/MCDeferredJobsHeld.tla | 3 + verification/tla/README.md | 11 +- verification/tla/TraceJobs.tla | 9 +- verification/tla/negative_checks.py | 2 + 7 files changed, 160 insertions(+), 38 deletions(-) create mode 100644 verification/tla/MCDeferredJobsHeld.cfg create mode 100644 verification/tla/MCDeferredJobsHeld.tla diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index 44faf9fd..3736e88f 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -647,8 +647,11 @@ any emit. To keep the terminal message last, the work's result is **held** while scheduled emits remain: the last one to fire enqueues it. Cancellation drops the scheduled emits and a held result; a job deadline takes the notification as usual (deadlines are hard), and later scheduled emits are -then dropped at their instant. Not modelled in `DeferredJobs.tla`; Loom -model `scheduled_emit_vs_finish_and_cancel`. +then dropped at their instant. Modelled in `DeferredJobs.tla` +(`MCDeferredJobsHeld`: scheduled emits, the held result, the deadline taking +it and the job's credit kept until the terminal message; controls +`EarlyCredit` and `HeldDeadlineLost`); Loom model +`scheduled_emit_vs_finish_and_cancel`. **Fencing at dispatch.** Every job message carries the job's `JobState` and the owner's `ActorId`. The run loop drops it before the handler runs if: diff --git a/verification/tla/DeferredJobs.tla b/verification/tla/DeferredJobs.tla index c7dbaad0..423393da 100644 --- a/verification/tla/DeferredJobs.tla +++ b/verification/tla/DeferredJobs.tla @@ -34,12 +34,28 @@ (* ReplacePendingOnly(NoStaleKeyed) replacement skips notified jobs *) (* EmitAfterTerminal (TerminalLast) emits continue after the terminal *) (* NoEmitCredit (EmitCreditsBounded) *) +(* EarlyCredit (HeldCounted) a held result returns its credit *) +(* when the work ends *) +(* HeldDeadlineLost (DeadlineArmedWhileHeld) the deadline cannot take *) +(* a held result's notification *) +(* *) +(* Delayed emits (emit_at) and held results: a running job may schedule an *) +(* emit (it takes an emit credit at once and is enqueued, fenced like any *) +(* emit, when it fires). When the work ends while emits are scheduled, its *) +(* result is held: the terminal message is enqueued after the last *) +(* scheduled emit fires, so it stays last. Each job owns one *) +(* MAX_OUTSTANDING credit (`credit`) from admission until its terminal *) +(* message is enqueued, or, when the notification was taken or suppressed *) +(* otherwise, until execution ends: a held result keeps it (HeldCounted). *) +(* The job deadline can take a held result's notification (it stays armed *) +(* until the terminal message). *) (***************************************************************************) EXTENDS Naturals, FiniteSets, Sequences CONSTANTS J, Keyed, JobBound, MaxGen, MaxEmit, Credits, Fault ASSUME Fault \in {"None", "NoInstanceCheck", "NoCancelFence", "DoubleTerminal", "ReleaseOnTimeout", "KeyNoCancel", "ReplacePendingOnly", - "EmitAfterTerminal", "NoEmitCredit"} + "EmitAfterTerminal", "NoEmitCredit", "EarlyCredit", + "HeldDeadlineLost"} Jobs == 1..J ASSUME Keyed \subseteq Jobs @@ -55,10 +71,17 @@ VARIABLES started, \* order in which keyed jobs started (for NoStaleKeyed) mbox, \* FIFO sequence of [j, kind] termSent, \* terminal messages enqueued per job - seen \* history of dispatched messages + seen, \* history of dispatched messages + sched, \* scheduled (emit_at) emits not yet fired, each with a credit + held, \* the work ended with emits scheduled: its result is held + credit \* the job holds its MAX_OUTSTANDING credit vars == <> + started, mbox, termSent, seen, sched, held, credit>> + +\* Variables of delayed emits and held results, unchanged by the steps that +\* do not concern them. +heldVars == <> Init == /\ inst = 1 /\ alive = TRUE @@ -68,77 +91,141 @@ Init == /\ emitted = [j \in Jobs |-> 0] /\ inflight = [j \in Jobs |-> 0] /\ started = <<>> /\ mbox = <<>> /\ termSent = [j \in Jobs |-> 0] /\ seen = {} + /\ sched = [j \in Jobs |-> 0] /\ held = [j \in Jobs |-> FALSE] + /\ credit = [j \in Jobs |-> FALSE] Running == {j \in Jobs : exec[j] = "running"} -Permits == IF Fault = "ReleaseOnTimeout" THEN {j \in Running : notif[j] = "pending"} - ELSE Running +\* Jobs holding a MAX_OUTSTANDING credit (admission checks the bound). +Permits == {j \in Jobs : credit[j]} LiveKeyed == {j \in Keyed : owner[j] = inst /\ exec[j] # "none" /\ ~cancelled[j] /\ notif[j] = "pending"} \* Earlier keyed jobs a replacement must cancel. ToReplace == IF Fault = "ReplacePendingOnly" THEN LiveKeyed ELSE {j \in Keyed : owner[j] = inst /\ exec[j] # "none" /\ ~cancelled[j]} -CancelSet(S) == +\* Cancellation drops scheduled emits (their credits return) and a held +\* result, whose run then releases the job's credit. +CancelCore(S) == /\ cancelled' = [j \in Jobs |-> IF j \in S THEN TRUE ELSE cancelled[j]] /\ notif' = [j \in Jobs |-> IF j \in S /\ notif[j] = "pending" THEN "suppressed" ELSE notif[j]] + /\ sched' = [j \in Jobs |-> IF j \in S THEN 0 ELSE sched[j]] + /\ held' = [j \in Jobs |-> IF j \in S THEN FALSE ELSE held[j]] +CreditAfterCancel(S) == + [j \in Jobs |-> IF j \in S /\ exec[j] = "finished" THEN FALSE ELSE credit[j]] +CancelSet(S) == CancelCore(S) /\ credit' = CreditAfterCancel(S) Start(j) == /\ alive /\ exec[j] = "none" /\ Cardinality(Permits) < JobBound /\ exec' = [exec EXCEPT ![j] = "running"] /\ owner' = [owner EXCEPT ![j] = inst] /\ \E d \in BOOLEAN : hasDl' = [hasDl EXCEPT ![j] = d] - /\ IF j \in Keyed + /\ IF j \in Keyed /\ Fault # "KeyNoCancel" THEN /\ started' = Append(started, j) - /\ IF Fault = "KeyNoCancel" THEN UNCHANGED <> - ELSE CancelSet(ToReplace) - ELSE UNCHANGED <> + /\ CancelCore(ToReplace) + /\ credit' = [CreditAfterCancel(ToReplace) EXCEPT ![j] = TRUE] + ELSE /\ started' = IF j \in Keyed THEN Append(started, j) ELSE started + /\ credit' = [credit EXCEPT ![j] = TRUE] + /\ UNCHANGED <> /\ UNCHANGED <> +Accepting(j) == notif[j] = "pending" /\ ~cancelled[j] + CanEmit(j) == /\ exec[j] = "running" /\ emitted[j] < MaxEmit - /\ (notif[j] = "pending" /\ ~cancelled[j]) \/ Fault = "EmitAfterTerminal" - /\ inflight[j] < Credits \/ Fault = "NoEmitCredit" + /\ Accepting(j) \/ Fault = "EmitAfterTerminal" + /\ inflight[j] + sched[j] < Credits \/ Fault = "NoEmitCredit" Emit(j) == /\ CanEmit(j) /\ emitted' = [emitted EXCEPT ![j] = @ + 1] /\ inflight' = [inflight EXCEPT ![j] = @ + 1] /\ mbox' = Append(mbox, [j |-> j, kind |-> "emit"]) - /\ UNCHANGED <> + /\ UNCHANGED <> + +\* emit_at: take an emit credit now; the emit is enqueued when it fires. +ScheduleEmit(j) == + /\ exec[j] = "running" /\ emitted[j] < MaxEmit /\ Accepting(j) + /\ inflight[j] + sched[j] < Credits + /\ emitted' = [emitted EXCEPT ![j] = @ + 1] + /\ sched' = [sched EXCEPT ![j] = @ + 1] + /\ UNCHANGED <> + +\* A scheduled emit fires: enqueued while the notification is pending (else +\* dropped, its credit returning). The last one releases a held result: its +\* terminal message, then the job's credit. +FireScheduled(j) == + /\ sched[j] > 0 + /\ sched' = [sched EXCEPT ![j] = @ - 1] + /\ LET fires == Accepting(j) + release == held[j] /\ sched[j] = 1 + emitMsg == IF fires THEN <<[j |-> j, kind |-> "emit"]>> ELSE <<>> + doneMsg == IF release THEN <<[j |-> j, kind |-> "done"]>> ELSE <<>> + IN /\ mbox' = mbox \o emitMsg \o doneMsg + /\ inflight' = IF fires THEN [inflight EXCEPT ![j] = @ + 1] ELSE inflight + /\ IF release + THEN /\ held' = [held EXCEPT ![j] = FALSE] + /\ notif' = [notif EXCEPT ![j] = "notified"] + /\ termSent' = [termSent EXCEPT ![j] = @ + 1] + /\ credit' = [credit EXCEPT ![j] = FALSE] + ELSE UNCHANGED <> + /\ UNCHANGED <> -\* Physical completion; the notification is a one-shot. +\* Physical completion; the notification is a one-shot. With emits still +\* scheduled the result is held (keeping the credit); otherwise the terminal +\* message is enqueued, or nothing when the notification was already taken +\* or suppressed, and the credit returns. Finish(j) == /\ exec[j] = "running" /\ exec' = [exec EXCEPT ![j] = "finished"] - /\ IF notif[j] = "pending" \/ (Fault = "DoubleTerminal" /\ notif[j] = "notified") + /\ IF (Accepting(j) /\ sched[j] = 0) + \/ (Fault = "DoubleTerminal" /\ notif[j] = "notified") THEN /\ notif' = [notif EXCEPT ![j] = "notified"] /\ mbox' = Append(mbox, [j |-> j, kind |-> "done"]) /\ termSent' = [termSent EXCEPT ![j] = @ + 1] - ELSE UNCHANGED <> - /\ UNCHANGED <> + /\ credit' = [credit EXCEPT ![j] = FALSE] + /\ UNCHANGED held + ELSE IF Accepting(j) + THEN /\ held' = [held EXCEPT ![j] = TRUE] + /\ credit' = IF Fault = "EarlyCredit" THEN [credit EXCEPT ![j] = FALSE] + ELSE credit + /\ UNCHANGED <> + ELSE /\ credit' = [credit EXCEPT ![j] = FALSE] + /\ UNCHANGED <> + /\ UNCHANGED <> \* Job deadline (only for jobs started with one): the notification is -\* DeadlineExceeded now; the work keeps running and keeps its permit. +\* DeadlineExceeded now. Running work keeps running and keeps its credit; a +\* held result is dropped and its credit returns. JobDeadline(j) == - /\ hasDl[j] /\ exec[j] = "running" /\ notif[j] = "pending" + /\ hasDl[j] /\ notif[j] = "pending" + /\ exec[j] = "running" \/ (held[j] /\ Fault # "HeldDeadlineLost") /\ notif' = [notif EXCEPT ![j] = "notified"] /\ mbox' = Append(mbox, [j |-> j, kind |-> "deadline"]) /\ termSent' = [termSent EXCEPT ![j] = @ + 1] - /\ UNCHANGED <> + /\ held' = [held EXCEPT ![j] = FALSE] + /\ credit' = IF held[j] \/ Fault = "ReleaseOnTimeout" + THEN [credit EXCEPT ![j] = FALSE] ELSE credit + /\ UNCHANGED <> Cancel(j) == /\ alive /\ owner[j] = inst /\ exec[j] # "none" /\ ~cancelled[j] /\ CancelSet({j}) /\ UNCHANGED <> -\* Owner exit cancels the live job set. Queued messages stay addressed to -\* the slot (the conservative case); they are fenced at dispatch. +\* Owner exit cancels the live job set (running, or with emits scheduled or +\* a result held). Queued messages stay addressed to the slot (the +\* conservative case); they are fenced at dispatch. Exit == /\ alive /\ alive' = FALSE - /\ CancelSet({j \in Jobs : owner[j] = inst /\ exec[j] = "running"}) + /\ CancelSet({j \in Jobs : owner[j] = inst + /\ (exec[j] = "running" \/ sched[j] > 0 \/ held[j])}) /\ UNCHANGED <> Restart == /\ ~alive /\ inst < MaxGen /\ inst' = inst + 1 /\ alive' = TRUE - /\ UNCHANGED <> + /\ UNCHANGED <> Fenced(m) == \/ owner[m.j] # inst /\ Fault # "NoInstanceCheck" @@ -157,14 +244,17 @@ Dispatch == ELSE seen' = seen \cup {[j |-> m.j, kind |-> m.kind, owner |-> owner[m.j], at |-> inst, cancelled |-> cancelled[m.j], superseded |-> m.j \in Keyed /\ Pos(m.j) < Len(started)]} - /\ UNCHANGED <> + /\ UNCHANGED <> Next == \/ Exit \/ Restart \/ Dispatch \/ \E j \in Jobs : Start(j) \/ Emit(j) \/ Finish(j) \/ JobDeadline(j) \/ Cancel(j) + \/ ScheduleEmit(j) \/ FireScheduled(j) Spec == Init /\ [][Next]_vars -FairSpec == Spec /\ (\A j \in Jobs : WF_vars(JobDeadline(j))) /\ WF_vars(Dispatch) +FairSpec == Spec /\ (\A j \in Jobs : WF_vars(JobDeadline(j)) /\ WF_vars(FireScheduled(j))) + /\ WF_vars(Dispatch) ------------------------------------------------------------------------------ \* Properties @@ -183,17 +273,23 @@ NoStaleAfterCancel == \A s \in seen : ~s.cancelled NoStaleKeyed == \A s \in seen : ~s.superseded AtMostOneTerminal == \A j \in Jobs : termSent[j] <= 1 /\ Cardinality(TermSeen(j)) <= 1 PermitsBounded == Cardinality(Running) <= JobBound +\* Running work and held results count against the bound: a held result +\* keeps the job's credit until its terminal message. +HeldCounted == Cardinality({j \in Jobs : exec[j] = "running" \/ held[j]}) <= JobBound +\* The job deadline stays armed while a result is held. +DeadlineArmedWhileHeld == + \A j \in Jobs : (held[j] /\ hasDl[j] /\ notif[j] = "pending") => ENABLED JobDeadline(j) OneLivePerKey == Cardinality(LiveKeyed) <= 1 \* A job's terminal message is the last message it puts in the mailbox. TerminalLast == \A a, b \in 1..Len(mbox) : (a < b /\ mbox[a].j = mbox[b].j /\ IsTerm(mbox[a])) => FALSE -EmitCreditsBounded == \A j \in Jobs : inflight[j] <= Credits +EmitCreditsBounded == \A j \in Jobs : inflight[j] + sched[j] <= Credits Safety == /\ TypeOK /\ NoLateResult /\ NoStaleAfterCancel /\ NoStaleKeyed /\ AtMostOneTerminal /\ PermitsBounded /\ OneLivePerKey - /\ TerminalLast /\ EmitCreditsBounded + /\ TerminalLast /\ EmitCreditsBounded /\ HeldCounted /\ DeadlineArmedWhileHeld \* A job started with a deadline and never cancelled is reported to its \* live owner instance even if its work never finishes. diff --git a/verification/tla/MCDeferredJobsHeld.cfg b/verification/tla/MCDeferredJobsHeld.cfg new file mode 100644 index 00000000..a189b64b --- /dev/null +++ b/verification/tla/MCDeferredJobsHeld.cfg @@ -0,0 +1,14 @@ +\* Fast instance with delayed emits and held results (CI, every PR): three +\* unkeyed jobs, two permits, one instance, one emit per job, one credit. +\* Checks that a held result keeps its credit and its armed deadline. +SPECIFICATION FairSpec +INVARIANT Safety +PROPERTY JobReported +CONSTANTS + J = 3 + Keyed = {} + JobBound = 2 + MaxGen = 1 + MaxEmit = 1 + Credits = 1 + Fault = "None" diff --git a/verification/tla/MCDeferredJobsHeld.tla b/verification/tla/MCDeferredJobsHeld.tla new file mode 100644 index 00000000..911a435c --- /dev/null +++ b/verification/tla/MCDeferredJobsHeld.tla @@ -0,0 +1,3 @@ +------------------------ MODULE MCDeferredJobsHeld ------------------------ +EXTENDS DeferredJobs +============================================================================= diff --git a/verification/tla/README.md b/verification/tla/README.md index 0b41526e..b206d437 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -14,7 +14,7 @@ this fits with Loom, Shuttle, Miri, ThreadSanitizer and mutation testing. | `Coordinator.tla`, `MCCoordinator.tla/.cfg` | System shutdown coordinator: ordered stop of admitted roots, panic fallback that signals all remaining roots, records failure and retains runtime ownership until joins; ordered-shutdown invariants, cooperative cleanup liveness and four negative mutations. | | `NestedSupervision.tla`, `MCNested.tla/.cfg` | Dynamic subtree ownership and shutdown delivery; parent scope completion waits for descendant completion. Bounded three-node/two-generation model with fair cleanup. | | `DeferredReplies.tla`, `MCDeferredReplies*.tla/.cfg` | Rank 11 design model (not implemented), part 1: reply slots, send-time deadlines with shortening, deferral credits shared by a token and its first linked job, the exiting phase with exit-pending drops (unwound or dropped while exiting, answered at death with the final reason), a stuck phase (cleanup deadline passed while `stopped()` runs: nothing closed early, death later or never), death, and downstream request cleanup. Fourteen named negative controls. | -| `DeferredJobs.tla`, `MCDeferredJobs*.tla/.cfg` | Rank 11 design model, part 2: pipe jobs with separate execution, cancellation and notification states; keyed replacement including notified jobs; optional job deadlines; a FIFO mailbox with emit credits and the terminal message last; `ActorId` fencing. `MCDeferredJobsEmits` is the fast configuration with emits. Nine named negative controls. | +| `DeferredJobs.tla`, `MCDeferredJobs*.tla/.cfg` | Rank 11 design model, part 2: pipe jobs with separate execution, cancellation and notification states; keyed replacement including notified jobs; optional job deadlines; a FIFO mailbox with emit credits and the terminal message last; `ActorId` fencing. `MCDeferredJobsEmits` is the fast configuration with emits; `MCDeferredJobsHeld` the one with delayed emits (`emit_at`) and held results (completion, held notification, scheduled emits and credit ownership as separate steps). Eleven named negative controls. | | `Lifecycle.tla`, `MCLifecycle{,Live,Fence,Deep,LiveDeep,WideDeep}.tla/.cfg` | Base actor lifecycle as proposed for v0.8 ([design](../../docs/design/otp-exit-signals.md)): links, monitors, `trap_exit`, exit signals (`Killed`, ignored `Normal`, `NoProc`, parent exits), administrative stops, requests around death, `Exit`/`Down` ordering. Seventeen negative-control faults (17 safety checks, 2 liveness checks): the v0.7 exit-signal behaviours (direct stops ignoring `trap_exit`, kills ending `Kill`, unfenced `unlink`/`demonitor`, real reasons for late registrations, `stopped()` after kills and untrapped signals, late signals ignored during exit), the PR #182/#185 bugs, early exit errors, kill and panic precedence, runtime-drop reasons, early notification, a split death point and a parent exit reaching `exit_received`. Per PR: `MCLifecycle` (7.5M states, fewer reasons, where the safety controls run), `MCLifecycleFence`, `MCLifecycleLive`; weekly: `MCLifecycleWideDeep` (the former 43M-state per-PR instance), `MCLifecycleDeep`, `MCLifecycleLiveDeep`. | | `TraceCohorts.tla`, `TraceNested.tla` | Policy and ownership projections of real cohort/nested Rust executions. | | `trace_check.py`, `negative_checks.py` | Replay recorded traces; deliberately wrong selections, premature resume/join and broken propagation must be rejected or produce counterexamples. | @@ -615,6 +615,8 @@ exactly the property it targets: | `DeferredJobs` (`MCDeferredJobsEmits`) | `NoCancelFence` | `NoStaleAfterCancel` | | `DeferredJobs` (`MCDeferredJobsEmits`) | `EmitAfterTerminal` | `TerminalLast` | | `DeferredJobs` (`MCDeferredJobsEmits`) | `NoEmitCredit` | `EmitCreditsBounded` | +| `DeferredJobs` (`MCDeferredJobsHeld`) | `EarlyCredit` (a held result returns its credit when the work ends) | `HeldCounted` | +| `DeferredJobs` (`MCDeferredJobsHeld`) | `HeldDeadlineLost` (the deadline cannot take a held result) | `DeadlineArmedWhileHeld` | Measured 2026-11-10 (TLC 2.19, fede; the two `MCDeferredReplies` rows on 2026-11-11, after the exit-pending change): @@ -624,10 +626,11 @@ Measured 2026-11-10 (TLC 2.19, fede; the two `MCDeferredReplies` rows on | `MCDeferredReplies` (fast: 2 requests, `BoundA` 2, `BoundB` 1, clock 2; safety and liveness) | 4,035,345 | 21,287,403 | 33 | 2 min 19 s, 4 workers | | `MCDeferredRepliesDeep` (clock 3; safety) | 11,016,668 | 58,096,711 | 34 | 55 s, 4 workers | | `MCDeferredJobs` (fast: 3 jobs, 2 keyed, 2 permits, 2 instances, no emits; safety and liveness) | 229,070 | 487,283 | 17 | 5 s, 8 workers | -| `MCDeferredJobsEmits` (fast: 2 keyed jobs, 1 instance, 2 emits each, 1 credit; safety and liveness) | 20,422 | 37,220 | 18 | 1 s, 8 workers | -| `MCDeferredJobsDeep` (3 jobs, 2 instances, 1 emit each; safety) | 5,876,658 | 12,792,039 | 23 | 25 s, 12 workers | +| `MCDeferredJobsEmits` (fast: 2 keyed jobs, 1 instance, 2 emits each, 1 credit; safety and liveness) | 24,842 | 50,944 | 18 | 1 s, 2 workers | +| `MCDeferredJobsHeld` (fast: 3 unkeyed jobs, 2 permits, 1 instance, 1 emit or scheduled emit each, 1 credit; safety and liveness) | 1,946,047 | 6,230,609 | 22 | 2 min, 2 workers | +| `MCDeferredJobsDeep` (3 jobs, 2 instances, 1 emit or scheduled emit each; safety) | 7,980,942 | 19,676,218 | 23 | 82 s, 2 workers | -All 22 named controls find their counterexample, each in a few seconds. +All 24 named controls find their counterexample, each in a few seconds. ### Job trace validation diff --git a/verification/tla/TraceJobs.tla b/verification/tla/TraceJobs.tla index d0425061..1f15d136 100644 --- a/verification/tla/TraceJobs.tla +++ b/verification/tla/TraceJobs.tla @@ -21,10 +21,11 @@ (* Dispatch(j) the mailbox head, of kind x, fenced or delivered *) (* *) (* One trace is one owner instance (MaxGen = 1) and at most one key. The *) -(* model's permit bound is not checked (JobBound = J): the code releases *) -(* a job's credit just before Finish, and rejected admissions are not *) -(* model actions. The trace is accepted iff some behavior consumes the *) -(* whole log (TLC reports TraceIncomplete violated). *) +(* model's permit bound is not checked (JobBound = J): rejected *) +(* admissions are not model actions. Delayed emits (emit_at) are not *) +(* recorded; the recorded scenarios use none (MCDeferredJobsHeld checks *) +(* them). The trace is accepted iff some behavior consumes the whole log *) +(* (TLC reports TraceIncomplete violated). *) (***************************************************************************) EXTENDS DeferredJobs, TLC diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 07db8f11..d3c03278 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -249,6 +249,8 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCDeferredJobsEmits", "NoCancelFence", "NoStaleAfterCancel"), ("MCDeferredJobsEmits", "EmitAfterTerminal", "TerminalLast"), ("MCDeferredJobsEmits", "NoEmitCredit", "EmitCreditsBounded"), + ("MCDeferredJobsHeld", "EarlyCredit", "HeldCounted"), + ("MCDeferredJobsHeld", "HeldDeadlineLost", "DeadlineArmedWhileHeld"), ] From 39c18527cb25d4d29d7f69ddfe3e5d8d5f80d555 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:06:13 -0300 Subject: [PATCH 137/208] style: rustfmt the unused-permit test --- concurrency/tests/native_jobs.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/concurrency/tests/native_jobs.rs b/concurrency/tests/native_jobs.rs index ac0b120c..e2f0fa6b 100644 --- a/concurrency/tests/native_jobs.rs +++ b/concurrency/tests/native_jobs.rs @@ -265,7 +265,10 @@ async fn an_unused_permit_does_not_hold_the_scope(borrowed: bool) { // Reserved through a cloned context and kept outside the actor, unused. let permit: BlockingJobPermit = actor.context().reserve_blocking_job().unwrap(); actor.context().stop(); - assert_eq!(actor.wait_exit().await, spawned_concurrency::ExitReason::Normal); + assert_eq!( + actor.wait_exit().await, + spawned_concurrency::ExitReason::Normal + ); let report = system.shutdown(Duration::from_secs(2)).await; assert!( report.complete, From b5f419f324b6a4fa19f801dbfa039a3529bc8ca6 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:19:37 -0300 Subject: [PATCH 138/208] test: harden the death-boundary and job regression tests (round-two review) - failpoints: threads-mode versions of no_reply_succeeds_after_noproc_is_observable (Codex 4) and an_exit_signal_is_not_overtaken_by_its_senders_death_notice (Codex 10). The failpoint suites build without lifecycle tracing, so both races run without the trace's serialization. - deferred_api: a tasks-mode blocking-job variant of the held-result deadline test (Codex 8); the held-credit tests wait for the job's execution to end (`JobHandle::is_running`) instead of sleeping, so they check the held phase, not a timing guess. - (death_boundary's gates no longer release within an assertion window: in the runtime-drop commit.) Not added: a threads-mode wrapper test for job admission versus exit (Codex 6). Threads mode has no user code between its exit check and the admission, so the race needs an injected pause; the shared protocol is covered by Loom (`admission_vs_owner_exit`). --- concurrency/tests/deferred_api.rs | 46 +++++++++-- concurrency/tests/failpoints.rs | 125 ++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+), 6 deletions(-) diff --git a/concurrency/tests/deferred_api.rs b/concurrency/tests/deferred_api.rs index 650dcf37..cc542590 100644 --- a/concurrency/tests/deferred_api.rs +++ b/concurrency/tests/deferred_api.rs @@ -306,6 +306,34 @@ mod tasks_mode { actor.join().await; } + /// Review P1 (Codex 8), tasks-mode blocking job: the deadline overtakes + /// a result held for a scheduled emit. + #[cfg(not(spawned_sim))] // native jobs are not simulated + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_deadline_overtakes_a_blocking_jobs_held_result() { + let (actor, log) = start(); + in_actor(&actor, |_, ctx| { + let opts = JobOptions::default().deadline_after(Duration::from_millis(40)); + ctx.pipe_blocking_with( + opts, + |job| { + let at = Instant::now() + Duration::from_millis(400); + job.emit_at_blocking(at, Note("late".into())).unwrap(); + 7 + }, + |o| describe("job", o), + ) + .unwrap(); + }) + .await; + assert_eq!(wait_for(&log, 1).await, ["job:deadline"]); + // Past the emit's instant: it was dropped (the job is over). + spawned_rt::tasks::sleep(Duration::from_millis(500)).await; + assert_eq!(log.snapshot(), ["job:deadline"]); + actor.context().stop(); + actor.join().await; + } + /// Review P1 (agy 5): a result held for scheduled emits keeps the job's /// `MAX_OUTSTANDING` credit until its terminal message is enqueued. It /// used to return the credit when the work returned, so held jobs (and @@ -313,7 +341,7 @@ mod tasks_mode { #[spawned_macros::sim_test] async fn a_held_result_keeps_its_credit_until_the_terminal_message() { let (actor, log) = start(); - in_actor(&actor, |_, ctx| { + let job = in_actor(&actor, |_, ctx| { ctx.pipe_with_ctx( JobOptions::default(), |job| async move { @@ -323,10 +351,13 @@ mod tasks_mode { }, |o| describe("job", o), ) - .unwrap(); + .unwrap() }) .await; - spawned_rt::tasks::sleep(Duration::from_millis(50)).await; + // Handshake: the work has returned (its result is now held). + while job.is_running() { + spawned_rt::tasks::sleep(Duration::from_millis(1)).await; + } assert!(log.snapshot().is_empty()); let outstanding = in_actor(&actor, |_, ctx| ctx.deferred_outstanding()).await; assert_eq!(outstanding, 1, "a held result returned its credit early"); @@ -514,7 +545,7 @@ mod threads_mode { fn a_held_result_keeps_its_credit_until_the_terminal_message() { let log = Arc::new(Log::default()); let actor = Owner { log: log.clone() }.start(); - in_actor(&actor, |_, ctx| { + let job = in_actor(&actor, |_, ctx| { ctx.pipe_blocking_with( JobOptions::default(), |job| { @@ -523,9 +554,12 @@ mod threads_mode { }, |o| label("job", o), ) - .unwrap(); + .unwrap() }); - std::thread::sleep(Duration::from_millis(50)); + // Handshake: the work has returned (its result is now held). + while job.is_running() { + std::thread::sleep(Duration::from_millis(1)); + } assert!(log.snapshot().is_empty()); assert_eq!( in_actor(&actor, |_, ctx| ctx.deferred_outstanding()), diff --git a/concurrency/tests/failpoints.rs b/concurrency/tests/failpoints.rs index 9fd8b056..4e69bee2 100644 --- a/concurrency/tests/failpoints.rs +++ b/concurrency/tests/failpoints.rs @@ -1106,6 +1106,131 @@ mod native { panic!("injected native failure"); } } + /// Holds deferred tokens for an outside replier (threads mode). + struct NativeHolder; + impl NativeActor for NativeHolder {} + impl NativeHandler for NativeHolder { + fn handle( + &mut self, + msg: Hold, + reply: spawned_concurrency::reply::PendingReply<'_, u32>, + _: &NativeContext, + ) { + msg.0.send(reply.defer().unwrap()).unwrap(); + } + } + impl NativeHandler for NativeHolder { + fn handle( + &mut self, + _: Ping, + _: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &NativeContext, + ) { + } + } + + /// Review P1 (Codex 4), threads mode: no external reply succeeds once + /// `NoProc` is observable. + #[test] + fn no_reply_succeeds_after_noproc_is_observable() { + let _scenario = FailScenario::setup(); + let holder = NativeHolder.start(); + let (tokens, token) = std::sync::mpsc::channel(); + let caller = holder.request_raw(Hold(tokens)).unwrap(); + let token = token.recv_timeout(LIMIT).unwrap(); + failpoints::configure( + "actor.at_death", + FailPoint::new(Action::Delay(Duration::from_millis(300))).times(1), + ) + .unwrap(); + holder.context().stop(); + let began = Instant::now(); + while holder.send(Ping).is_ok() { + assert!(began.elapsed() < LIMIT, "the owner never died"); + std::thread::sleep(Duration::from_millis(1)); + } + assert_ne!( + token.reply(7), + spawned_concurrency::Delivery::Delivered, + "a reply succeeded after NoProc was observable" + ); + assert!( + matches!( + caller.recv(), + Err(spawned_concurrency::ActorError::ActorStopped( + ExitReason::Normal + )) + ), + "the caller did not get the final reason" + ); + holder.join(); + } + + /// Traps exits and records them (threads mode). + struct NativeRecorder(Arc>>); + impl NativeActor for NativeRecorder { + fn started(&mut self, ctx: &NativeContext) { + ctx.trap_exit(true); + } + fn exit_received(&mut self, exit: spawned_concurrency::Exit, _: &NativeContext) { + self.0.lock().unwrap().push(exit.reason); + } + } + struct NativeSender; + impl NativeActor for NativeSender {} + impl NativeHandler for NativeSender { + fn handle( + &mut self, + msg: LinkTo, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &NativeContext, + ) { + ctx.link(&msg.0); + reply.reply(()); + } + } + + /// Review P1 (Codex 10), threads mode: a cloned context's signal sent + /// before its actor's death reaches the target before the death notice. + #[test] + fn an_exit_signal_is_not_overtaken_by_its_senders_death_notice() { + let _scenario = FailScenario::setup(); + let seen = Arc::new(Mutex::new(Vec::new())); + let recorder = NativeRecorder(seen.clone()).start(); + let sender = NativeSender.start(); + sender.request(LinkTo(recorder.child_handle())).unwrap(); + failpoints::configure( + "actor.exit_before_deliver", + FailPoint::new(Action::Delay(Duration::from_millis(300))).times(1), + ) + .unwrap(); + let ctx = sender.context(); + let target = recorder.child_handle(); + let helper = std::thread::spawn(move || ctx.exit(&target, ExitReason::Shutdown)); + let began = Instant::now(); + while failpoints::stats("actor.exit_before_deliver").hits < 1 { + assert!(began.elapsed() < LIMIT); + std::thread::sleep(Duration::from_millis(1)); + } + sender.context().stop(); + while seen.lock().unwrap().len() < 2 { + assert!( + began.elapsed() < LIMIT, + "exits seen: {:?}", + seen.lock().unwrap() + ); + std::thread::sleep(Duration::from_millis(2)); + } + helper.join().unwrap(); + assert_eq!( + *seen.lock().unwrap(), + [ExitReason::Shutdown, ExitReason::Normal], + "the death notice overtook the sender's own signal" + ); + recorder.context().stop(); + recorder.join(); + } + struct Peer; impl NativeActor for Peer {} impl NativeHandler for Peer { From b945a71b89d4d462259d0c7c1c4186270bee2413 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:29:54 -0300 Subject: [PATCH 139/208] style(fuzz): rustfmt the ported harness handlers --- fuzz/src/world.rs | 197 +++++++++++++++++++++++----------------------- 1 file changed, 99 insertions(+), 98 deletions(-) diff --git a/fuzz/src/world.rs b/fuzz/src/world.rs index 7984c5b2..467cb66d 100644 --- a/fuzz/src/world.rs +++ b/fuzz/src/world.rs @@ -332,77 +332,78 @@ impl Handler for Worker { ctx: &Context, ) { let __value: u64 = async { - { - let mut s = self.world.lock(); - self.turn(&mut s, "Do"); - if msg.nonce != 0 { - if let Some(other) = s.handled.insert(msg.nonce, self.instance) { - let detail = format!("nonce {} by w{other} and w{}", msg.nonce, self.instance); - s.violate("D2: a message was handled twice", detail); - } - if msg.lane == 0 { - let inst = &mut s.insts[self.instance]; - let last = inst.last_lane0; - inst.last_lane0 = msg.nonce; - if msg.nonce <= last { - let detail = format!("w{}: {} after {last}", self.instance, msg.nonce); - s.violate("D1: per-sender FIFO broken", detail); + { + let mut s = self.world.lock(); + self.turn(&mut s, "Do"); + if msg.nonce != 0 { + if let Some(other) = s.handled.insert(msg.nonce, self.instance) { + let detail = + format!("nonce {} by w{other} and w{}", msg.nonce, self.instance); + s.violate("D2: a message was handled twice", detail); } - } - } - } - match msg.act { - Act::Noop => {} - Act::Panic => { - self.world.lock().insts[self.instance].panicked = true; - panic!("fuzz: injected handler panic"); - } - Act::Sleep(d) => rt::sleep(d).await, - Act::StopSelf(reason) => ctx.stop_with_reason(reason), - Act::Note(to) => { - let _ = to.send(Note { from: ctx.id() }); - } - Act::Link(peer) => { - self.uncertain(ctx.id(), Some(peer.id())); - ctx.link(&peer); - } - Act::Unlink(peer) => { - self.uncertain(ctx.id(), Some(peer.id())); - ctx.unlink(&peer); - } - Act::Monitor(target) => { - let mref = ctx.monitor(&target); - self.world.lock().monitors.push(MonRec { - watcher: ctx.id(), - target, - mref, - demonitored: false, - outside: false, - }); - } - Act::Trap(on) => { - self.uncertain(ctx.id(), None); - ctx.trap_exit(on); - } - Act::Register(name) => { - let _ = ctx.register_name(&name); - } - Act::Helper(v) => { - let _ = ctx.spawn_helper(async move { - match v % 3 { - 0 => std::future::pending::<()>().await, - 1 => rt::sleep(Duration::from_millis(10 * v as u64)).await, - _ => panic!("fuzz: injected helper panic"), + if msg.lane == 0 { + let inst = &mut s.insts[self.instance]; + let last = inst.last_lane0; + inst.last_lane0 = msg.nonce; + if msg.nonce <= last { + let detail = format!("w{}: {} after {last}", self.instance, msg.nonce); + s.violate("D1: per-sender FIFO broken", detail); + } } - }); + } } - Act::Stop(target, reason) => { - self.world.lock().may_stop.insert(target.id()); - target.stop_with_reason(reason); + match msg.act { + Act::Noop => {} + Act::Panic => { + self.world.lock().insts[self.instance].panicked = true; + panic!("fuzz: injected handler panic"); + } + Act::Sleep(d) => rt::sleep(d).await, + Act::StopSelf(reason) => ctx.stop_with_reason(reason), + Act::Note(to) => { + let _ = to.send(Note { from: ctx.id() }); + } + Act::Link(peer) => { + self.uncertain(ctx.id(), Some(peer.id())); + ctx.link(&peer); + } + Act::Unlink(peer) => { + self.uncertain(ctx.id(), Some(peer.id())); + ctx.unlink(&peer); + } + Act::Monitor(target) => { + let mref = ctx.monitor(&target); + self.world.lock().monitors.push(MonRec { + watcher: ctx.id(), + target, + mref, + demonitored: false, + outside: false, + }); + } + Act::Trap(on) => { + self.uncertain(ctx.id(), None); + ctx.trap_exit(on); + } + Act::Register(name) => { + let _ = ctx.register_name(&name); + } + Act::Helper(v) => { + let _ = ctx.spawn_helper(async move { + match v % 3 { + 0 => std::future::pending::<()>().await, + 1 => rt::sleep(Duration::from_millis(10 * v as u64)).await, + _ => panic!("fuzz: injected helper panic"), + } + }); + } + Act::Stop(target, reason) => { + self.world.lock().may_stop.insert(target.id()); + target.stop_with_reason(reason); + } } + msg.nonce } - msg.nonce - } .await; __reply.reply(__value); } @@ -428,34 +429,34 @@ impl Handler for Worker { ctx: &Context, ) { let __value: () = async { - let mut s = self.world.lock(); - self.turn(&mut s, "Down"); - let me = ctx.id(); - let target = s - .monitors - .iter() - .find(|m| m.mref == down.monitor_ref) - .map(|m| (m.watcher, m.target.id())); - match target { - Some((watcher, target)) if watcher == me => { - s.insts[self.instance].ended.insert(target); + let mut s = self.world.lock(); + self.turn(&mut s, "Down"); + let me = ctx.id(); + let target = s + .monitors + .iter() + .find(|m| m.mref == down.monitor_ref) + .map(|m| (m.watcher, m.target.id())); + match target { + Some((watcher, target)) if watcher == me => { + s.insts[self.instance].ended.insert(target); + } + Some(_) => s.violate( + "X1: Down delivered to an actor that did not create the monitor", + format!("w{}", self.instance), + ), + None => s.violate( + "X1: Down for an unknown monitor", + format!("w{}", self.instance), + ), } - Some(_) => s.violate( - "X1: Down delivered to an actor that did not create the monitor", - format!("w{}", self.instance), - ), - None => s.violate( - "X1: Down for an unknown monitor", - format!("w{}", self.instance), - ), - } - let inst = &mut s.insts[self.instance]; - let duplicate = inst.downs.iter().any(|(r, _)| *r == down.monitor_ref); - inst.downs.push((down.monitor_ref, down.reason)); - if duplicate { - s.violate("X1: Down delivered twice", format!("w{}", self.instance)); - } + let inst = &mut s.insts[self.instance]; + let duplicate = inst.downs.iter().any(|(r, _)| *r == down.monitor_ref); + inst.downs.push((down.monitor_ref, down.reason)); + if duplicate { + s.violate("X1: Down delivered twice", format!("w{}", self.instance)); } + } .await; __reply.reply(__value); } @@ -469,13 +470,13 @@ impl Handler for Worker { _ctx: &Context, ) { let __value: () = async { - let mut s = self.world.lock(); - self.turn(&mut s, "Note"); - if s.insts[self.instance].ended.contains(¬e.from) { - let detail = format!("from {} at w{}", s.name(note.from), self.instance); - s.violate("X7: a message arrived after the sender's Down/Exit", detail); - } + let mut s = self.world.lock(); + self.turn(&mut s, "Note"); + if s.insts[self.instance].ended.contains(¬e.from) { + let detail = format!("from {} at w{}", s.name(note.from), self.instance); + s.violate("X7: a message arrived after the sender's Down/Exit", detail); } + } .await; __reply.reply(__value); } From 93f00e36a240c2f7431986edf097bc11d0e113c7 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:36:33 -0300 Subject: [PATCH 140/208] test(fuzz): an exiting actor's link is inert in the X3 oracle The 60 s smoke run found "X3: a non-trapping linked actor survived its peer's abnormal exit": the linker had stopped itself with `Kill` and was in its slow `stopped()` when it called `link`. In 0.8, as in OTP, `link` from an actor whose exit began is inert (no record, no notice), so its peer correctly survives. The oracle treated only an already published exit as inert; it now also checks `Context::exit_pending()` at the link. The input is kept as regression `link_from_an_exiting_actor`. --- .../link_from_an_exiting_actor.input | Bin 0 -> 145 bytes .../link_from_an_exiting_actor.tape | 46 +++++++ fuzz/src/driver.rs | 12 +- fuzz/tests/link_from_an_exiting_actor.rs | 118 ++++++++++++++++++ 4 files changed, 172 insertions(+), 4 deletions(-) create mode 100644 fuzz/regressions/link_from_an_exiting_actor.input create mode 100644 fuzz/regressions/link_from_an_exiting_actor.tape create mode 100644 fuzz/tests/link_from_an_exiting_actor.rs diff --git a/fuzz/regressions/link_from_an_exiting_actor.input b/fuzz/regressions/link_from_an_exiting_actor.input new file mode 100644 index 0000000000000000000000000000000000000000..5febc85c7869b3b55fa76a236d42d16d2afb1a5f GIT binary patch literal 145 zcmXrkRz?S!AOY@G5CE2B29wIlP%$9IMF4 --emit link_from_an_exiting_actor`. +//! +//! Failure: none (passing input) +//! Strategy random, seed 2531906049332685619. +//! +//! ```text +//! Scenario { +//! seed: 2531906049332685619, +//! sched: Random, +//! ops: [ +//! Link { +//! a: 35, +//! b: 35, +//! }, +//! Link { +//! a: 35, +//! b: 35, +//! }, +//! CrashChild { +//! sup: 170, +//! child: 170, +//! how: Panic, +//! }, +//! Spawn { +//! how: Unmanaged, +//! hooks: Hooks { +//! flags: 35, +//! started_sleep: 35, +//! stopped_sleep: 35, +//! }, +//! }, +//! CrashChild { +//! sup: 170, +//! child: 170, +//! how: Stop( +//! Normal, +//! ), +//! }, +//! Spawn { +//! how: Unmanaged, +//! hooks: Hooks { +//! flags: 10, +//! started_sleep: 10, +//! stopped_sleep: 10, +//! }, +//! }, +//! Spawn { +//! how: Unmanaged, +//! hooks: Hooks { +//! flags: 10, +//! started_sleep: 10, +//! stopped_sleep: 10, +//! }, +//! }, +//! Spawn { +//! how: Managed { +//! group: 170, +//! }, +//! hooks: Hooks { +//! flags: 170, +//! started_sleep: 95, +//! stopped_sleep: 95, +//! }, +//! }, +//! Stop { +//! who: 95, +//! reason: Kill, +//! }, +//! CrashChild { +//! sup: 170, +//! child: 170, +//! how: Stop( +//! Kill, +//! ), +//! }, +//! CrashChild { +//! sup: 170, +//! child: 170, +//! how: Stop( +//! Shutdown, +//! ), +//! }, +//! Link { +//! a: 35, +//! b: 154, +//! }, +//! TrapExit { +//! who: 1, +//! on: false, +//! }, +//! Spawn { +//! how: Unmanaged, +//! hooks: Hooks { +//! flags: 0, +//! started_sleep: 0, +//! stopped_sleep: 0, +//! }, +//! }, +//! ], +//! } +//! ``` +#![cfg(spawned_sim)] + +const INPUT: &[u8] = include_bytes!("../regressions/link_from_an_exiting_actor.input"); +const TAPE: &str = include_str!("../regressions/link_from_an_exiting_actor.tape"); + +/// The input under its own schedule and under its recorded tape. +#[test] +fn link_from_an_exiting_actor_tape() { + spawned_fuzz::assert_passes(INPUT, Some(TAPE)); +} + +/// The same operations under other simulator schedules. +#[spawned_macros::sim_test(seeds = 32)] +async fn link_from_an_exiting_actor_seeds(seed: u64) { + spawned_fuzz::init(); + spawned_fuzz::run_body(spawned_fuzz::decode(INPUT).unwrap(), seed).await; +} From 9684e012391a5e2defc5382b993cc8b8114d24d0 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:37:18 -0300 Subject: [PATCH 141/208] test(supervision): the sibling signalled after the system deadline ends Shutdown or Killed (single-cpu race) --- concurrency/tests/supervision.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/concurrency/tests/supervision.rs b/concurrency/tests/supervision.rs index 9c34033a..7ee71153 100644 --- a/concurrency/tests/supervision.rs +++ b/concurrency/tests/supervision.rs @@ -1338,12 +1338,15 @@ async fn system_deadline_signals_remaining_supervised_siblings() { .await .unwrap() .unwrap(); - assert_eq!( - spawned_rt::tasks::timeout(Duration::from_secs(1), actor.wait_exit()) - .await - .expect("earlier sibling was not stopped after the system deadline"), - ExitReason::Killed, - "signalled at the system's grace expiry: the kill (E3)" + let reason = spawned_rt::tasks::timeout(Duration::from_secs(1), actor.wait_exit()) + .await + .expect("earlier sibling was not stopped after the system deadline"); + // Signalled after the system deadline: by the supervisor's `Shutdown` + // or by the grace-expiry kill (E3), whichever reaches it first (on one + // CPU the shutdown can). + assert!( + matches!(reason, ExitReason::Killed | ExitReason::Shutdown), + "{reason:?}" ); release.send(()).unwrap(); assert!(report.wait(LIMIT).await.complete); From c27f37a9ae30785f66588a8ad07cc71731f715a3 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:41:59 -0300 Subject: [PATCH 142/208] test(fuzz): a late link's Exit carries NoProc in the X3 oracle (0.8) --- fuzz/regressions/late_link_noproc_exit.input | Bin 0 -> 153 bytes fuzz/regressions/late_link_noproc_exit.tape | 137 +++++++++++++++++++ fuzz/src/driver.rs | 4 +- fuzz/tests/late_link_noproc_exit.rs | 130 ++++++++++++++++++ 4 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 fuzz/regressions/late_link_noproc_exit.input create mode 100644 fuzz/regressions/late_link_noproc_exit.tape create mode 100644 fuzz/tests/late_link_noproc_exit.rs diff --git a/fuzz/regressions/late_link_noproc_exit.input b/fuzz/regressions/late_link_noproc_exit.input new file mode 100644 index 0000000000000000000000000000000000000000..bab1ed3a190a3526a336a299857ef509b09c5cb4 GIT binary patch literal 153 zcmZ3$wIH%;&b;*h|Nk?9!Ab@Oz90q$g50bUl1A3wt$NfD19gL|Nr{` zKmrLsv@tMzms4h7_~`&tc0lfV=W~Y6v?|?VBi(d^xi=Osa$2<5Ndjap14HHv1TaD} S2gIB)gMs0eJcE44|5*U{mq`@> literal 0 HcmV?d00001 diff --git a/fuzz/regressions/late_link_noproc_exit.tape b/fuzz/regressions/late_link_noproc_exit.tape new file mode 100644 index 00000000..24dc5535 --- /dev/null +++ b/fuzz/regressions/late_link_noproc_exit.tape @@ -0,0 +1,137 @@ +# spawned-sim tape v2 +seed 7466577281530464928 +spin 10000 +c 0 +c 1 +c 2 +c 0 +c 3 +c 0 +t 1000000 +c 0 +c 3 +c 0 +t 2000000 +c 0 +c 3 +c 0 +c 3 +c 0 +t 3000000 +c 0 +c 3 +c 0 +t 4000000 +c 0 +c 3 +c 0 +c 3 +c 0 +t 5000000 +c 0 +c 3 +c 0 +t 6000000 +c 0 +c 3 +c 0 +c 3 +c 0 +t 7000000 +c 0 +c 3 +c 0 +t 8000000 +c 0 +c 3 +c 0 +p 5 +p 3 +p 4 +c 0 +c 6 +p 7 +c 0 +c 3 +c 0 +t 9000000 +c 0 +c 3 +c 0 +t 10000000 +c 0 +c 3 +c 0 +c 3 +c 0 +c 3 +c 0 +t 11000000 +c 0 +c 3 +c 0 +t 12000000 +c 0 +c 3 +c 0 +c 3 +c 0 +c 3 +c 0 +t 13000000 +c 0 +c 3 +c 0 +t 14000000 +c 0 +c 3 +c 0 +c 3 +c 0 +c 3 +c 0 +t 15000000 +c 0 +c 3 +c 0 +t 16000000 +c 0 +c 3 +c 0 +c 3 +c 0 +c 3 +c 0 +c 8 +c 0 +c 9 +c 0 +t 17000000 +c 0 +c 3 +c 0 +c 9 +c 0 +t 18000000 +c 0 +c 3 +c 0 +c 9 +c 0 +c 3 +c 0 +c 5 +c 0 +c 6 +c 7 +c 6 +c 0 +c 10 +c 3 +t 2768000000 +c 3 +c 10 +c 1 +c 0 +c 9 +c 0 diff --git a/fuzz/src/driver.rs b/fuzz/src/driver.rs index 3b02a518..1a0bb80f 100644 --- a/fuzz/src/driver.rs +++ b/fuzz/src/driver.rs @@ -1628,7 +1628,9 @@ impl Driver { .unwrap_or_default() }; for r in exits(self) { - if r != published { + // 0.8 (X3): a link made after the peer's death point is a + // late link, whose notice is `NoProc`. + if r != published && r != ExitReason::NoProc { self.violate( "X3: Exit carried a different reason than the peer published", format!("{r:?} vs {published:?}"), diff --git a/fuzz/tests/late_link_noproc_exit.rs b/fuzz/tests/late_link_noproc_exit.rs new file mode 100644 index 00000000..8e5a75ed --- /dev/null +++ b/fuzz/tests/late_link_noproc_exit.rs @@ -0,0 +1,130 @@ +//! Generated by `cargo run --example repro -- --emit late_link_noproc_exit`. +//! +//! Failure: none (passing input) +//! Strategy random, seed 7466577281530464928. +//! +//! ```text +//! Scenario { +//! seed: 7466577281530464928, +//! sched: Random, +//! ops: [ +//! Spawn { +//! how: Unmanaged, +//! hooks: Hooks { +//! flags: 14, +//! started_sleep: 82, +//! stopped_sleep: 0, +//! }, +//! }, +//! Spawn { +//! how: Managed { +//! group: 38, +//! }, +//! hooks: Hooks { +//! flags: 126, +//! started_sleep: 31, +//! stopped_sleep: 255, +//! }, +//! }, +//! Stop { +//! who: 6, +//! reason: Normal, +//! }, +//! TrapExit { +//! who: 255, +//! on: true, +//! }, +//! Fence, +//! Fence, +//! Fence, +//! Fence, +//! Spawn { +//! how: Unmanaged, +//! hooks: Hooks { +//! flags: 247, +//! started_sleep: 30, +//! stopped_sleep: 35, +//! }, +//! }, +//! Monitor { +//! watcher: 1, +//! target: 0, +//! }, +//! DropReply { +//! to: 137, +//! cmd: Note( +//! 122, +//! ), +//! }, +//! Link { +//! a: 103, +//! b: 48, +//! }, +//! Supervisor( +//! SupSpec { +//! system: 66, +//! policy: Policy { +//! strategy: 162, +//! max_restarts: 163, +//! window: 6210, +//! backoff: 14, +//! jitter: false, +//! stable: 0, +//! }, +//! children: [], +//! }, +//! ), +//! Supervisor( +//! SupSpec { +//! system: 152, +//! policy: Policy { +//! strategy: 152, +//! max_restarts: 152, +//! window: 39064, +//! backoff: 152, +//! jitter: false, +//! stable: 152, +//! }, +//! children: [], +//! }, +//! ), +//! Fence, +//! Fence, +//! Fence, +//! Fence, +//! Spawn { +//! how: Unmanaged, +//! hooks: Hooks { +//! flags: 31, +//! started_sleep: 136, +//! stopped_sleep: 255, +//! }, +//! }, +//! Spawn { +//! how: Unmanaged, +//! hooks: Hooks { +//! flags: 0, +//! started_sleep: 0, +//! stopped_sleep: 0, +//! }, +//! }, +//! ], +//! } +//! ``` +#![cfg(spawned_sim)] + +const INPUT: &[u8] = include_bytes!("../regressions/late_link_noproc_exit.input"); +const TAPE: &str = include_str!("../regressions/late_link_noproc_exit.tape"); + +/// The input under its own schedule and under its recorded tape. +#[test] +fn late_link_noproc_exit_tape() { + spawned_fuzz::assert_passes(INPUT, Some(TAPE)); +} + +/// The same operations under other simulator schedules. +#[spawned_macros::sim_test(seeds = 32)] +async fn late_link_noproc_exit_seeds(seed: u64) { + spawned_fuzz::init(); + spawned_fuzz::run_body(spawned_fuzz::decode(INPUT).unwrap(), seed).await; +} From 1f321b9a4283425d10eabfac29ce6380958aeb63 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:45:52 -0300 Subject: [PATCH 143/208] test(fuzz): a late Shutdown notice may replace a panic reason in the L4 oracle A 5000-input sweep found "L4: an actor that panicked exited with a non-abnormal reason": w0 panicked in a handler (reason Panic), and while its slow stopped() ran, its linked peer died Shutdown. The untrapped link notice arrived before the death point and replaced the reason, as the contract says (semantics.md L4, Lifecycle!LateSignalWins) and as in OTP. The oracle now allows exactly that case (Shutdown on a non-trapping actor). Kept as regression panic_then_late_shutdown_notice. --- .../panic_then_late_shutdown_notice.input | Bin 0 -> 659 bytes fuzz/src/driver.rs | 9 +- fuzz/tests/panic_then_late_shutdown_notice.rs | 425 ++++++++++++++++++ 3 files changed, 433 insertions(+), 1 deletion(-) create mode 100644 fuzz/regressions/panic_then_late_shutdown_notice.input create mode 100644 fuzz/tests/panic_then_late_shutdown_notice.rs diff --git a/fuzz/regressions/panic_then_late_shutdown_notice.input b/fuzz/regressions/panic_then_late_shutdown_notice.input new file mode 100644 index 0000000000000000000000000000000000000000..5a8357ea4ac95209b67cc345fb4971db5a0bd972 GIT binary patch literal 659 zcmV;E0&M-8GOAD0m>J1zl1lSrxMeTnf@f_qyc< zESS|!xD(ZOFzA!wRgL(s=5JXiBMI3YppUMDYM$@FrD9CJ$=ccES5X;T*;IyEoe~tJ ziwgARvP;kS(exXiv}xC36g&Xvx#$N8TV?+wrkT=FnF{R38ctT$Pl2j=rQ)e}2FQ=# zn?_^`=)rt$Mk;WYA5H&mIvmzh8-Fdc{2}?8(W>#l?cxK&zS{)yGv(kN6*uiJ+8=#M z4S%J4gn=O3#~%8_2FfR`Dhj}tZnnYB3lZB2G9bLAa3m1affoLMKLJhY#>vD58(*L9 zJsmeY1#fRh(LRtpt*Su>zofa+d!sBezOj13u@(|w7SPL}Uu%IrUSxXE!#w2WKqh3o zK}q!*Y|J%X6}6bzB79_k;Z!o#v!?Z?!*cjN<0wW#8j^pRAX<`zbVy9H?J|<3I@pz- z|L~#A%_RD#WI8N%7Ua1U{Mp)vfMcUXE;KcnZ!r28vZju zTd#~M1@$_f7lk}FDxO!tiko%1@qhL;yT^|hzKSxpJu;q@$h~JL6;^oRiNHzcf literal 0 HcmV?d00001 diff --git a/fuzz/src/driver.rs b/fuzz/src/driver.rs index 1a0bb80f..5c4286c4 100644 --- a/fuzz/src/driver.rs +++ b/fuzz/src/driver.rs @@ -1686,7 +1686,14 @@ impl Driver { format!("w{instance} {exit:?}"), ); } - if panicked && !exit.is_abnormal() { + // A panic sets the reason, but until the death point an + // untrapped exit signal or link notice still replaces it + // (semantics.md L4, `Lifecycle!LateSignalWins`; OTP: a signal + // during `terminate` kills with its own reason). The only + // non-abnormal reason a signal can carry is `Shutdown`. + let trapping = self.traps.get(&id).is_some_and(|t| t.0); + let late_shutdown = exit == ExitReason::Shutdown && !trapping; + if panicked && !exit.is_abnormal() && !late_shutdown { self.violate( "L4: an actor that panicked exited with a non-abnormal reason", format!("w{instance} {exit:?}"), diff --git a/fuzz/tests/panic_then_late_shutdown_notice.rs b/fuzz/tests/panic_then_late_shutdown_notice.rs new file mode 100644 index 00000000..83add4e9 --- /dev/null +++ b/fuzz/tests/panic_then_late_shutdown_notice.rs @@ -0,0 +1,425 @@ +//! Generated by `cargo run --example repro -- --emit panic_then_late_shutdown_notice`. +//! +//! Failure: none (passing input) +//! Strategy dfs:bound=0, seed 14490781309383357083. +//! +//! ```text +//! Scenario { +//! seed: 14490781309383357083, +//! sched: Canonical, +//! ops: [ +//! CrashChild { +//! sup: 204, +//! child: 45, +//! how: Panic, +//! }, +//! Advance { +//! ms: 1002, +//! }, +//! CrashChild { +//! sup: 51, +//! child: 131, +//! how: Stop( +//! Kill, +//! ), +//! }, +//! StopSupervisor { +//! sup: 170, +//! mode: 111, +//! ms: 7477, +//! }, +//! DropReply { +//! to: 136, +//! cmd: Sleep( +//! 36, +//! ), +//! }, +//! CrashChild { +//! sup: 234, +//! child: 109, +//! how: Stop( +//! Normal, +//! ), +//! }, +//! Supervisor( +//! SupSpec { +//! system: 148, +//! policy: Policy { +//! strategy: 181, +//! max_restarts: 0, +//! window: 3540, +//! backoff: 239, +//! jitter: false, +//! stable: 89, +//! }, +//! children: [ +//! Worker( +//! WorkerDef { +//! restart: 210, +//! group: 69, +//! fail_mask: 247, +//! panic_mask: 186, +//! hook_mask: 229, +//! hooks: Hooks { +//! flags: 6, +//! started_sleep: 44, +//! stopped_sleep: 152, +//! }, +//! startup_timeout: 213, +//! shutdown_timeout: 78, +//! }, +//! ), +//! ], +//! }, +//! ), +//! Unlink { +//! a: 232, +//! b: 147, +//! }, +//! Fence, +//! TrapExit { +//! who: 39, +//! on: true, +//! }, +//! Supervisor( +//! SupSpec { +//! system: 143, +//! policy: Policy { +//! strategy: 174, +//! max_restarts: 131, +//! window: 40554, +//! backoff: 239, +//! jitter: false, +//! stable: 165, +//! }, +//! children: [], +//! }, +//! ), +//! DropSystem { +//! sys: 217, +//! }, +//! Request { +//! to: 91, +//! cmd: Note( +//! 157, +//! ), +//! timeout: None, +//! detached: false, +//! }, +//! WaitGeneration { +//! sup: 229, +//! child: 178, +//! ms: 53067, +//! }, +//! Request { +//! to: 159, +//! cmd: Note( +//! 20, +//! ), +//! timeout: None, +//! detached: false, +//! }, +//! Spawn { +//! how: Linked { +//! parent: 36, +//! }, +//! hooks: Hooks { +//! flags: 166, +//! started_sleep: 153, +//! stopped_sleep: 210, +//! }, +//! }, +//! WaitGeneration { +//! sup: 199, +//! child: 26, +//! ms: 22094, +//! }, +//! CrashChild { +//! sup: 121, +//! child: 165, +//! how: Panic, +//! }, +//! Supervisor( +//! SupSpec { +//! system: 70, +//! policy: Policy { +//! strategy: 100, +//! max_restarts: 10, +//! window: 49640, +//! backoff: 124, +//! jitter: false, +//! stable: 70, +//! }, +//! children: [], +//! }, +//! ), +//! Demonitor { +//! which: 255, +//! }, +//! DropSystem { +//! sys: 83, +//! }, +//! StopSupervisor { +//! sup: 252, +//! mode: 33, +//! ms: 39673, +//! }, +//! NewSystem, +//! NewSystem, +//! WaitGeneration { +//! sup: 51, +//! child: 229, +//! ms: 7648, +//! }, +//! Unlink { +//! a: 218, +//! b: 31, +//! }, +//! CancelTimer { +//! which: 165, +//! }, +//! Request { +//! to: 220, +//! cmd: Register( +//! 6, +//! ), +//! timeout: None, +//! detached: true, +//! }, +//! NewSystem, +//! NewSystem, +//! DropSystem { +//! sys: 10, +//! }, +//! CrashChild { +//! sup: 112, +//! child: 36, +//! how: Panic, +//! }, +//! Spawn { +//! how: Linked { +//! parent: 196, +//! }, +//! hooks: Hooks { +//! flags: 5, +//! started_sleep: 27, +//! stopped_sleep: 95, +//! }, +//! }, +//! Request { +//! to: 55, +//! cmd: Link( +//! 71, +//! ), +//! timeout: Some( +//! 36926, +//! ), +//! detached: true, +//! }, +//! Spawn { +//! how: Linked { +//! parent: 123, +//! }, +//! hooks: Hooks { +//! flags: 163, +//! started_sleep: 44, +//! stopped_sleep: 50, +//! }, +//! }, +//! NewSystem, +//! Stop { +//! who: 22, +//! reason: Shutdown, +//! }, +//! Stop { +//! who: 100, +//! reason: Normal, +//! }, +//! Link { +//! a: 100, +//! b: 188, +//! }, +//! Request { +//! to: 108, +//! cmd: Panic, +//! timeout: Some( +//! 55704, +//! ), +//! detached: false, +//! }, +//! DropReply { +//! to: 84, +//! cmd: Monitor( +//! 245, +//! ), +//! }, +//! Fence, +//! Demonitor { +//! which: 66, +//! }, +//! Supervisor( +//! SupSpec { +//! system: 32, +//! policy: Policy { +//! strategy: 90, +//! max_restarts: 146, +//! window: 29829, +//! backoff: 72, +//! jitter: false, +//! stable: 178, +//! }, +//! children: [ +//! Worker( +//! WorkerDef { +//! restart: 216, +//! group: 149, +//! fail_mask: 158, +//! panic_mask: 255, +//! hook_mask: 240, +//! hooks: Hooks { +//! flags: 161, +//! started_sleep: 204, +//! stopped_sleep: 205, +//! }, +//! startup_timeout: 36, +//! shutdown_timeout: 250, +//! }, +//! ), +//! Worker( +//! WorkerDef { +//! restart: 22, +//! group: 228, +//! fail_mask: 185, +//! panic_mask: 20, +//! hook_mask: 252, +//! hooks: Hooks { +//! flags: 217, +//! started_sleep: 218, +//! stopped_sleep: 135, +//! }, +//! startup_timeout: 128, +//! shutdown_timeout: 99, +//! }, +//! ), +//! Worker( +//! WorkerDef { +//! restart: 104, +//! group: 220, +//! fail_mask: 233, +//! panic_mask: 162, +//! hook_mask: 41, +//! hooks: Hooks { +//! flags: 173, +//! started_sleep: 24, +//! stopped_sleep: 184, +//! }, +//! startup_timeout: 163, +//! shutdown_timeout: 34, +//! }, +//! ), +//! ], +//! }, +//! ), +//! Yield { +//! n: 186, +//! }, +//! Demonitor { +//! which: 177, +//! }, +//! Monitor { +//! watcher: 91, +//! target: 175, +//! }, +//! Fence, +//! Advance { +//! ms: 13884, +//! }, +//! NewSystem, +//! CancelTimer { +//! which: 86, +//! }, +//! NewSystem, +//! Yield { +//! n: 126, +//! }, +//! StopSupervisor { +//! sup: 148, +//! mode: 79, +//! ms: 29051, +//! }, +//! Send { +//! to: 76, +//! cmd: Noop, +//! via_recipient: false, +//! }, +//! Fence, +//! Advance { +//! ms: 33906, +//! }, +//! Monitor { +//! watcher: 216, +//! target: 27, +//! }, +//! StopSupervisor { +//! sup: 190, +//! mode: 135, +//! ms: 44670, +//! }, +//! Yield { +//! n: 50, +//! }, +//! Request { +//! to: 89, +//! cmd: Stop( +//! 227, +//! Shutdown, +//! ), +//! timeout: Some( +//! 48379, +//! ), +//! detached: true, +//! }, +//! Request { +//! to: 240, +//! cmd: StopSelf( +//! Panic, +//! ), +//! timeout: Some( +//! 29471, +//! ), +//! detached: true, +//! }, +//! Spawn { +//! how: Managed { +//! group: 80, +//! }, +//! hooks: Hooks { +//! flags: 224, +//! started_sleep: 141, +//! stopped_sleep: 52, +//! }, +//! }, +//! CrashChild { +//! sup: 81, +//! child: 239, +//! how: StopNested, +//! }, +//! ], +//! } +//! ``` +#![cfg(spawned_sim)] + +const INPUT: &[u8] = include_bytes!("../regressions/panic_then_late_shutdown_notice.input"); + +/// The input under its own schedule and under its recorded tape. +#[test] +fn panic_then_late_shutdown_notice_tape() { + spawned_fuzz::assert_passes(INPUT, None); +} + +/// The same operations under other simulator schedules. +#[spawned_macros::sim_test(seeds = 32)] +async fn panic_then_late_shutdown_notice_seeds(seed: u64) { + spawned_fuzz::init(); + spawned_fuzz::run_body(spawned_fuzz::decode(INPUT).unwrap(), seed).await; +} From df0e8da633e30e34d9e6138676533a00c7d429fa Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 13:45:19 -0300 Subject: [PATCH 144/208] refactor!: one job call per shape, options first (0.8 API pass) pipe, pipe_blocking, pipe_reply and pipe_reply_blocking take JobOptions first and hand every work closure its JobCtx by value; pipe_with, pipe_with_ctx, pipe_keyed and the *_with forms are gone. reserve_job and reserve_blocking_job take the options; their permits have one method. JobOptions is non_exhaustive with a builder (key, deadline taking impl Into, emit_credits taking NonZeroU16, cancel_on_caller_gone, debug-asserted outside pipe_reply*). JobOutcome is Result; PipeError is the non_exhaustive enum and PipeReplyError is non_exhaustive. Deadline gains From and From. DEFAULT_EMIT_CREDITS and default_max_native_jobs are crate-private. GUIDE, MIGRATION (new 0.7 to 0.8 call-site table), CHANGELOG, design docs, example and a trybuild pass case follow. --- CHANGELOG.md | 31 +- concurrency/src/reply/jobs.rs | 129 ++++---- concurrency/src/reply/mod.rs | 15 + concurrency/src/tasks/actor.rs | 2 +- concurrency/src/tasks/jobs.rs | 309 +++++++----------- concurrency/src/tasks/mod.rs | 3 +- concurrency/src/threads/actor.rs | 2 +- concurrency/src/threads/jobs.rs | 125 ++++--- concurrency/src/threads/mod.rs | 3 +- concurrency/tests/deadline_service.rs | 10 +- concurrency/tests/death_boundary.rs | 8 +- concurrency/tests/deferred_api.rs | 162 +++++---- concurrency/tests/deferred_jobs.rs | 125 ++++--- concurrency/tests/deferred_sim.rs | 17 +- concurrency/tests/job_trace.rs | 42 +-- concurrency/tests/metrics_deferred.rs | 36 +- concurrency/tests/native_jobs.rs | 40 +-- concurrency/tests/ui-pass/job_api.rs | 71 ++++ docs/GUIDE.md | 48 +-- docs/MIGRATION.md | 57 +++- .../design/deferred-replies-client-mapping.md | 21 +- docs/design/deferred-replies.md | 61 ++-- .../capabilities/src/bin/deferred_replies.rs | 19 +- verification/tla/DeferredJobs.tla | 2 +- 24 files changed, 734 insertions(+), 604 deletions(-) create mode 100644 concurrency/tests/ui-pass/job_api.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 28e38fde..badd4ef3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,22 @@ ## Unreleased +- **Breaking (0.8), API pass before publishing** (each change is a row of the + [0.7 to 0.8 call-site table](docs/MIGRATION.md#07-to-08-api-call-site-table)): + - Jobs: one call per shape, options first (`pipe`, `pipe_blocking`, + `pipe_reply`, `pipe_reply_blocking`, `reserve_job(opts)`, + `reserve_blocking_job(opts)`); every `work` receives its `JobCtx` by + value. `pipe_with`, `pipe_with_ctx`, `pipe_keyed`, `pipe_blocking_with`, + `pipe_reply_with` and `pipe_reply_blocking_with` are gone. + `JobOptions` is `#[non_exhaustive]` with a builder (`new`, `key`, + `deadline(impl Into)`, `emit_credits(NonZeroU16)`, + `cancel_on_caller_gone`, which `debug_assert!`s outside `pipe_reply*`); + `deadline_after` is folded into `deadline`. `JobOutcome` is + `Result`. `PipeError` is the `#[non_exhaustive]` enum (no + `kind` wrapper); `PipeReplyError` is `#[non_exhaustive]` and its + `Failed` variant carries `error`. `Deadline` implements + `From` and `From`. `DEFAULT_EMIT_CREDITS` and + `default_max_native_jobs` are no longer public. - Deferred replies and jobs, fixes from the 0.8 pre-release review: a job deadline still fires while the result waits for scheduled emits, and the job keeps its `MAX_OUTSTANDING` credit until its terminal message; a job @@ -18,8 +34,7 @@ stops the process-wide deadline service; a tasks handler that panics while building its future is a handler panic (`stopped()` runs). - **Breaking (0.8):** `DeferError` and `Delivery` are `#[non_exhaustive]` - (like `JobOutcome`, `JobError` and `PipeErrorKind`): match them with a - wildcard arm. + (like `JobError` and `PipeError`): match them with a wildcard arm. - **Breaking (0.8):** the restart budget counts restart actions, as OTP's intensity does (ROADMAP rank 5f): a OneForAll or RestForOne recovery round takes one `max_restarts` grant when the coordinator selects it, before the @@ -163,9 +178,8 @@ deadlines are fixed at send (`Deadline`, `request_raw_until`; `request_raw` gains the 5 s default); a per-actor deadline heap and service expire stored tokens without a polling caller. `PendingReplies` is a bounded token - table. Pipe-to-self jobs: `ctx.pipe`, `pipe_keyed`, `pipe_with`, - `pipe_blocking_with`, `pipe_reply`, `pipe_reply_with`, - `pipe_reply_blocking_with` (threads mode: the blocking forms on owned native + table. Pipe-to-self jobs: `ctx.pipe`, `pipe_blocking`, `pipe_reply`, + `pipe_reply_blocking` (threads mode: the blocking forms on owned native threads), with `JobOutcome`/`JobError::{Panicked, DeadlineExceeded}`, keyed replacement and cancellation fenced at dispatch, `JobCtx::emit`/ `emit_blocking` with emit credits and the terminal message last, @@ -182,19 +196,18 @@ thread each, still under `MAX_NATIVE_JOBS`, with panics as `JobError::Panicked` and cancellation and fencing unchanged (threads mode keeps owned threads; a reservation pre-spawns the job's thread). - Reserve-then-spawn: `ctx.reserve_job()` / `ctx.reserve_blocking_job()` + Reserve-then-spawn: `ctx.reserve_job(opts)` / `ctx.reserve_blocking_job(opts)` return a `JobPermit` / `BlockingJobPermit` whose `pipe*` cannot fail, so a refused caller keeps what it would have moved into `work` and `done`. `JobCtx::emit_at` / `emit_at_blocking` deliver a fenced, credit-accounted message at an instant; the job's terminal message waits for its scheduled emits (new Loom model `scheduled_emit_vs_finish_and_cancel`). - `JobOptions::deadline_after(Duration)`; `spawned_concurrency::Instant` + `spawned_concurrency::Instant` (also `tasks::`/`threads::Instant`) re-exports the deadline clock. `#[send_handler]` on a manual `impl Handler` (for `Result = ()`) drops the reply parameter and answers `()` when `handle` returns. `spawned_test::tasks::in_actor` / `threads::in_actor` run a closure in a - live actor's turn. `JobOutcome`, `JobError` and `PipeErrorKind` are - `#[non_exhaustive]`. GUIDE: hard job deadlines and the soft-deadline + live actor's turn. GUIDE: hard job deadlines and the soft-deadline pattern, replies carrying a reason through `R = Result<..>`, the `request_raw` rows. - ROADMAP: a Path to 1.0 section (0.8, 0.9, 0.10 contents and what 1.0 diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index a2e1dfc9..bbf883b5 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -24,10 +24,10 @@ use crate::ActorId; use std::sync::{Arc, Condvar, Mutex, MutexGuard}; /// Default emit credits per job (`JobOptions::emit_credits`). -pub const DEFAULT_EMIT_CREDITS: u16 = 16; +pub(crate) const DEFAULT_EMIT_CREDITS: u16 = 16; /// Default `Actor::MAX_NATIVE_JOBS`: `min(4, available_parallelism)`. -pub fn default_max_native_jobs() -> usize { +pub(crate) fn default_max_native_jobs() -> usize { std::thread::available_parallelism() .map(|n| n.get()) .unwrap_or(1) @@ -58,7 +58,7 @@ impl Bounds { } } -/// A key for cancel-previous jobs (`pipe_keyed`, `JobOptions::key`). +/// A key for cancel-previous jobs (`JobOptions::key`). #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct JobKey(std::borrow::Cow<'static, str>); @@ -76,35 +76,67 @@ impl From<&'static str> for JobKey { } } -/// Options for `pipe_with`, `pipe_blocking_with`, `pipe_reply_with` and -/// `pipe_reply_blocking_with`. Passed first; never chained afterwards. +/// Options of a job, passed first to every `pipe*` call and to +/// `reserve_*_job` (as OTP's `*_opt` functions take an option list). +/// Non-exhaustive: build it with the methods, starting from +/// [`JobOptions::new`] (or `default()`). +/// +/// ```ignore +/// ctx.pipe(JobOptions::new().key("fetch").deadline(Duration::from_secs(2)), work, done)?; +/// ``` #[derive(Debug, Clone)] +#[non_exhaustive] pub struct JobOptions { - /// Deliver `JobError::DeadlineExceeded` at this instant if the job has - /// not finished. The work keeps running and keeps its credit. - pub deadline: Option, - /// Cancel the previous job with this key (whatever its notification - /// state) and fence its queued messages. - pub key: Option, - /// `pipe_reply` only: cancel the job when its caller leaves or its - /// deadline passes. Off by default, as in OTP. - pub cancel_on_caller_gone: bool, - /// Emits in flight before `emit` waits (blocks in `emit_blocking`). - pub emit_credits: u16, + pub(crate) deadline: Option, + pub(crate) key: Option, + pub(crate) cancel_on_caller_gone: bool, + pub(crate) emit_credits: u16, } impl JobOptions { - /// Set [`deadline`](Self::deadline) to `timeout` from now, on the - /// runtime's clock (the simulated one under `spawned_sim`): - /// `JobOptions::default().deadline_after(Duration::from_secs(2))`. - /// - /// The deadline is hard: when it passes, `done` receives - /// `JobError::DeadlineExceeded`, the job's later emits fail with - /// [`JobGone`], and its eventual result is dropped. For a soft deadline - /// (stop starting new work, finish what is in hand), check the time - /// inside the work instead. - pub fn deadline_after(mut self, timeout: std::time::Duration) -> Self { - self.deadline = Some(spawned_rt::time::Instant::now() + timeout); + /// No deadline, no key, 16 emit credits, not cancelled when the caller + /// leaves. + pub fn new() -> Self { + Self::default() + } + + /// Cancel the previous job with this key (whatever its notification + /// state) and fence its queued messages: the replacement is the only + /// live job of the key. + pub fn key(mut self, key: impl Into) -> Self { + self.key = Some(key.into()); + self + } + + /// A hard deadline: a `Duration` from now or an `Instant` (on the + /// runtime's clock, the simulated one under `spawned_sim`), or + /// `Deadline::Never`. When it passes, `done` receives + /// `Err(JobError::DeadlineExceeded)`, the job's later emits fail with + /// [`JobGone`] and its eventual result is dropped; the work keeps + /// running (cooperative cancellation) and keeps its credit. For a soft + /// deadline (stop starting new work, finish what is in hand), check the + /// time inside the work instead. + pub fn deadline(mut self, deadline: impl Into) -> Self { + self.deadline = match deadline.into() { + super::Deadline::At(at) => Some(at), + super::Deadline::Never => None, + }; + self + } + + /// Emits in flight before `emit` waits (`emit_blocking` blocks). The + /// default is 16. + pub fn emit_credits(mut self, credits: std::num::NonZeroU16) -> Self { + self.emit_credits = credits.get(); + self + } + + /// Reply jobs (`pipe_reply*`) only: cancel the job when its caller + /// leaves or the request's deadline passes. Off by default, as in OTP. + /// Plain `pipe*` jobs have no caller: setting it there is a bug + /// (`debug_assert!`). + pub fn cancel_on_caller_gone(mut self) -> Self { + self.cancel_on_caller_gone = true; self } } @@ -120,22 +152,10 @@ impl Default for JobOptions { } } -/// How a job ended, delivered to `done` in the owner's turn. -#[derive(Debug)] -#[non_exhaustive] -pub enum JobOutcome { - Ok(T), - Err(JobError), -} - -impl JobOutcome { - pub fn into_result(self) -> Result { - match self { - Self::Ok(value) => Ok(value), - Self::Err(error) => Err(error), - } - } -} +/// How a job ended, delivered to `done` in the owner's turn: the work's +/// value, or why there is none. A plain `Result`, so `?` and every +/// `Result` combinator work. +pub type JobOutcome = Result; /// Why a job produced no value. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] @@ -144,17 +164,19 @@ pub enum JobError { /// The work panicked; caught at the job boundary. #[error("job panicked: {0}")] Panicked(String), - /// `JobOptions::deadline` passed first. The deadline is hard: the job's - /// later emits fail and its eventual result is dropped. The work may - /// still be running (cooperative cancellation). + /// The job's deadline (`JobOptions::deadline`) passed first. The + /// deadline is hard: the job's later emits fail and its eventual result + /// is dropped. The work may still be running (cooperative + /// cancellation). #[error("job deadline exceeded")] DeadlineExceeded, } -/// Why a job was not spawned. +/// Why a job was not spawned. No job runs, and any credit or native slot +/// reserved for it was released. #[derive(Debug, thiserror::Error)] #[non_exhaustive] -pub enum PipeErrorKind { +pub enum PipeError { /// The owner's `MAX_OUTSTANDING` credits are in use. #[error("deferred credits at the bound")] Overloaded, @@ -169,19 +191,14 @@ pub enum PipeErrorKind { OwnerStopping, } -/// `pipe*` failed; no job was spawned and any reserved credit was released. -#[derive(Debug, thiserror::Error)] -#[error("{kind}")] -pub struct PipeError { - pub kind: PipeErrorKind, -} - /// `pipe_reply*` failed; the token comes back in every case. #[derive(Debug)] +#[non_exhaustive] pub enum PipeReplyError { + /// The job could not be spawned. Failed { token: super::ReplyTo, - kind: PipeErrorKind, + error: PipeError, }, /// The request is no longer open (or past its deadline). Closed(super::ReplyTo), diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 8f9094bd..0ba4cda4 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -118,6 +118,21 @@ impl Deadline { } } +/// A timeout from now: [`Deadline::after`]. Every call that takes +/// `impl Into` takes a `Duration`, an `Instant` or a `Deadline`. +impl From for Deadline { + fn from(timeout: Duration) -> Self { + Self::after(timeout) + } +} + +/// An absolute instant on the `spawned_rt::time` clock. +impl From for Deadline { + fn from(at: Instant) -> Self { + Self::At(at) + } +} + fn record(_delivery: Delivery) { #[cfg(feature = "metrics")] crate::instrumentation::reply(_delivery == Delivery::Delivered); diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 32cfad61..a3696e9b 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -83,7 +83,7 @@ pub trait Actor: Send + Sized + 'static { /// /// The default is small: an actor that runs many blocking jobs at once /// (I/O, proofs, signature checks) usually needs more, and under load the - /// only symptom is `PipeErrorKind::NativeBudget` refusals (the first one + /// only symptom is `PipeError::NativeBudget` refusals (the first one /// per instance is logged as a `native_budget_exhausted` warning). Set it /// with `#[actor(max_native_jobs = N)]` or `ChildSpec::max_native_jobs`. const MAX_NATIVE_JOBS: Option = None; diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index 780f1df7..996c4ccd 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -8,8 +8,8 @@ use super::{Actor, Context, Envelope, Handler}; use crate::message::Message; use crate::reply::deadline::Expirable; use crate::reply::jobs::{ - JobCredit, JobDeadline, JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, JobState, - PipeError, PipeErrorKind, PipeReplyError, SendScheduled, + JobCredit, JobDeadline, JobError, JobGone, JobHandle, JobOptions, JobOutcome, JobState, + PipeError, PipeReplyError, SendScheduled, }; use crate::reply::slot::{Refused, Slot}; use crate::reply::{now_ticks, PendingReply, ReplyTo}; @@ -91,7 +91,7 @@ where ctx: &'a Context, ) -> Pin + Send + 'a>> { let TerminalEnvelope { job, done, outcome } = *self; - let kind = if matches!(outcome, JobOutcome::Err(JobError::DeadlineExceeded)) { + let kind = if matches!(outcome, Err(JobError::DeadlineExceeded)) { "deadline" } else { "done" @@ -133,9 +133,9 @@ struct Run { fn end_of(outcome: &JobOutcome) -> &'static str { match outcome { - JobOutcome::Ok(_) => "ok", - JobOutcome::Err(JobError::Panicked(_)) => "panicked", - JobOutcome::Err(JobError::DeadlineExceeded) => "deadline", + Ok(_) => "ok", + Err(JobError::Panicked(_)) => "panicked", + Err(JobError::DeadlineExceeded) => "deadline", } } @@ -293,14 +293,11 @@ impl Drop for Ended { } fn caught(result: std::thread::Result) -> JobOutcome { - match result { - Ok(value) => JobOutcome::Ok(value), - Err(panic) => { - let message = crate::error::panic_message(&*panic); - crate::mailbox::dispose_panic(panic); - JobOutcome::Err(JobError::Panicked(message)) - } - } + result.map_err(|panic| { + let message = crate::error::panic_message(&*panic); + crate::mailbox::dispose_panic(panic); + JobError::Panicked(message) + }) } /// The job's handle on its owner while it runs. @@ -431,22 +428,24 @@ impl JobCtx { // Context API // --------------------------------------------------------------------------- -/// A job admission reserved ahead of building the work: `ctx.reserve_job()` -/// takes the credit (and checks the owner is not stopping) without consuming -/// anything, so a caller that is refused still holds what it would have -/// moved into `work` and `done`. Spawning from a permit cannot fail. -/// Dropping an unused permit returns its credit. +/// A job admission reserved ahead of building the work: +/// `ctx.reserve_job(opts)` takes the credit (and checks the owner is not +/// stopping) without consuming anything, so a caller that is refused still +/// holds what it would have moved into `work` and `done`. Spawning from a +/// permit cannot fail. Dropping an unused permit returns its credit. #[must_use = "an unused permit only holds a credit; spawn the job with it or drop it"] pub struct JobPermit { ctx: Context, + opts: JobOptions, credit: JobCredit, } /// [`JobPermit`] for a blocking job: it also holds a native slot -/// (`MAX_NATIVE_JOBS`). From `ctx.reserve_blocking_job()`. +/// (`MAX_NATIVE_JOBS`). From `ctx.reserve_blocking_job(opts)`. #[must_use = "an unused permit only holds a credit and a native slot; spawn the job with it or drop it"] pub struct BlockingJobPermit { ctx: Context, + opts: JobOptions, credit: JobCredit, native: NativeSlot, } @@ -464,49 +463,11 @@ impl std::fmt::Debug for BlockingJobPermit { } impl JobPermit { - /// [`Context::pipe`] with this permit's credit. - pub fn pipe(self, work: W, done: D) -> JobHandle - where - A: Handler, - M: Message, - T: Send + 'static, - W: Future + Send + 'static, - D: FnOnce(JobOutcome) -> M + Send + 'static, - { - self.pipe_with(JobOptions::default(), work, done) - } - - /// [`Context::pipe_with`] with this permit's credit. If the owner began - /// exiting since the reservation, nothing runs: `work` and `done` are - /// dropped and the handle reports the job cancelled (as owner exit - /// cancels every job). - pub fn pipe_with(self, opts: JobOptions, work: W, done: D) -> JobHandle - where - A: Handler, - M: Message, - T: Send + 'static, - W: Future + Send + 'static, - D: FnOnce(JobOutcome) -> M + Send + 'static, - { - let JobPermit { ctx, credit } = self; - let Some(state) = ctx.start_job(&opts) else { - return ctx.refused_job(&opts, (work, done)); - }; - if !ctx.admit_job(&state, &opts) { - return ctx.refuse(state, (work, done, credit)); - } - ctx.launch_async::( - opts, - state, - Hold::Credit(credit), - work, - Box::new(done), - None, - ) - } - - /// [`Context::pipe_with_ctx`] with this permit's credit. - pub fn pipe_with_ctx(self, opts: JobOptions, work: F, done: D) -> JobHandle + /// [`Context::pipe`] with this permit's options and credit; cannot + /// fail. If the owner began exiting since the reservation, nothing runs: + /// `work` and `done` are dropped and the handle reports the job + /// cancelled (as owner exit cancels every job). + pub fn pipe(self, work: F, done: D) -> JobHandle where A: Handler, M: Message, @@ -515,7 +476,7 @@ impl JobPermit { W: Future + Send + 'static, D: FnOnce(JobOutcome) -> M + Send + 'static, { - let JobPermit { ctx, credit } = self; + let JobPermit { ctx, opts, credit } = self; let Some(state) = ctx.start_job(&opts) else { return ctx.refused_job(&opts, (work, done)); }; @@ -541,19 +502,20 @@ impl JobPermit { } impl BlockingJobPermit { - /// [`Context::pipe_blocking_with`] with this permit's credit and native - /// slot; cannot fail. If the owner began exiting since the reservation, - /// nothing runs (see [`JobPermit::pipe_with`]). - pub fn pipe_blocking_with(self, opts: JobOptions, work: W, done: D) -> JobHandle + /// [`Context::pipe_blocking`] with this permit's options, credit and + /// native slot; cannot fail. If the owner began exiting since the + /// reservation, nothing runs (see [`JobPermit::pipe`]). + pub fn pipe_blocking(self, work: W, done: D) -> JobHandle where A: Handler, M: Message, T: Send + 'static, - W: FnOnce(&JobCtx) -> T + Send + 'static, + W: FnOnce(JobCtx) -> T + Send + 'static, D: FnOnce(JobOutcome) -> M + Send + 'static, { let BlockingJobPermit { ctx, + opts, credit, mut native, } = self; @@ -606,13 +568,9 @@ impl Context { fn reserve_credit(&self) -> Result { if self.shared.replies.is_exiting() { - return Err(PipeError { - kind: PipeErrorKind::OwnerStopping, - }); + return Err(PipeError::OwnerStopping); } - JobCredit::reserve(&self.shared.replies).ok_or(PipeError { - kind: PipeErrorKind::Overloaded, - }) + JobCredit::reserve(&self.shared.replies).ok_or(PipeError::Overloaded) } /// Reserve a native slot (`MAX_NATIVE_JOBS`) and where the job will run: @@ -620,9 +578,9 @@ impl Context { /// here, never at launch). A failure releases the slot. A managed /// actor's scope lease is taken later, at admission /// ([`admit_native`](Self::admit_native)). - fn reserve_native(&self) -> Result, PipeErrorKind> { + fn reserve_native(&self) -> Result, PipeError> { if !self.shared.jobs.reserve_native() { - return Err(PipeErrorKind::NativeBudget); + return Err(PipeError::NativeBudget); } let mut slot = NativeSlot { ctx: self.clone(), @@ -648,7 +606,7 @@ impl Context { task(); } }) - .map_err(PipeErrorKind::SpawnFailed)?; + .map_err(PipeError::SpawnFailed)?; RunnerPool::Thread(start) } }; @@ -688,25 +646,38 @@ impl Context { /// and spawning from the permit cannot fail. /// /// ```ignore - /// let Ok(permit) = ctx.reserve_job() else { + /// let Ok(permit) = ctx.reserve_job(JobOptions::new()) else { /// return self.report(id, Verdict::Overloaded); // `id` was not moved /// }; - /// permit.pipe(fetch(object), move |res| Fetched { id, res }); + /// permit.pipe(|_| fetch(object), move |res| Fetched { id, res }); /// ``` - pub fn reserve_job(&self) -> Result, PipeError> { + pub fn reserve_job(&self, opts: JobOptions) -> Result, PipeError> { + debug_assert!( + !opts.cancel_on_caller_gone, + "cancel_on_caller_gone applies to pipe_reply* only" + ); Ok(JobPermit { ctx: self.clone(), credit: self.reserve_credit()?, + opts, }) } /// [`reserve_job`](Self::reserve_job) for a blocking job: also takes a /// native slot (`MAX_NATIVE_JOBS`, `NativeBudget` at the bound). - pub fn reserve_blocking_job(&self) -> Result, PipeError> { + pub fn reserve_blocking_job( + &self, + opts: JobOptions, + ) -> Result, PipeError> { + debug_assert!( + !opts.cancel_on_caller_gone, + "cancel_on_caller_gone applies to pipe_reply* only" + ); let credit = self.reserve_credit()?; - let native = self.reserve_native().map_err(|kind| PipeError { kind })?; + let native = self.reserve_native()?; Ok(BlockingJobPermit { ctx: self.clone(), + opts, credit, native, }) @@ -727,7 +698,7 @@ impl Context { state: run.state.clone(), notify: Mutex::new(Some(Box::new(move || { if let Some(run) = weak.upgrade() { - run.notify(JobOutcome::Err(JobError::DeadlineExceeded)); + run.notify(Err(JobError::DeadlineExceeded)); } }))), }); @@ -810,7 +781,7 @@ impl Context { M: Message, T: Send + 'static, R: Send + 'static, - W: FnOnce(&JobCtx) -> T + Send + 'static, + W: FnOnce(JobCtx) -> T + Send + 'static, { let run = Arc::new(Run { state: state.clone(), @@ -831,7 +802,7 @@ impl Context { .expect("a native slot is reserved with its runner"); runner.run(move || { let _ended = ended; - let result = std::panic::catch_unwind(AssertUnwindSafe(|| work(&job))); + let result = std::panic::catch_unwind(AssertUnwindSafe(|| work(job))); // Execution ended: release the native slot now; the run returns // the credit when the terminal message is enqueued. drop(native); @@ -840,67 +811,21 @@ impl Context { JobHandle::new(state) } - /// Run `work` as a job owned by this actor instance and deliver - /// `done(outcome)` as one message in a later turn (`handle_info`). A - /// panic arrives as `JobOutcome::Err(JobError::Panicked)`. Holds one - /// `MAX_OUTSTANDING` credit while it executes. Rejects at once at a - /// bound; never waits. On `Err`, `work` and `done` are dropped: use - /// [`reserve_job`](Self::reserve_job) to keep them. - pub fn pipe(&self, work: W, done: D) -> Result - where - A: Handler, - M: Message, - T: Send + 'static, - W: Future + Send + 'static, - D: FnOnce(JobOutcome) -> M + Send + 'static, - { - self.pipe_with(JobOptions::default(), work, done) - } - - /// [`pipe`](Self::pipe) that first cancels the previous job with `key` - /// (whatever its notification state) and fences its queued messages. - pub fn pipe_keyed( - &self, - key: impl Into, - work: W, - done: D, - ) -> Result - where - A: Handler, - M: Message, - T: Send + 'static, - W: Future + Send + 'static, - D: FnOnce(JobOutcome) -> M + Send + 'static, - { - let opts = JobOptions { - key: Some(key.into()), - ..JobOptions::default() - }; - self.pipe_with(opts, work, done) - } - - /// [`pipe`](Self::pipe) with options (deadline, key, emit credits). - pub fn pipe_with( - &self, - opts: JobOptions, - work: W, - done: D, - ) -> Result - where - A: Handler, - M: Message, - T: Send + 'static, - W: Future + Send + 'static, - D: FnOnce(JobOutcome) -> M + Send + 'static, - { - Ok(self.reserve_job()?.pipe_with(opts, work, done)) - } - - /// [`pipe_with`](Self::pipe_with) for async work that needs its - /// [`JobCtx`]: `work` receives it to check cancellation and `emit` fenced - /// messages (bounded by `JobOptions::emit_credits`; the terminal message - /// is always last). - pub fn pipe_with_ctx( + /// Run async work as a job owned by this actor instance and deliver + /// `done(outcome)` as one message in a later turn (`handle_info`), as + /// `Task.async` + `handle_info` would. `work` receives the job's + /// [`JobCtx`] (to check cancellation and `emit` fenced messages, bounded + /// by `emit_credits`; the terminal message is always last) and returns + /// the future to run; `|_| fut` when it needs neither. A panic arrives + /// as `Err(JobError::Panicked)`. Holds one `MAX_OUTSTANDING` credit + /// while it executes. Rejects at once at a bound; never waits. On `Err`, + /// `work` and `done` are dropped: use [`reserve_job`](Self::reserve_job) + /// to keep them. + /// + /// ```ignore + /// ctx.pipe(JobOptions::new().key("fetch"), |_| fetch(url), Fetched)?; + /// ``` + pub fn pipe( &self, opts: JobOptions, work: F, @@ -914,16 +839,16 @@ impl Context { W: Future + Send + 'static, D: FnOnce(JobOutcome) -> M + Send + 'static, { - Ok(self.reserve_job()?.pipe_with_ctx(opts, work, done)) + Ok(self.reserve_job(opts)?.pipe(work, done)) } /// Run blocking `work` on Tokio's blocking pool, under the actor's /// `MAX_NATIVE_JOBS` budget (cancellation stays cooperative: the job - /// checks its [`JobCtx`]). `work` gets a [`JobCtx`] to check + /// checks its [`JobCtx`]). `work` gets the [`JobCtx`] to check /// cancellation and `emit_blocking` fenced messages; a panic arrives as /// `JobError::Panicked`. On `Err`, `work` and `done` are dropped: use /// [`reserve_blocking_job`](Self::reserve_blocking_job) to keep them. - pub fn pipe_blocking_with( + pub fn pipe_blocking( &self, opts: JobOptions, work: W, @@ -933,43 +858,25 @@ impl Context { A: Handler, M: Message, T: Send + 'static, - W: FnOnce(&JobCtx) -> T + Send + 'static, + W: FnOnce(JobCtx) -> T + Send + 'static, D: FnOnce(JobOutcome) -> M + Send + 'static, { - Ok(self - .reserve_blocking_job()? - .pipe_blocking_with(opts, work, done)) + Ok(self.reserve_blocking_job(opts)?.pipe_blocking(work, done)) } - /// Delegate a deferred reply: run `work` as a job linked to `token`'s - /// request and deliver `done(outcome, token)` in a later turn. The + /// Delegate a deferred reply: run async work as a job linked to + /// `token`'s request and deliver `done(outcome, token)` in a later turn. + /// `work` receives the [`JobCtx`] as in [`pipe`](Self::pipe). The /// obligation and credit stay in this actor: the token and its first - /// linked job share one credit. On every error the token comes back. - pub fn pipe_reply( - &self, - token: ReplyTo, - work: W, - done: D, - ) -> Result> - where - A: Handler, - M: Message, - R: Send + 'static, - T: Send + 'static, - W: Future + Send + 'static, - D: FnOnce(JobOutcome, ReplyTo) -> M + Send + 'static, - { - self.pipe_reply_with(JobOptions::default(), token, work, done) - } - - /// [`pipe_reply`](Self::pipe_reply) with options. With - /// `cancel_on_caller_gone` the job is cancelled when the caller leaves - /// or the request's deadline passes (off by default, as in OTP). - pub fn pipe_reply_with( + /// linked job share one credit. With + /// [`JobOptions::cancel_on_caller_gone`] the job is cancelled when the + /// caller leaves or the request's deadline passes (off by default, as + /// in OTP). On every error the token comes back. + pub fn pipe_reply( &self, opts: JobOptions, token: ReplyTo, - work: W, + work: F, done: D, ) -> Result> where @@ -977,18 +884,26 @@ impl Context { M: Message, R: Send + 'static, T: Send + 'static, + F: FnOnce(JobCtx) -> W, W: Future + Send + 'static, D: FnOnce(JobOutcome, ReplyTo) -> M + Send + 'static, { let (state, slot, token) = self.link_reply(&opts, token)?; let watch = opts.cancel_on_caller_gone.then(|| slot.clone()); let hold = Hold::Linked(LinkedJob(slot)); + let job = JobCtx { + state: state.clone(), + ctx: self.clone(), + }; + // User code; a panic here drops the hold, which detaches the + // linked job from the slot. + let work = work(job); if !self.admit_job(&state, &opts) { // Dropping the hold detaches the linked job from the slot. - drop((hold, watch)); + drop((hold, watch, work)); return Err(PipeReplyError::Failed { token, - kind: PipeErrorKind::OwnerStopping, + error: PipeError::OwnerStopping, }); } Ok(self.launch_async(opts, state, hold, work, linked_done(token, done), watch)) @@ -996,7 +911,7 @@ impl Context { /// Blocking [`pipe_reply`](Self::pipe_reply) on Tokio's blocking pool, /// under the `MAX_NATIVE_JOBS` budget. - pub fn pipe_reply_blocking_with( + pub fn pipe_reply_blocking( &self, opts: JobOptions, token: ReplyTo, @@ -1008,7 +923,7 @@ impl Context { M: Message, R: Send + 'static, T: Send + 'static, - W: FnOnce(&JobCtx) -> T + Send + 'static, + W: FnOnce(JobCtx) -> T + Send + 'static, D: FnOnce(JobOutcome, ReplyTo) -> M + Send + 'static, { let (state, slot, token) = self.link_reply(&opts, token)?; @@ -1016,16 +931,16 @@ impl Context { let mut native = match self.reserve_native() { Ok(native) => native, // Dropping the hold detaches the linked job from the slot. - Err(kind) => { + Err(error) => { drop(hold); - return Err(PipeReplyError::Failed { token, kind }); + return Err(PipeReplyError::Failed { token, error }); } }; if !self.admit_native(&state, &opts, &mut native) { drop((hold, native)); return Err(PipeReplyError::Failed { token, - kind: PipeErrorKind::OwnerStopping, + error: PipeError::OwnerStopping, }); } if opts.cancel_on_caller_gone { @@ -1049,7 +964,7 @@ impl Context { let Some(state) = self.start_job(opts) else { return Err(PipeReplyError::Failed { token, - kind: PipeErrorKind::OwnerStopping, + error: PipeError::OwnerStopping, }); }; let slot = token.slot_arc().clone(); @@ -1058,7 +973,7 @@ impl Context { Err(Refused::Closed) => Err(PipeReplyError::Closed(token)), Err(Refused::Overloaded) => Err(PipeReplyError::Failed { token, - kind: PipeErrorKind::Overloaded, + error: PipeError::Overloaded, }), } } @@ -1111,4 +1026,22 @@ mod tests { owner.context().stop(); owner.join().await; } + + /// `cancel_on_caller_gone` applies to reply jobs only: a plain job has no + /// caller, so setting it there is a bug, caught in debug builds. + #[cfg(debug_assertions)] + #[tokio::test] + async fn cancel_on_caller_gone_on_a_plain_job_is_a_bug() { + let owner = Owner.start(); + let ctx = owner.context(); + let refused = std::panic::catch_unwind(AssertUnwindSafe(|| { + ctx.reserve_blocking_job(JobOptions::new().cancel_on_caller_gone()) + .map(drop) + })); + assert!(refused.is_err(), "the option was accepted on a plain job"); + let plain = ctx.reserve_blocking_job(JobOptions::new()).map(drop); + assert!(plain.is_ok()); + ctx.stop(); + owner.join().await; + } } diff --git a/concurrency/src/tasks/mod.rs b/concurrency/src/tasks/mod.rs index f52f1157..5decdded 100644 --- a/concurrency/src/tasks/mod.rs +++ b/concurrency/src/tasks/mod.rs @@ -8,8 +8,7 @@ mod stream_tests; mod timer_tests; pub use crate::reply::jobs::{ - JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, PipeError, PipeErrorKind, - PipeReplyError, + JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, PipeError, PipeReplyError, }; pub use crate::reply::{ Deadline, DeferError, Delivery, InsertError, PendingReplies, PendingReply, ReplyFuture, diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index a6c2674c..67df8676 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -51,7 +51,7 @@ pub trait Actor: Send + Sized + 'static { /// /// The default is small: an actor that runs many blocking jobs at once /// usually needs more, and under load the only symptom is - /// `PipeErrorKind::NativeBudget` refusals (the first one per instance is + /// `PipeError::NativeBudget` refusals (the first one per instance is /// logged as a `native_budget_exhausted` warning). Set it with /// `#[actor(max_native_jobs = N)]`. const MAX_NATIVE_JOBS: Option = None; diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index 81bce755..c1562634 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -1,6 +1,6 @@ //! `threads` mode pipe-to-self jobs (docs/design/deferred-replies.md §9): //! one owned native thread per admitted job under `MAX_NATIVE_JOBS` -//! (`pipe_blocking_with`, `pipe_reply_blocking_with`), their envelopes and +//! (`pipe_blocking`, `pipe_reply_blocking`), their envelopes and //! `JobCtx`. The shared pool comes with rank 11b. //! //! A child module of `threads::actor`, so it reaches the private envelope and @@ -11,7 +11,7 @@ use crate::message::Message; use crate::reply::deadline::Expirable; use crate::reply::jobs::{ JobCredit, JobDeadline, JobError, JobGone, JobHandle, JobOptions, JobOutcome, JobState, - PipeError, PipeErrorKind, PipeReplyError, SendScheduled, + PipeError, PipeReplyError, SendScheduled, }; use crate::reply::slot::{Refused, Slot}; use crate::reply::{now_ticks, PendingReply, ReplyTo}; @@ -74,7 +74,7 @@ where { fn handle(self: Box, actor: &mut A, ctx: &Context) { let TerminalEnvelope { job, done, outcome } = *self; - let kind = if matches!(outcome, JobOutcome::Err(JobError::DeadlineExceeded)) { + let kind = if matches!(outcome, Err(JobError::DeadlineExceeded)) { "deadline" } else { "done" @@ -111,9 +111,9 @@ struct Run { fn end_of(outcome: &JobOutcome) -> &'static str { match outcome { - JobOutcome::Ok(_) => "ok", - JobOutcome::Err(JobError::Panicked(_)) => "panicked", - JobOutcome::Err(JobError::DeadlineExceeded) => "deadline", + Ok(_) => "ok", + Err(JobError::Panicked(_)) => "panicked", + Err(JobError::DeadlineExceeded) => "deadline", } } @@ -226,14 +226,11 @@ impl Drop for Ended { } fn caught(result: std::thread::Result) -> JobOutcome { - match result { - Ok(value) => JobOutcome::Ok(value), - Err(panic) => { - let message = crate::error::panic_message(&*panic); - crate::mailbox::dispose_panic(panic); - JobOutcome::Err(JobError::Panicked(message)) - } - } + result.map_err(|panic| { + let message = crate::error::panic_message(&*panic); + crate::mailbox::dispose_panic(panic); + JobError::Panicked(message) + }) } /// The job's handle on its owner while it runs. @@ -341,7 +338,7 @@ struct NativeWorker { } /// A blocking job's admission reserved ahead of building the work: -/// `ctx.reserve_blocking_job()` checks the owner is running and takes a +/// `ctx.reserve_blocking_job(opts)` checks the owner is running and takes a /// `MAX_OUTSTANDING` credit, a native slot and the job's (parked) thread, so /// a caller that is refused still holds what it would have moved into /// `work` and `done`, and spawning from the permit cannot fail. Dropping an @@ -349,6 +346,7 @@ struct NativeWorker { #[must_use = "an unused permit only holds a credit and a native thread; spawn the job with it or drop it"] pub struct BlockingJobPermit { ctx: Context, + opts: JobOptions, credit: JobCredit, worker: NativeWorker, } @@ -360,20 +358,21 @@ impl std::fmt::Debug for BlockingJobPermit { } impl BlockingJobPermit { - /// [`Context::pipe_blocking_with`] with this permit; cannot fail. If the - /// owner began exiting since the reservation, nothing runs: `work` and - /// `done` are dropped and the handle reports the job cancelled (as - /// owner exit cancels every job). - pub fn pipe_blocking_with(self, opts: JobOptions, work: W, done: D) -> JobHandle + /// [`Context::pipe_blocking`] with this permit's options, credit and + /// thread; cannot fail. If the owner began exiting since the + /// reservation, nothing runs: `work` and `done` are dropped and the + /// handle reports the job cancelled (as owner exit cancels every job). + pub fn pipe_blocking(self, work: W, done: D) -> JobHandle where A: Handler, M: Message, T: Send + 'static, - W: FnOnce(&JobCtx) -> T + Send + 'static, + W: FnOnce(JobCtx) -> T + Send + 'static, D: FnOnce(JobOutcome) -> M + Send + 'static, { let BlockingJobPermit { ctx, + opts, credit, worker, } = self; @@ -407,18 +406,18 @@ impl Context { .admit(state, opts.key.clone(), opts.deadline.is_some()) } - fn reserve_credit(&self) -> Result { + fn reserve_credit(&self) -> Result { if self.replies.is_exiting() { - return Err(PipeErrorKind::OwnerStopping); + return Err(PipeError::OwnerStopping); } - JobCredit::reserve(&self.replies).ok_or(PipeErrorKind::Overloaded) + JobCredit::reserve(&self.replies).ok_or(PipeError::Overloaded) } /// Reserve a native slot and spawn its thread, parked until the job is /// spawned. A spawn failure releases the slot. - fn spawn_worker(&self) -> Result, PipeErrorKind> { + fn spawn_worker(&self) -> Result, PipeError> { if !self.jobs.reserve_native() { - return Err(PipeErrorKind::NativeBudget); + return Err(PipeError::NativeBudget); } let native = NativeSlot(self.clone()); let (start, started) = std::sync::mpsc::channel::(); @@ -429,7 +428,7 @@ impl Context { task(); } }) - .map_err(PipeErrorKind::SpawnFailed)?; + .map_err(PipeError::SpawnFailed)?; Ok(NativeWorker { start, native }) } @@ -439,16 +438,24 @@ impl Context { /// and spawning from the permit cannot fail. /// /// ```ignore - /// let Ok(permit) = ctx.reserve_blocking_job() else { + /// let Ok(permit) = ctx.reserve_blocking_job(JobOptions::new()) else { /// return self.report(id, Verdict::Overloaded); // `id` was not moved /// }; - /// permit.pipe_blocking_with(opts, move |_| check(&object), move |res| Checked { id, res }); + /// permit.pipe_blocking(move |_| check(&object), move |res| Checked { id, res }); /// ``` - pub fn reserve_blocking_job(&self) -> Result, PipeError> { - let credit = self.reserve_credit().map_err(|kind| PipeError { kind })?; - let worker = self.spawn_worker().map_err(|kind| PipeError { kind })?; + pub fn reserve_blocking_job( + &self, + opts: JobOptions, + ) -> Result, PipeError> { + debug_assert!( + !opts.cancel_on_caller_gone, + "cancel_on_caller_gone applies to pipe_reply* only" + ); + let credit = self.reserve_credit()?; + let worker = self.spawn_worker()?; Ok(BlockingJobPermit { ctx: self.clone(), + opts, credit, worker, }) @@ -469,7 +476,7 @@ impl Context { state: run.state.clone(), notify: Mutex::new(Some(Box::new(move || { if let Some(run) = weak.upgrade() { - run.notify(JobOutcome::Err(JobError::DeadlineExceeded)); + run.notify(Err(JobError::DeadlineExceeded)); } }))), }); @@ -497,7 +504,7 @@ impl Context { M: Message, T: Send + 'static, R: Send + 'static, - W: FnOnce(&JobCtx) -> T + Send + 'static, + W: FnOnce(JobCtx) -> T + Send + 'static, { let run = Arc::new(Run { state: state.clone(), @@ -515,7 +522,7 @@ impl Context { let NativeWorker { start, native } = worker; let task: NativeTask = Box::new(move || { let _ended = ended; - let result = std::panic::catch_unwind(AssertUnwindSafe(|| work(&job))); + let result = std::panic::catch_unwind(AssertUnwindSafe(|| work(job))); // Execution ended: release the native slot now; the run returns // the credit when the terminal message is enqueued. drop(native); @@ -530,12 +537,12 @@ impl Context { /// Run blocking `work` as a job on an owned native thread, under the /// actor's `MAX_NATIVE_JOBS` budget, and deliver `done(outcome)` as one /// message in a later turn. A panic arrives as - /// `JobOutcome::Err(JobError::Panicked)`. `work` gets a [`JobCtx`] to + /// `Err(JobError::Panicked)`. `work` gets the [`JobCtx`] to /// check cancellation and `emit_blocking` fenced messages. Rejects at /// once at a bound; never waits. On `Err`, `work` and `done` are /// dropped: use [`reserve_blocking_job`](Self::reserve_blocking_job) to /// keep them. - pub fn pipe_blocking_with( + pub fn pipe_blocking( &self, opts: JobOptions, work: W, @@ -545,20 +552,20 @@ impl Context { A: Handler, M: Message, T: Send + 'static, - W: FnOnce(&JobCtx) -> T + Send + 'static, + W: FnOnce(JobCtx) -> T + Send + 'static, D: FnOnce(JobOutcome) -> M + Send + 'static, { - Ok(self - .reserve_blocking_job()? - .pipe_blocking_with(opts, work, done)) + Ok(self.reserve_blocking_job(opts)?.pipe_blocking(work, done)) } /// Delegate a deferred reply: run blocking `work` on an owned native /// thread, linked to `token`'s request, and deliver `done(outcome, token)` /// in a later turn. The obligation and credit stay in this actor: the /// token and its first linked job share one credit. On every error the - /// token comes back. - pub fn pipe_reply_blocking_with( + /// token comes back. With [`JobOptions::cancel_on_caller_gone`] the job + /// is cancelled when the caller leaves or the request's deadline passes + /// (off by default, as in OTP). + pub fn pipe_reply_blocking( &self, opts: JobOptions, token: ReplyTo, @@ -570,7 +577,7 @@ impl Context { M: Message, R: Send + 'static, T: Send + 'static, - W: FnOnce(&JobCtx) -> T + Send + 'static, + W: FnOnce(JobCtx) -> T + Send + 'static, D: FnOnce(JobOutcome, ReplyTo) -> M + Send + 'static, { let (state, slot, token) = self.link_reply(&opts, token)?; @@ -578,16 +585,16 @@ impl Context { let worker = match self.spawn_worker() { Ok(worker) => worker, // Dropping the hold detaches the linked job from the slot. - Err(kind) => { + Err(error) => { drop(hold); - return Err(PipeReplyError::Failed { token, kind }); + return Err(PipeReplyError::Failed { token, error }); } }; if !self.admit_job(&state, &opts) { drop((hold, worker)); return Err(PipeReplyError::Failed { token, - kind: PipeErrorKind::OwnerStopping, + error: PipeError::OwnerStopping, }); } if opts.cancel_on_caller_gone { @@ -612,7 +619,7 @@ impl Context { if self.replies.is_exiting() { return Err(PipeReplyError::Failed { token, - kind: PipeErrorKind::OwnerStopping, + error: PipeError::OwnerStopping, }); } let state = JobState::new(self.id(), opts.emit_credits); @@ -622,7 +629,7 @@ impl Context { Err(Refused::Closed) => Err(PipeReplyError::Closed(token)), Err(Refused::Overloaded) => Err(PipeReplyError::Failed { token, - kind: PipeErrorKind::Overloaded, + error: PipeError::Overloaded, }), } } @@ -657,7 +664,7 @@ mod tests { let ctx = owner.context(); for i in 0..2000 { let permit = ctx - .reserve_blocking_job() + .reserve_blocking_job(JobOptions::new()) .unwrap_or_else(|e| panic!("reservation {i} refused: {e}")); drop(permit); } @@ -683,4 +690,22 @@ mod tests { owner.context().stop(); owner.join(); } + + /// `cancel_on_caller_gone` applies to reply jobs only: a plain job has no + /// caller, so setting it there is a bug, caught in debug builds. + #[cfg(debug_assertions)] + #[test] + fn cancel_on_caller_gone_on_a_plain_job_is_a_bug() { + let owner = Owner.start(); + let ctx = owner.context(); + let refused = std::panic::catch_unwind(AssertUnwindSafe(|| { + ctx.reserve_blocking_job(JobOptions::new().cancel_on_caller_gone()) + .map(drop) + })); + assert!(refused.is_err(), "the option was accepted on a plain job"); + let plain = ctx.reserve_blocking_job(JobOptions::new()).map(drop); + assert!(plain.is_ok()); + ctx.stop(); + owner.join(); + } } diff --git a/concurrency/src/threads/mod.rs b/concurrency/src/threads/mod.rs index e40b7c5f..3eff7b52 100644 --- a/concurrency/src/threads/mod.rs +++ b/concurrency/src/threads/mod.rs @@ -9,8 +9,7 @@ mod timer_tests; mod stream_tests; pub use crate::reply::jobs::{ - JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, PipeError, PipeErrorKind, - PipeReplyError, + JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, PipeError, PipeReplyError, }; pub use crate::reply::{ Deadline, DeferError, Delivery, InsertError, PendingReplies, PendingReply, ReplyReceiver, diff --git a/concurrency/tests/deadline_service.rs b/concurrency/tests/deadline_service.rs index 00e76915..22234437 100644 --- a/concurrency/tests/deadline_service.rs +++ b/concurrency/tests/deadline_service.rs @@ -75,8 +75,8 @@ fn deadline_service_child() { let owner = Owner::default().start(); owner .__in_turn(|_, ctx| { - let opts = JobOptions::default().deadline_after(Duration::from_millis(20)); - ctx.pipe_blocking_with( + let opts = JobOptions::new().deadline(Duration::from_millis(20)); + ctx.pipe_blocking( opts, |job| { let at = Instant::now() + Duration::from_millis(40); @@ -108,12 +108,12 @@ fn deadline_service_child() { let fired = other .__in_turn(|_, ctx| { let (tx, rx) = std::sync::mpsc::channel(); - let opts = JobOptions::default().deadline_after(Duration::from_millis(20)); - ctx.pipe_blocking_with( + let opts = JobOptions::new().deadline(Duration::from_millis(20)); + ctx.pipe_blocking( opts, |_| std::thread::sleep(Duration::from_millis(300)), move |outcome: JobOutcome<()>| { - let _ = tx.send(outcome.into_result().is_err()); + let _ = tx.send(outcome.is_err()); Done }, ) diff --git a/concurrency/tests/death_boundary.rs b/concurrency/tests/death_boundary.rs index da711ef4..ee32b17b 100644 --- a/concurrency/tests/death_boundary.rs +++ b/concurrency/tests/death_boundary.rs @@ -183,8 +183,8 @@ mod tasks_mode { runtime.block_on(async { actor .__in_turn(move |_, ctx| { - ctx.pipe_with_ctx( - JobOptions::default(), + ctx.pipe( + JobOptions::new(), move |job| async move { let far = spawned_concurrency::Instant::now() + LIMIT * 10; if held { @@ -558,8 +558,8 @@ mod threads_mode { }; let (scheduled_tx, scheduled) = mpsc::channel(); spawned_test::threads::in_actor(&actor, move |_, ctx| { - ctx.pipe_blocking_with( - JobOptions::default(), + ctx.pipe_blocking( + JobOptions::new(), move |job| { let far = spawned_concurrency::Instant::now() + LIMIT * 10; job.emit_at_blocking(far, Waiting(payload)).unwrap(); diff --git a/concurrency/tests/deferred_api.rs b/concurrency/tests/deferred_api.rs index cc542590..d97c7c2a 100644 --- a/concurrency/tests/deferred_api.rs +++ b/concurrency/tests/deferred_api.rs @@ -36,7 +36,7 @@ mod tasks_mode { use spawned_concurrency::system::ActorSystem; use spawned_concurrency::tasks::{ Actor, ActorRef, ActorStart, Context, Handler, Instant, JobError, JobGone, JobOptions, - JobOutcome, PipeErrorKind, + JobOutcome, PipeError, }; use spawned_concurrency::{actor, send_handler}; use std::time::Duration; @@ -74,7 +74,7 @@ mod tasks_mode { } fn describe(name: &str, outcome: JobOutcome) -> Note { - Note(match outcome.into_result() { + Note(match outcome { Ok(v) => format!("{name}:ok:{v:?}"), Err(JobError::Panicked(m)) => format!("{name}:panicked:{m}"), Err(JobError::DeadlineExceeded) => format!("{name}:deadline"), @@ -108,24 +108,28 @@ mod tasks_mode { async fn macro_bounds_apply_and_permits_keep_the_parts() { let (actor, _log) = start(); let kinds = in_actor(&actor, |_, ctx| { - let a = ctx.reserve_blocking_job().expect("first native slot"); - let b = ctx.reserve_blocking_job().expect("second native slot"); + let a = ctx + .reserve_blocking_job(JobOptions::new()) + .expect("first native slot"); + let b = ctx + .reserve_blocking_job(JobOptions::new()) + .expect("second native slot"); // max_native_jobs = 2: the third is refused, and the caller keeps // whatever it meant to move into the job. - let third = ctx.reserve_blocking_job().err().map(|e| e.kind); + let third = ctx.reserve_blocking_job(JobOptions::new()).err(); drop((a, b)); - let c = ctx.reserve_job().expect("credit"); - let d = ctx.reserve_job().expect("credit"); - let e = ctx.reserve_job().expect("credit"); + let c = ctx.reserve_job(JobOptions::new()).expect("credit"); + let d = ctx.reserve_job(JobOptions::new()).expect("credit"); + let e = ctx.reserve_job(JobOptions::new()).expect("credit"); // max_outstanding = 3. - let fourth = ctx.reserve_job().err().map(|e| e.kind); + let fourth = ctx.reserve_job(JobOptions::new()).err(); drop((c, d, e)); - let again = ctx.reserve_job().is_ok(); + let again = ctx.reserve_job(JobOptions::new()).is_ok(); (third, fourth, again, ctx.deferred_outstanding()) }) .await; - assert!(matches!(kinds.0, Some(PipeErrorKind::NativeBudget))); - assert!(matches!(kinds.1, Some(PipeErrorKind::Overloaded))); + assert!(matches!(kinds.0, Some(PipeError::NativeBudget))); + assert!(matches!(kinds.1, Some(PipeError::Overloaded))); assert!(kinds.2, "dropped permits return their credits"); assert_eq!(kinds.3, 0); actor.context().stop(); @@ -137,14 +141,13 @@ mod tasks_mode { async fn blocking_jobs_run_on_the_tokio_blocking_pool() { let (actor, log) = start(); in_actor(&actor, |_, ctx| { - let permit = ctx.reserve_blocking_job().unwrap(); - permit.pipe_blocking_with( - JobOptions::default(), + let permit = ctx.reserve_blocking_job(JobOptions::new()).unwrap(); + permit.pipe_blocking( |_| std::thread::current().name().map(str::to_owned), |o| describe("thread", o), ); - ctx.pipe_blocking_with( - JobOptions::default(), + ctx.pipe_blocking( + JobOptions::new(), |_| -> u32 { panic!("check boom") }, |o| describe("panics", o), ) @@ -168,8 +171,8 @@ mod tasks_mode { async fn emit_at_arrives_late_but_before_the_terminal_message() { let (actor, log) = start(); in_actor(&actor, |_, ctx| { - ctx.pipe_blocking_with( - JobOptions::default(), + ctx.pipe_blocking( + JobOptions::new(), |job| { let at = Instant::now() + Duration::from_millis(80); job.emit_at_blocking(at, Note("late".into()))?; @@ -191,8 +194,8 @@ mod tasks_mode { async fn async_emit_at_holds_the_terminal_message() { let (actor, log) = start(); in_actor(&actor, |_, ctx| { - ctx.pipe_with_ctx( - JobOptions::default(), + ctx.pipe( + JobOptions::new(), |job| async move { let at = Instant::now() + Duration::from_millis(50); job.emit_at(at, Note("late".into())).await?; @@ -215,11 +218,8 @@ mod tasks_mode { async fn a_cancelled_job_fences_its_scheduled_emits() { let (actor, log) = start(); in_actor(&actor, |_, ctx| { - let opts = JobOptions { - key: Some("k".into()), - ..JobOptions::default() - }; - ctx.pipe_with_ctx( + let opts = JobOptions::new().key("k"); + ctx.pipe( opts.clone(), |job| async move { let at = Instant::now() + Duration::from_millis(60); @@ -233,11 +233,8 @@ mod tasks_mode { // The replacement cancels the previous job: its scheduled emit and // its held terminal message are dropped. in_actor(&actor, |_, ctx| { - let opts = JobOptions { - key: Some("k".into()), - ..JobOptions::default() - }; - ctx.pipe_with(opts, async { 1 }, |o| describe("new", o)) + let opts = JobOptions::new().key("k"); + ctx.pipe(opts, |_| async { 1 }, |o| describe("new", o)) .unwrap(); }) .await; @@ -254,8 +251,8 @@ mod tasks_mode { async fn job_deadlines_are_hard() { let (actor, log) = start(); in_actor(&actor, |_, ctx| { - let opts = JobOptions::default().deadline_after(Duration::from_millis(30)); - ctx.pipe_blocking_with( + let opts = JobOptions::new().deadline(Duration::from_millis(30)); + ctx.pipe_blocking( opts, |job| { std::thread::sleep(Duration::from_millis(120)); @@ -282,8 +279,8 @@ mod tasks_mode { async fn a_deadline_overtakes_a_result_held_for_scheduled_emits() { let (actor, log) = start(); in_actor(&actor, |_, ctx| { - let opts = JobOptions::default().deadline_after(Duration::from_millis(40)); - ctx.pipe_with_ctx( + let opts = JobOptions::new().deadline(Duration::from_millis(40)); + ctx.pipe( opts, |job| async move { let at = Instant::now() + Duration::from_millis(300); @@ -313,8 +310,8 @@ mod tasks_mode { async fn a_deadline_overtakes_a_blocking_jobs_held_result() { let (actor, log) = start(); in_actor(&actor, |_, ctx| { - let opts = JobOptions::default().deadline_after(Duration::from_millis(40)); - ctx.pipe_blocking_with( + let opts = JobOptions::new().deadline(Duration::from_millis(40)); + ctx.pipe_blocking( opts, |job| { let at = Instant::now() + Duration::from_millis(400); @@ -342,8 +339,8 @@ mod tasks_mode { async fn a_held_result_keeps_its_credit_until_the_terminal_message() { let (actor, log) = start(); let job = in_actor(&actor, |_, ctx| { - ctx.pipe_with_ctx( - JobOptions::default(), + ctx.pipe( + JobOptions::new(), |job| async move { let at = Instant::now() + Duration::from_millis(200); job.emit_at(at, Note("late".into())).await.unwrap(); @@ -374,8 +371,8 @@ mod tasks_mode { async fn emit_at_a_past_instant_is_delivered_immediately() { let (actor, log) = start(); in_actor(&actor, |_, ctx| { - ctx.pipe_with_ctx( - JobOptions::default(), + ctx.pipe( + JobOptions::new(), |job| async move { let past = Instant::now(); spawned_rt::tasks::sleep(Duration::from_millis(5)).await; @@ -424,18 +421,23 @@ mod tasks_mode { let _supervisor = builder.start().await.unwrap(); let actor = child.current().unwrap(); let refused = in_actor(&actor, |_, ctx| { - let native = ctx.reserve_blocking_job().expect("one native slot"); - let second_native = ctx.reserve_blocking_job().err().map(|e| e.kind); + let native = ctx + .reserve_blocking_job(JobOptions::new()) + .expect("one native slot"); + let second_native = ctx.reserve_blocking_job(JobOptions::new()).err(); drop(native); - let credits = (ctx.reserve_job(), ctx.reserve_job()); + let credits = ( + ctx.reserve_job(JobOptions::new()), + ctx.reserve_job(JobOptions::new()), + ); assert!(credits.0.is_ok() && credits.1.is_ok(), "two credits"); - let second_credit = ctx.reserve_job().err().map(|e| e.kind); + let second_credit = ctx.reserve_job(JobOptions::new()).err(); drop(credits); (second_native, second_credit) }) .await; - assert!(matches!(refused.0, Some(PipeErrorKind::NativeBudget))); - assert!(matches!(refused.1, Some(PipeErrorKind::Overloaded))); + assert!(matches!(refused.0, Some(PipeError::NativeBudget))); + assert!(matches!(refused.1, Some(PipeError::Overloaded))); let _ = system.shutdown(Duration::from_secs(5)).await; } } @@ -444,8 +446,7 @@ mod tasks_mode { mod threads_mode { use super::*; use spawned_concurrency::threads::{ - Actor, ActorStart, Context, Handler, Instant, JobGone, JobOptions, JobOutcome, - PipeErrorKind, + Actor, ActorStart, Context, Handler, Instant, JobGone, JobOptions, JobOutcome, PipeError, }; use spawned_concurrency::{actor, send_handler}; use spawned_test::threads::in_actor; @@ -485,7 +486,7 @@ mod threads_mode { } fn label(name: &str, outcome: JobOutcome>) -> Note { - Note(format!("{name}:{:?}", outcome.into_result())) + Note(format!("{name}:{:?}", outcome)) } #[test] @@ -493,11 +494,12 @@ mod threads_mode { let log = Arc::new(Log::default()); let actor = Owner { log: log.clone() }.start(); let refused = in_actor(&actor, |_, ctx| { - let permit = ctx.reserve_blocking_job().expect("one native slot"); + let permit = ctx + .reserve_blocking_job(JobOptions::new()) + .expect("one native slot"); // max_native_jobs = 1. - let second = ctx.reserve_blocking_job().err().map(|e| e.kind); - permit.pipe_blocking_with( - JobOptions::default(), + let second = ctx.reserve_blocking_job(JobOptions::new()).err(); + permit.pipe_blocking( |job| { let at = Instant::now() + Duration::from_millis(60); job.emit_at_blocking(at, Note("late".into()))?; @@ -507,7 +509,7 @@ mod threads_mode { ); second }); - assert!(matches!(refused, Some(PipeErrorKind::NativeBudget))); + assert!(matches!(refused, Some(PipeError::NativeBudget))); assert_eq!(wait_for(&log, 3), ["now", "late", "job:Ok(Ok(()))"]); actor.context().stop(); actor.join(); @@ -520,8 +522,8 @@ mod threads_mode { let log = Arc::new(Log::default()); let actor = Owner { log: log.clone() }.start(); in_actor(&actor, |_, ctx| { - let opts = JobOptions::default().deadline_after(Duration::from_millis(40)); - ctx.pipe_blocking_with( + let opts = JobOptions::new().deadline(Duration::from_millis(40)); + ctx.pipe_blocking( opts, |job| { let at = Instant::now() + Duration::from_millis(300); @@ -546,8 +548,8 @@ mod threads_mode { let log = Arc::new(Log::default()); let actor = Owner { log: log.clone() }.start(); let job = in_actor(&actor, |_, ctx| { - ctx.pipe_blocking_with( - JobOptions::default(), + ctx.pipe_blocking( + JobOptions::new(), |job| { let at = Instant::now() + Duration::from_millis(200); job.emit_at_blocking(at, Note("late".into())) @@ -578,8 +580,8 @@ mod threads_mode { let log = Arc::new(Log::default()); let actor = Owner { log: log.clone() }.start(); in_actor(&actor, |_, ctx| { - ctx.pipe_blocking_with( - JobOptions::default(), + ctx.pipe_blocking( + JobOptions::new(), |job| { let past = Instant::now(); std::thread::sleep(Duration::from_millis(5)); @@ -645,13 +647,11 @@ mod caller_gone_vs_shortening_tasks { ctx: &Context, ) { let token = reply.defer().unwrap(); - let opts = JobOptions { - cancel_on_caller_gone: true, - ..JobOptions::default() - } - .deadline_after(Duration::from_millis(20)); + let opts = JobOptions::new() + .cancel_on_caller_gone() + .deadline(Duration::from_millis(20)); let job = ctx - .pipe_reply_blocking_with( + .pipe_reply_blocking( opts, token, |_| std::thread::sleep(Duration::from_millis(300)), @@ -681,12 +681,9 @@ mod caller_gone_vs_shortening_tasks { impl Handler for Delegator { async fn handle(&mut self, _: Leave, reply: PendingReply<'_, ()>, ctx: &Context) { let token = reply.defer().unwrap(); - let opts = JobOptions { - cancel_on_caller_gone: true, - ..JobOptions::default() - }; + let opts = JobOptions::new().cancel_on_caller_gone(); let job = ctx - .pipe_reply_blocking_with( + .pipe_reply_blocking( opts, token, |job| { @@ -708,7 +705,7 @@ mod caller_gone_vs_shortening_tasks { impl Handler for Delegator { async fn handle(&mut self, msg: Delegated, _: PendingReply<'_, ()>, _: &Context) { let Delegated(outcome, token) = msg; - let deadline = matches!(outcome.into_result(), Err(JobError::DeadlineExceeded)); + let deadline = matches!(outcome, Err(JobError::DeadlineExceeded)); token.with_deadline(Instant::now() + Duration::from_secs(30)); let cancelled = self.job.as_ref().unwrap().is_cancelled(); token.reply((deadline, cancelled)); @@ -773,13 +770,11 @@ mod caller_gone_vs_shortening_threads { impl Handler for Delegator { fn handle(&mut self, _: Ask, reply: PendingReply<'_, (bool, bool)>, ctx: &Context) { let token = reply.defer().unwrap(); - let opts = JobOptions { - cancel_on_caller_gone: true, - ..JobOptions::default() - } - .deadline_after(Duration::from_millis(20)); + let opts = JobOptions::new() + .cancel_on_caller_gone() + .deadline(Duration::from_millis(20)); let job = ctx - .pipe_reply_blocking_with( + .pipe_reply_blocking( opts, token, |_| std::thread::sleep(Duration::from_millis(300)), @@ -809,12 +804,9 @@ mod caller_gone_vs_shortening_threads { impl Handler for Delegator { fn handle(&mut self, _: Leave, reply: PendingReply<'_, ()>, ctx: &Context) { let token = reply.defer().unwrap(); - let opts = JobOptions { - cancel_on_caller_gone: true, - ..JobOptions::default() - }; + let opts = JobOptions::new().cancel_on_caller_gone(); let job = ctx - .pipe_reply_blocking_with( + .pipe_reply_blocking( opts, token, |job| { @@ -836,7 +828,7 @@ mod caller_gone_vs_shortening_threads { impl Handler for Delegator { fn handle(&mut self, msg: Delegated, _: PendingReply<'_, ()>, _: &Context) { let Delegated(outcome, token) = msg; - let deadline = matches!(outcome.into_result(), Err(JobError::DeadlineExceeded)); + let deadline = matches!(outcome, Err(JobError::DeadlineExceeded)); token.with_deadline(Instant::now() + Duration::from_secs(30)); let cancelled = self.job.as_ref().unwrap().is_cancelled(); token.reply((deadline, cancelled)); diff --git a/concurrency/tests/deferred_jobs.rs b/concurrency/tests/deferred_jobs.rs index 0a040c85..d98a2bbf 100644 --- a/concurrency/tests/deferred_jobs.rs +++ b/concurrency/tests/deferred_jobs.rs @@ -21,7 +21,7 @@ mod tasks_mode { use super::*; use spawned_concurrency::tasks::{ Actor, ActorRef, ActorStart, Context, Deadline, Handler, JobError, JobGone, JobOptions, - JobOutcome, PendingReply, PipeErrorKind, PipeReplyError, ReplyTo, + JobOutcome, PendingReply, PipeError, PipeReplyError, ReplyTo, }; use spawned_concurrency::ActorError; use spawned_rt::tasks::{oneshot, sleep}; @@ -78,7 +78,7 @@ mod tasks_mode { } fn describe(name: &str, outcome: JobOutcome) -> Done { - Done(match outcome.into_result() { + Done(match outcome { Ok(v) => format!("{name}:ok:{v:?}"), Err(JobError::Panicked(m)) => format!("{name}:panicked:{m}"), Err(JobError::DeadlineExceeded) => format!("{name}:deadline"), @@ -110,49 +110,59 @@ mod tasks_mode { ) { let result = match msg { Start::Pipe(name, v) => ctx - .pipe(async move { v * 2 }, move |o| describe(name, o)) + .pipe( + JobOptions::new(), + |_| async move { v * 2 }, + move |o| describe(name, o), + ) .map(drop) .map_err(|e| e.to_string()), Start::Panics => ctx - .pipe(async { panic!("job boom") }, |o: JobOutcome<()>| { - describe("p", o) - }) + .pipe( + JobOptions::new(), + |_| async { panic!("job boom") }, + |o: JobOutcome<()>| describe("p", o), + ) .map(drop) .map_err(|e| e.to_string()), Start::Gated(name, deadline) => { let (open, gate) = oneshot::channel::<()>(); self.gates.push(open); - let opts = JobOptions { - deadline: deadline.map(|d| spawned_rt::time::Instant::now() + d), - ..JobOptions::default() - }; - ctx.pipe_with(opts, async move { gate.await.is_ok() }, move |o| { - describe(name, o) - }) + let opts = JobOptions::new() + .deadline(deadline.map_or(Deadline::Never, Deadline::after)); + ctx.pipe( + opts, + |_| async move { gate.await.is_ok() }, + move |o| describe(name, o), + ) .map(drop) .map_err(|e| e.to_string()) } Start::Keyed(name) => { let (open, gate) = oneshot::channel::<()>(); self.gates.push(open); - ctx.pipe_keyed("k", async move { gate.await.is_ok() }, move |o| { - describe(name, o) - }) + ctx.pipe( + JobOptions::new().key("k"), + |_| async move { gate.await.is_ok() }, + move |o| describe(name, o), + ) .map(drop) .map_err(|e| e.to_string()) } Start::CancelAfterSpawn => { let (open, gate) = oneshot::channel::<()>(); self.gates.push(open); - ctx.pipe(async move { gate.await.is_ok() }, |o| { - describe("cancelled", o) - }) + ctx.pipe( + JobOptions::new(), + |_| async move { gate.await.is_ok() }, + |o| describe("cancelled", o), + ) .map(|job| job.cancel()) .map_err(|e| e.to_string()) } Start::Blocking(v) => ctx - .pipe_blocking_with( - JobOptions::default(), + .pipe_blocking( + JobOptions::new(), move |_| v + 1, |o| describe("blocking", o), ) @@ -160,31 +170,26 @@ mod tasks_mode { .map_err(|e| e.to_string()), Start::NativeBudget => { let (hold, held) = std::sync::mpsc::channel::<()>(); - let first = ctx.pipe_blocking_with( - JobOptions::default(), + let first = ctx.pipe_blocking( + JobOptions::new(), move |_| { let _ = held.recv(); }, |o| describe("first", o), ); - let second = ctx.pipe_blocking_with( - JobOptions::default(), - |_| (), - |o| describe("second", o), - ); + let second = + ctx.pipe_blocking(JobOptions::new(), |_| (), |o| describe("second", o)); drop(hold); match (&first, &second) { - (Ok(_), Err(e)) if matches!(e.kind, PipeErrorKind::NativeBudget) => Ok(()), + (Ok(_), Err(PipeError::NativeBudget)) => Ok(()), _ => Err(format!("{first:?} / {second:?}")), } } Start::Emits(name, n) => ctx - .pipe_blocking_with( - JobOptions { - key: Some("emits".into()), - emit_credits: 2, - ..JobOptions::default() - }, + .pipe_blocking( + JobOptions::new() + .key("emits") + .emit_credits(std::num::NonZeroU16::new(2).unwrap()), move |job| { for i in 0..n { job.emit_blocking(Done(format!("{name}:emit:{i}")))?; @@ -273,9 +278,9 @@ mod tasks_mode { ) { let (open, gate) = oneshot::channel::<()>(); let job = ctx - .pipe_keyed( - "r", - async move { gate.await.map(|()| 1).unwrap_or(0) }, + .pipe( + JobOptions::new().key("r"), + |_| async move { gate.await.map(|()| 1).unwrap_or(0) }, |o| describe("old", o), ) .unwrap(); @@ -294,8 +299,12 @@ mod tasks_mode { impl Handler for Owner { async fn handle(&mut self, msg: ReplaceWhen, _: PendingReply<'_, ()>, ctx: &Context) { msg.0.await.unwrap(); - ctx.pipe_keyed("r", async { 2 }, |o| describe("new", o)) - .unwrap(); + ctx.pipe( + JobOptions::new().key("r"), + |_| async { 2 }, + |o| describe("new", o), + ) + .unwrap(); } } @@ -332,9 +341,11 @@ mod tasks_mode { } impl Handler for Owner { async fn handle(&mut self, _: Bomb, reply: PendingReply<'_, ()>, ctx: &Context) { - ctx.pipe(async {}, |_: JobOutcome<()>| -> Done { - panic!("done boom") - }) + ctx.pipe( + JobOptions::new(), + |_| async {}, + |_: JobOutcome<()>| -> Done { panic!("done boom") }, + ) .unwrap(); reply.reply(()); } @@ -480,7 +491,12 @@ mod tasks_mode { Err(e) => Err(e), } }; - if let Err(e) = ctx.pipe_reply(token, work, |res, token| Delegated { res, token }) { + if let Err(e) = ctx.pipe_reply( + JobOptions::new(), + token, + |_| work, + |res, token| Delegated { res, token }, + ) { drop(e.into_token()); } } @@ -488,7 +504,7 @@ mod tasks_mode { impl Handler for Router { async fn handle(&mut self, msg: Delegated, _: PendingReply<'_, ()>, _: &Context) { match msg.res { - JobOutcome::Ok(Ok(v)) => { + Ok(Ok(v)) => { msg.token.reply(v); } _ => msg.token.overloaded(), @@ -512,10 +528,15 @@ mod tasks_mode { reply: PendingReply<'_, bool>, ctx: &Context, ) { - let refused = ctx.pipe_reply(msg.0, async { 1u64 }, |_, token| Delegated { - res: JobOutcome::Ok(Ok(0)), - token, - }); + let refused = ctx.pipe_reply( + JobOptions::new(), + msg.0, + |_| async { 1u64 }, + |_, token| Delegated { + res: Ok(Ok(0)), + token, + }, + ); let wrong_owner = matches!(refused, Err(PipeReplyError::WrongOwner(_))); if let Err(e) = refused { e.into_token().reply(7); @@ -634,8 +655,8 @@ mod threads_mode { fn handle(&mut self, msg: Compute, reply: PendingReply<'_, u64>, ctx: &Context) { let Ok(token) = reply.defer() else { return }; let n = msg.0; - let _ = ctx.pipe_reply_blocking_with( - JobOptions::default(), + let _ = ctx.pipe_reply_blocking( + JobOptions::new(), token, move |job| { job.emit_blocking(Done(format!("computing {n}"))).unwrap(); @@ -650,7 +671,7 @@ mod threads_mode { } impl Handler for Owner { fn handle(&mut self, msg: Computed, _: PendingReply<'_, ()>, _: &Context) { - match msg.res.into_result() { + match msg.res { Ok(v) => { msg.token.reply(v); } diff --git a/concurrency/tests/deferred_sim.rs b/concurrency/tests/deferred_sim.rs index cac04a19..87d035b2 100644 --- a/concurrency/tests/deferred_sim.rs +++ b/concurrency/tests/deferred_sim.rs @@ -96,13 +96,13 @@ impl Message for StartKeyed { impl Handler for Owner { async fn handle(&mut self, msg: StartKeyed, _: PendingReply<'_, ()>, ctx: &Context) { let StartKeyed(name, delay) = msg; - ctx.pipe_keyed( - "k", - async move { + ctx.pipe( + JobOptions::new().key("k"), + |_| async move { sleep(Duration::from_millis(delay)).await; name }, - |outcome: JobOutcome<&'static str>| Done(outcome.into_result().unwrap()), + |outcome: JobOutcome<&'static str>| Done(outcome.unwrap()), ) .unwrap(); push(&self.log, format!("start:{name}")); @@ -135,12 +135,9 @@ impl Message for StartEmitter { } impl Handler for Owner { async fn handle(&mut self, _: StartEmitter, _: PendingReply<'_, ()>, ctx: &Context) { - let opts = JobOptions { - emit_credits: 1, - ..Default::default() - }; + let opts = JobOptions::new().emit_credits(std::num::NonZeroU16::new(1).unwrap()); let job = ctx - .pipe_with_ctx( + .pipe( opts, |job: JobCtx| async move { for i in 0..3 { @@ -151,7 +148,7 @@ impl Handler for Owner { } "emitter" }, - |outcome: JobOutcome<&'static str>| Done(outcome.into_result().unwrap()), + |outcome: JobOutcome<&'static str>| Done(outcome.unwrap()), ) .unwrap(); self.job = Some(job); diff --git a/concurrency/tests/job_trace.rs b/concurrency/tests/job_trace.rs index 37cfd2e5..0404bd22 100644 --- a/concurrency/tests/job_trace.rs +++ b/concurrency/tests/job_trace.rs @@ -74,7 +74,7 @@ impl Handler for Owner { fn done(name: &'static str) -> impl FnOnce(JobOutcome) -> Done { move |outcome| { - Done(match outcome.into_result() { + Done(match outcome { Ok(v) => format!("{name}:ok:{v:?}"), Err(e) => format!("{name}:{e}"), }) @@ -99,25 +99,23 @@ impl Handler for Owner { async fn handle(&mut self, msg: Run, reply: PendingReply<'_, ()>, ctx: &Context) { match msg.0 { Scenario::Plain => { - ctx.pipe(async { 1u32 }, done("plain")).unwrap(); + ctx.pipe(JobOptions::new(), |_| async { 1u32 }, done("plain")) + .unwrap(); } Scenario::KeyedFenced => { let (open, gate) = oneshot::channel::<()>(); let old = ctx - .pipe_keyed( - "k", - async move { gate.await.map(|()| 1u32).unwrap_or(0) }, + .pipe( + JobOptions::new().key("k"), + |_| async move { gate.await.map(|()| 1u32).unwrap_or(0) }, done("old"), ) .unwrap(); *self.old.lock().unwrap() = Some((old, open)); } Scenario::Emits => { - let opts = JobOptions { - emit_credits: 1, - ..Default::default() - }; - ctx.pipe_with_ctx( + let opts = JobOptions::new().emit_credits(std::num::NonZeroU16::new(1).unwrap()); + ctx.pipe( opts, |job: JobCtx| async move { for i in 0..3 { @@ -135,7 +133,8 @@ impl Handler for Owner { let (open, gate) = oneshot::channel::<()>(); let handle = ctx .pipe( - async move { gate.await.map(|()| 1u32).unwrap_or(0) }, + JobOptions::new(), + |_| async move { gate.await.map(|()| 1u32).unwrap_or(0) }, done("gated"), ) .unwrap(); @@ -143,13 +142,10 @@ impl Handler for Owner { self.gates.push(open); } Scenario::Deadline => { - let opts = JobOptions { - deadline: Some(spawned_rt::time::Instant::now() + Duration::from_millis(30)), - ..Default::default() - }; - ctx.pipe_with( + let opts = JobOptions::new().deadline(Duration::from_millis(30)); + ctx.pipe( opts, - async { + |_| async { sleep(Duration::from_millis(120)).await; 1u32 }, @@ -158,13 +154,10 @@ impl Handler for Owner { .unwrap(); } Scenario::Blocking => { - let opts = JobOptions { - emit_credits: 2, - ..Default::default() - }; - ctx.pipe_blocking_with( + let opts = JobOptions::new().emit_credits(std::num::NonZeroU16::new(2).unwrap()); + ctx.pipe_blocking( opts, - |job: &JobCtx| { + |job: JobCtx| { let _ = job.emit_blocking(Emitted(0)); let _ = job.emit_blocking(Emitted(1)); 5u32 @@ -189,7 +182,8 @@ impl Message for ReplaceWhen { impl Handler for Owner { async fn handle(&mut self, msg: ReplaceWhen, _: PendingReply<'_, ()>, ctx: &Context) { msg.0.await.unwrap(); - ctx.pipe_keyed("k", async { 2u32 }, done("new")).unwrap(); + ctx.pipe(JobOptions::new().key("k"), |_| async { 2u32 }, done("new")) + .unwrap(); } } diff --git a/concurrency/tests/metrics_deferred.rs b/concurrency/tests/metrics_deferred.rs index 163faa4b..d9ee05df 100644 --- a/concurrency/tests/metrics_deferred.rs +++ b/concurrency/tests/metrics_deferred.rs @@ -96,19 +96,19 @@ impl Message for Jobs { impl Handler for Owner { async fn handle(&mut self, msg: Jobs, reply: PendingReply<'_, ()>, ctx: &Context) { match msg { - Jobs::Ok => drop(ctx.pipe(async { 1 }, done).unwrap()), - Jobs::Panics => drop(ctx.pipe(async { panic!("job") }, done::<()>).unwrap()), + Jobs::Ok => drop(ctx.pipe(JobOptions::new(), |_| async { 1 }, done).unwrap()), + Jobs::Panics => drop( + ctx.pipe(JobOptions::new(), |_| async { panic!("job") }, done::<()>) + .unwrap(), + ), Jobs::Deadline => { - let opts = JobOptions { - deadline: Some(spawned_rt::time::Instant::now() + Duration::from_millis(10)), - ..Default::default() - }; - ctx.pipe_with(opts, sleep(Duration::from_millis(60)), done) + let opts = JobOptions::new().deadline(Duration::from_millis(10)); + ctx.pipe(opts, |_| sleep(Duration::from_millis(60)), done) .unwrap(); } Jobs::Emits => { - ctx.pipe_with_ctx( - JobOptions::default(), + ctx.pipe( + JobOptions::new(), |job: JobCtx| async move { job.emit(Done).await.unwrap(); job.emit(Done).await.unwrap(); @@ -119,7 +119,7 @@ impl Handler for Owner { } Jobs::Fenced => { let job = ctx - .pipe_blocking_with(JobOptions::default(), |_: &JobCtx| 1, done) + .pipe_blocking(JobOptions::new(), |_: JobCtx| 1, done) .unwrap(); while job.is_running() { std::thread::sleep(Duration::from_millis(1)); @@ -128,20 +128,26 @@ impl Handler for Owner { } Jobs::Budget => { let (open, gate) = std::sync::mpsc::channel::<()>(); - ctx.pipe_blocking_with( - JobOptions::default(), - move |_: &JobCtx| gate.recv().ok(), + ctx.pipe_blocking( + JobOptions::new(), + move |_: JobCtx| gate.recv().ok(), done, ) .unwrap(); assert!(ctx - .pipe_blocking_with(JobOptions::default(), |_: &JobCtx| (), done) + .pipe_blocking(JobOptions::new(), |_: JobCtx| (), done) .is_err()); open.send(()).unwrap(); } Jobs::Cancel => { let (open, gate) = oneshot::channel::<()>(); - let job = ctx.pipe(async move { gate.await.is_ok() }, done).unwrap(); + let job = ctx + .pipe( + JobOptions::new(), + |_| async move { gate.await.is_ok() }, + done, + ) + .unwrap(); job.cancel(); self.gates.push(open); } diff --git a/concurrency/tests/native_jobs.rs b/concurrency/tests/native_jobs.rs index e2f0fa6b..f8599c0a 100644 --- a/concurrency/tests/native_jobs.rs +++ b/concurrency/tests/native_jobs.rs @@ -31,8 +31,8 @@ impl Actor for Gated { async fn started(&mut self, ctx: &Context) { let gate = self.gate.lock().unwrap().take().unwrap(); let running = self.running.clone(); - ctx.pipe_blocking_with( - JobOptions::default(), + ctx.pipe_blocking( + JobOptions::new(), move |_| { running.store(true, Ordering::SeqCst); let _ = gate.recv(); @@ -120,8 +120,8 @@ async fn a_supervisor_does_not_replace_a_child_whose_blocking_job_still_runs() { if let Some(gated) = &mut self.0 { let gate = gated.gate.lock().unwrap().take().unwrap(); let running = gated.running.clone(); - ctx.pipe_blocking_with( - JobOptions::default(), + ctx.pipe_blocking( + JobOptions::new(), move |_| { running.store(true, Ordering::SeqCst); let _ = gate.recv(); @@ -193,7 +193,7 @@ async fn a_supervisor_does_not_replace_a_child_whose_blocking_job_still_runs() { } /// Review P1 (Codex 6): job admission is one step with the owner's exit. -/// `pipe_with_ctx` builds the work (user code) between its exit check and +/// `pipe` builds the work (user code) between its exit check and /// the admission; an owner that exits meanwhile refuses the job, so it /// never runs. It used to be admitted into the job set the exit had already /// drained: uncancelled work outliving its owner. @@ -211,8 +211,8 @@ async fn a_job_admitted_after_its_owner_exited_never_runs() { let ran = Arc::new(AtomicBool::new(false)); let work_ran = ran.clone(); let handle = tokio::task::spawn_blocking(move || { - ctx.pipe_with_ctx( - JobOptions::default(), + ctx.pipe( + JobOptions::new(), |job| { // While the work is built, the owner exits and dies. owner.context().stop(); @@ -263,7 +263,10 @@ async fn an_unused_permit_does_not_hold_the_scope(borrowed: bool) { let actor = Owner.start_on(&group).unwrap(); actor.ready().await.unwrap(); // Reserved through a cloned context and kept outside the actor, unused. - let permit: BlockingJobPermit = actor.context().reserve_blocking_job().unwrap(); + let permit: BlockingJobPermit = actor + .context() + .reserve_blocking_job(JobOptions::new()) + .unwrap(); actor.context().stop(); assert_eq!( actor.wait_exit().await, @@ -275,7 +278,7 @@ async fn an_unused_permit_does_not_hold_the_scope(borrowed: bool) { "an unused blocking permit kept the dead actor's scope owned" ); // Using it now is refused: nothing runs. - let handle = permit.pipe_blocking_with(JobOptions::default(), |_| (), |_| Done); + let handle = permit.pipe_blocking(|_| (), |_| Done); assert!(handle.is_cancelled()); } @@ -309,7 +312,7 @@ fn a_failed_job_thread_spawn_is_a_reservation_error() { #[ignore = "subprocess helper exercised by the enabled parent"] fn job_thread_spawn_failure_child() { use nix::sys::resource::{getrlimit, setrlimit, Resource}; - use spawned_concurrency::tasks::{ActorStart, Backend, PipeErrorKind}; + use spawned_concurrency::tasks::{ActorStart, Backend, PipeError}; if std::env::var_os("SPAWNED_MAILBOX_DROP_CHILD").is_none() { return; } @@ -333,19 +336,16 @@ fn job_thread_spawn_failure_child() { println!("job thread spawn failure reported (RLIMIT_NPROC not enforced)"); return; } - let reserved = ctx.reserve_blocking_job(); - let piped = ctx.pipe_blocking_with(JobOptions::default(), |_| (), |_| Done); + let reserved = ctx.reserve_blocking_job(JobOptions::new()); + let piped = ctx.pipe_blocking(JobOptions::new(), |_| (), |_| Done); setrlimit(Resource::RLIMIT_NPROC, soft, hard).unwrap(); match reserved { - Err(error) => assert!( - matches!(error.kind, PipeErrorKind::SpawnFailed(_)), - "{error}" - ), + Err(error) => assert!(matches!(error, PipeError::SpawnFailed(_)), "{error}"), Ok(_) => panic!("the reservation succeeded without a job thread"), } assert!( - matches!(piped, Err(ref e) if matches!(e.kind, PipeErrorKind::SpawnFailed(_))), - "pipe_blocking_with returned {piped:?}" + matches!(piped, Err(ref e) if matches!(e, PipeError::SpawnFailed(_))), + "pipe_blocking returned {piped:?}" ); assert_eq!( ctx.deferred_outstanding(), @@ -353,7 +353,9 @@ fn job_thread_spawn_failure_child() { "a refused job kept its credit" ); // With threads available again, the same call runs the job. - let permit = ctx.reserve_blocking_job().expect("a job thread"); + let permit = ctx + .reserve_blocking_job(JobOptions::new()) + .expect("a job thread"); drop(permit); ctx.stop(); std::thread::sleep(Duration::from_millis(50)); diff --git a/concurrency/tests/ui-pass/job_api.rs b/concurrency/tests/ui-pass/job_api.rs new file mode 100644 index 00000000..03423bf5 --- /dev/null +++ b/concurrency/tests/ui-pass/job_api.rs @@ -0,0 +1,71 @@ +// The 0.8 job API: one call per shape, options first, the work receives +// its `JobCtx`, `JobOutcome` is a `Result`, `PipeError` is the enum. +use spawned_concurrency::message::Message; +use spawned_concurrency::tasks::{ + Actor, Context, Deadline, Handler, JobCtx, JobError, JobOptions, JobOutcome, PendingReply, + PipeError, PipeReplyError, ReplyTo, +}; +use spawned_concurrency::Instant; +use std::num::NonZeroU16; +use std::time::Duration; + +struct Owner; +impl Actor for Owner {} + +struct Done(JobOutcome); +impl Message for Done { + type Result = (); +} +struct Answered(JobOutcome, ReplyTo); +impl Message for Answered { + type Result = (); +} + +impl Handler for Owner { + async fn handle(&mut self, msg: Done, _: PendingReply<'_, ()>, _: &Context) { + // A plain `Result`: `?`-style handling and wildcard error arms. + let _value: Result = msg.0; + } +} +impl Handler for Owner { + async fn handle(&mut self, msg: Answered, _: PendingReply<'_, ()>, _: &Context) { + let Answered(outcome, token) = msg; + let _ = token.reply(outcome.unwrap_or(0)); + } +} + +fn spawn_all(ctx: &Context, token: ReplyTo) -> Result<(), PipeError> { + let credits = NonZeroU16::new(4).unwrap(); + ctx.pipe(JobOptions::new(), |_| async { 1 }, Done)?; + ctx.pipe( + JobOptions::new().key("k").emit_credits(credits), + |job: JobCtx| async move { u32::from(job.is_cancelled()) }, + Done, + )?; + ctx.pipe_blocking(JobOptions::new().deadline(Duration::from_secs(1)), |_| 2, Done)?; + ctx.pipe_blocking(JobOptions::new().deadline(Instant::now()), |_| 3, Done)?; + ctx.pipe_blocking(JobOptions::new().deadline(Deadline::Never), |_| 4, Done)?; + if let Ok(permit) = ctx.reserve_job(JobOptions::new()) { + permit.pipe(|_| async { 5 }, Done); + } + let permit = ctx.reserve_blocking_job(JobOptions::default())?; + permit.pipe_blocking(|job| u32::from(job.is_cancelled()), Done); + match ctx.pipe_reply( + JobOptions::new().cancel_on_caller_gone(), + token, + |_| async { 6 }, + Answered, + ) { + Ok(_) => {} + Err(PipeReplyError::Failed { token, error }) => { + let _ = matches!(error, PipeError::Overloaded | PipeError::NativeBudget); + let _ = ctx.pipe_reply_blocking(JobOptions::new(), token, |_| 7, Answered); + } + Err(other) => drop(other.into_token()), + } + Ok(()) +} + +fn main() { + let _ = spawn_all; +} diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 23d8cadb..ac34de50 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -641,17 +641,17 @@ reports exactly one outcome per object even when the actor is at a bound: #[send_handler] async fn on_gossip(&mut self, msg: Gossip, ctx: &Context) { // Reserve first: on refusal nothing has been moved into the job yet. - let Ok(permit) = ctx.reserve_blocking_job() else { + let opts = JobOptions::new().deadline(VERDICT_DEADLINE); + let Ok(permit) = ctx.reserve_blocking_job(opts) else { return self.report(msg.id, Verdict::Ignore(Overloaded)); }; let Gossip { id, object } = msg; - let opts = JobOptions::default().deadline_after(VERDICT_DEADLINE); - permit.pipe_blocking_with(opts, move |_job| check(&object), move |res| Checked { id, res }); + permit.pipe_blocking(move |_job| check(&object), move |res| Checked { id, res }); } #[send_handler] async fn on_checked(&mut self, msg: Checked, _ctx: &Context) { - let verdict = match msg.res.into_result() { + let verdict = match msg.res { Ok(outcome) => outcome, Err(JobError::DeadlineExceeded) => Verdict::Ignore(TimedOut), Err(_) => Verdict::Reject(CheckFailed), // a panic in `check` @@ -662,21 +662,28 @@ async fn on_checked(&mut self, msg: Checked, _ctx: &Context) { // Delegation: the obligation and its credit stay here. let token = reply.defer()?; let downstream = other.request_raw_until(Lookup(key), token.deadline().map_or(Deadline::Never, Deadline::At)); -ctx.pipe_reply(token, async move { downstream?.await }, |res, token| Delegated { res, token })?; +ctx.pipe_reply(JobOptions::new(), token, |_| async move { downstream?.await }, |res, token| Delegated { res, token })?; ``` -- `done(outcome)` runs in the owner's turn; `JobOutcome::Err` carries - `JobError::Panicked` or `JobError::DeadlineExceeded`. `JobOutcome`, - `JobError` and `PipeErrorKind` are `#[non_exhaustive]`: match with a - wildcard arm (or `outcome.into_result()`). +- Four calls, one per shape: `ctx.pipe` (async work), `ctx.pipe_blocking`, + `ctx.pipe_reply` and `ctx.pipe_reply_blocking` (linked to a deferred + reply). Each takes `JobOptions` first, built with `JobOptions::new()` and + `.key(k)`, `.deadline(d)` (a `Duration`, `Instant` or `Deadline`), + `.emit_credits(n)` and, for `pipe_reply*` only, + `.cancel_on_caller_gone()`. The work receives the job's `JobCtx` by + value (`|_| fut` when it needs neither cancellation nor emits). +- `done(outcome)` runs in the owner's turn. `JobOutcome` is + `Result`; `Err` carries `JobError::Panicked` or + `JobError::DeadlineExceeded`. `JobError` and `PipeError` are + `#[non_exhaustive]`: match with a wildcard arm. - `ctx.pipe*` reject at once at a bound (`PipeError`, never waiting) and drop `work` and `done`. When the caller must still produce a result, - `ctx.reserve_job()` / `ctx.reserve_blocking_job()` take the admission - first (owner running, a credit, and for blocking jobs a native slot) and - return a permit whose `pipe*` cannot fail. `pipe_reply*` errors hand the - token back. -- `pipe_keyed` / `JobOptions::key` cancel the previous job with that key and - fence its queued messages; `JobHandle::cancel` fences one job. + `ctx.reserve_job(opts)` / `ctx.reserve_blocking_job(opts)` take the + admission first (owner running, a credit, and for blocking jobs a native + slot) and return a permit whose `pipe` / `pipe_blocking` cannot fail. + `pipe_reply*` errors hand the token back. +- `JobOptions::key` cancels the previous job with that key and + fences its queued messages; `JobHandle::cancel` fences one job. - `JobCtx::emit_blocking` (and `emit` for async jobs) sends fenced messages, bounded by `JobOptions::emit_credits`; the terminal message is always last. `JobCtx::emit_at(at, msg)` (`emit_at_blocking` in blocking jobs) delivers @@ -689,8 +696,7 @@ ctx.pipe_reply(token, async move { downstream?.await }, |res, token| Delegated { deadline passes first it takes the notification and the emit is dropped at its instant (if the job finishes first, the emit and then the result arrive as usual). -- **Job deadlines are hard.** When `JobOptions::deadline` (or - `deadline_after(d)`) passes, `done` gets `DeadlineExceeded` at once, later +- **Job deadlines are hard.** When `JobOptions::deadline` passes, `done` gets `DeadlineExceeded` at once, later emits fail with `JobGone` and the work's eventual result is dropped. The work itself keeps running (cooperative cancellation) and keeps its credit and native slot until it returns. For a **soft deadline** (stop starting @@ -699,7 +705,7 @@ ctx.pipe_reply(token, async move { downstream?.await }, |res, token| Delegated { ```rust let stop_starting_at = Instant::now() + SOFT_DEADLINE; - ctx.pipe_blocking_with(JobOptions::default(), move |job| { + ctx.pipe_blocking(JobOptions::new(), move |job| { let mut done = Vec::new(); for item in batch { if job.is_cancelled() || Instant::now() >= stop_starting_at { break; } @@ -722,7 +728,7 @@ ctx.pipe_reply(token, async move { downstream?.await }, |res, token| Delegated { > actor is easy to outgrow now that blocking jobs run on Tokio's blocking > pool: an actor that verifies, proves or does I/O in blocking jobs > usually needs tens or hundreds (one client port needed 352). Under load - > the only symptom is `PipeErrorKind::NativeBudget` refusals; the first one + > the only symptom is `PipeError::NativeBudget` refusals; the first one > per actor instance is logged as a `native_budget_exhausted` warning. Set both in the macro, @@ -733,8 +739,8 @@ ctx.pipe_reply(token, async move { downstream?.await }, |res, token| Delegated { blocking jobs on Tokio's blocking pool (`spawn_blocking` of the actor's group runtime, or the current runtime), still bounded by the native budget; a runtime that is dropped waits for blocking jobs still running, - as with any `spawn_blocking`. `threads` mode offers `pipe_blocking_with` - and `pipe_reply_blocking_with` on owned native threads (a reservation + as with any `spawn_blocking`. `threads` mode offers `pipe_blocking` + and `pipe_reply_blocking` on owned native threads (a reservation spawns the thread, parked until the job starts). - Owner exit cancels every job; after death the jobs are joined, bounded by the cleanup deadline, and stragglers are reported (`unfinished_at_stop`). diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 1664435a..629250dc 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -5,6 +5,32 @@ every change; this guide covers what you need to edit when upgrading. ## Unreleased +### 0.7 to 0.8: API call-site table + +0.8 is a breaking release. Every renamed or reshaped call is listed here +(old call, new call); the sections below explain the semantics. Rows marked +*(0.8 pre-release)* changed an API that only existed on the 0.8 branch +(ported clients used it), not in 0.7. + +| Old call | New call | +|---|---| +| `ctx.pipe(work_future, done)` *(0.8 pre-release)* | `ctx.pipe(JobOptions::new(), \|_\| work_future, done)` | +| `ctx.pipe_with(opts, work_future, done)` *(0.8 pre-release)* | `ctx.pipe(opts, \|_\| work_future, done)` | +| `ctx.pipe_with_ctx(opts, \|job\| fut, done)` *(0.8 pre-release)* | `ctx.pipe(opts, \|job\| fut, done)` | +| `ctx.pipe_keyed(key, work_future, done)` *(0.8 pre-release)* | `ctx.pipe(JobOptions::new().key(key), \|_\| work_future, done)` | +| `ctx.pipe_blocking_with(opts, \|job: &JobCtx<_>\| .., done)` *(0.8 pre-release)* | `ctx.pipe_blocking(opts, \|job: JobCtx<_>\| .., done)` (the job context is passed by value) | +| `ctx.pipe_reply(token, work_future, done)` *(0.8 pre-release)* | `ctx.pipe_reply(JobOptions::new(), token, \|_\| work_future, done)` | +| `ctx.pipe_reply_with(opts, token, work_future, done)` *(0.8 pre-release)* | `ctx.pipe_reply(opts, token, \|_\| work_future, done)` | +| `ctx.pipe_reply_blocking_with(opts, token, work, done)` *(0.8 pre-release)* | `ctx.pipe_reply_blocking(opts, token, work, done)` | +| `ctx.reserve_job()?.pipe_with(opts, work, done)` *(0.8 pre-release)* | `ctx.reserve_job(opts)?.pipe(\|_\| work, done)` | +| `ctx.reserve_blocking_job()?.pipe_blocking_with(opts, work, done)` *(0.8 pre-release)* | `ctx.reserve_blocking_job(opts)?.pipe_blocking(work, done)` | +| `JobOptions { deadline: Some(at), key: Some(k), emit_credits: n, cancel_on_caller_gone: true }` *(0.8 pre-release)* | `JobOptions::new().deadline(at).key(k).emit_credits(NonZeroU16::new(n).unwrap()).cancel_on_caller_gone()` | +| `JobOptions::default().deadline_after(d)` *(0.8 pre-release)* | `JobOptions::new().deadline(d)` (a `Duration`, an `Instant` or a `Deadline`) | +| `JobOutcome::Ok(v)` / `JobOutcome::Err(e)` / `outcome.into_result()` *(0.8 pre-release)* | `Ok(v)` / `Err(e)` / `outcome` (`JobOutcome` is `Result`) | +| `PipeError { kind: PipeErrorKind::Overloaded }`, `e.kind` *(0.8 pre-release)* | `PipeError::Overloaded`, `e` | +| `PipeReplyError::Failed { token, kind }` *(0.8 pre-release)* | `PipeReplyError::Failed { token, error }` (`#[non_exhaustive]`) | +| `DEFAULT_EMIT_CREDITS`, `default_max_native_jobs()` *(0.8 pre-release)* | removed from the public API (the defaults, 16 and `min(4, cores)`, are documented) | + ### Exit signals, startup and runtime drops as in OTP (ranks 5e, 5f, 5g, targets 0.8, in progress) Design: [otp-exit-signals.md](design/otp-exit-signals.md) (its migration @@ -194,32 +220,41 @@ that builds a `Snapshot` with a struct literal must add them. **Jobs (API refinements).** -- `JobOutcome`, `JobError` and `PipeErrorKind` are `#[non_exhaustive]`: add - a wildcard arm, or match `outcome.into_result()` with an `Err(_)` arm. -- `tasks` mode: `pipe_blocking_with` / `pipe_reply_blocking_with` run on - Tokio's blocking pool. Remove a `pipe_with(.., async { spawn_blocking(f) +- One spawn per shape: `pipe` (async), `pipe_blocking`, `pipe_reply`, + `pipe_reply_blocking`, each taking `JobOptions` first (as OTP's `*_opt` + calls take an option list). The work of every form receives its + `JobCtx` (by value; `|_| fut` when unused). See the table above. +- `JobOptions` is `#[non_exhaustive]`: build it with `JobOptions::new()` + and the `key`, `deadline`, `emit_credits` and `cancel_on_caller_gone` + methods. `deadline` takes a `Duration`, an `Instant` or a `Deadline`; + `emit_credits` takes a `NonZeroU16`. `cancel_on_caller_gone` applies to + `pipe_reply*` only (a `debug_assert!` in the plain forms). +- `JobOutcome` is `Result`: `?` and the `Result` + combinators work, `into_result()` is gone. `JobError` and `PipeError` + (now the enum itself, without the `kind` wrapper) are + `#[non_exhaustive]`: add a wildcard arm. +- `tasks` mode: `pipe_blocking` / `pipe_reply_blocking` run on + Tokio's blocking pool. Remove a `pipe(.., |_| async { spawn_blocking(f) .await })` workaround (and its `resume_unwind` of the join error): - `pipe_blocking_with(opts, move |_job| f(), done)` reports a panic as + `pipe_blocking(opts, move |_job| f(), done)` reports a panic as `JobError::Panicked`. **Size the native budget:** the default is `min(4, available_parallelism)` blocking jobs per actor, easy to outgrow (one client port needed 352), and under load its only symptom is - `PipeErrorKind::NativeBudget` refusals (the first one per actor is logged + `PipeError::NativeBudget` refusals (the first one per actor is logged as `native_budget_exhausted`). Raise it with `#[actor(max_native_jobs = N)]` (or `ChildSpec::max_native_jobs(N)` for a supervised child) instead of avoiding native jobs. A Tokio runtime being dropped now waits for blocking jobs still running, as for any `spawn_blocking`. - To keep "exactly one result" when a job is refused, reserve first instead - of cloning ahead: `let Ok(permit) = ctx.reserve_blocking_job() else { + of cloning ahead: `let Ok(permit) = ctx.reserve_blocking_job(opts) else { /* report with the values you still own */ }; - permit.pipe_blocking_with(opts, work, done);` (`reserve_job` for async + permit.pipe_blocking(work, done);` (`reserve_job` for async jobs). Spawning from a permit cannot fail. - A job that re-scheduled its messages with `send_after` (unfenced) can use `job.emit_at(at, msg)` / `emit_at_blocking`: fenced with the job, counted against its emit credits, and the terminal message waits for it. -- `JobOptions::default().deadline_after(d)` replaces - `JobOptions { deadline: Some(spawned_rt::time::Instant::now() + d), .. }`; - use `spawned_concurrency::Instant` instead of `spawned_rt::time::Instant`. +- Use `spawned_concurrency::Instant` instead of `spawned_rt::time::Instant`. Job deadlines are hard (later emits fail, the result is dropped); for a soft deadline, check the time in the work (see the GUIDE). diff --git a/docs/design/deferred-replies-client-mapping.md b/docs/design/deferred-replies-client-mapping.md index e9f4c413..3dcf487d 100644 --- a/docs/design/deferred-replies-client-mapping.md +++ b/docs/design/deferred-replies-client-mapping.md @@ -203,11 +203,11 @@ Gaps: ### After (v1, before the rank 11b pool) ```rust -let opts = JobOptions { deadline: Some(now + VERDICT_DEADLINE), ..Default::default() }; -match ctx.pipe_blocking_with(opts, move || object.stateful_checks(&store).map(|o| (o, object)), - move |res| GossipVerdict { id, res }) { +let opts = JobOptions::new().deadline(VERDICT_DEADLINE); +match ctx.pipe_blocking(opts, move |_| object.stateful_checks(&store).map(|o| (o, object)), + move |res| GossipVerdict { id, res }) { Ok(_job) => {} - Err(PipeError { kind: PipeErrorKind::Overloaded | PipeErrorKind::NativeBudget }) => { + Err(PipeError::Overloaded | PipeError::NativeBudget) => { report(server, id, Ignore(Overloaded)); forward(..); } Err(e) => { report(server, id, Ignore(Internal)); warn!(?e); } @@ -215,9 +215,9 @@ match ctx.pipe_blocking_with(opts, move || object.stateful_checks(&store).map(|o impl Handler for P2PServer { async fn handle(&mut self, msg: GossipVerdict, _: PendingReply<'_, ()>, _: &Context) { let outcome = match msg.res { - JobOutcome::Ok((outcome, _object)) => outcome, - JobOutcome::Err(JobError::Panicked(_)) => Ignore(Internal), - JobOutcome::Err(JobError::DeadlineExceeded) => Ignore(Timeout), // new: hung checks still yield + Ok((outcome, _object)) => outcome, + Err(JobError::DeadlineExceeded) => Ignore(Timeout), // new: hung checks still yield + Err(_) => Ignore(Internal), // a panic }; self.settle(msg.id, outcome, ..); } @@ -274,11 +274,10 @@ Hazards: ```rust fn start_aggregation_session(&mut self, slot: u64, ctx: &Context) -> Result<(), PipeError> { let snapshot = self.snapshot_for(slot); - let opts = JobOptions { key: Some(AGGREGATION), deadline: Some(interval_2_boundary(slot)), - ..Default::default() }; + let opts = JobOptions::new().key(AGGREGATION).deadline(interval_2_boundary(slot)); // Keyed: cancels the prior session's job and fences its queued messages; // no join in the turn (the job set joins at stop, with a deadline). - ctx.pipe_blocking_with(opts, move |job: &JobCtx| { + ctx.pipe_blocking(opts, move |job: JobCtx| { for proof in snapshot.jobs() { if job.is_cancelled() { break } job.emit_blocking(AggregateProduced { slot, proof: prove(proof) })?; // fenced per emit @@ -317,7 +316,7 @@ Also: `PendingReplies` (sections 1 and 2). Expired requests are still dispatched (OTP), so handlers check `reply.is_closed()` or rely on `defer()` returning `Closed`. -- `pipe_blocking_with`, `JobOptions { key, deadline }` and +- `pipe_blocking`, `JobOptions::new().key(..).deadline(..)` and `JobCtx::emit_blocking` (sections 3 and 4). - Rank 11b for pool-based admission classes (section 3). - Ethrex is on spawned 0.5.0. These rewrites assume it first migrates to v0.7 diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index 3736e88f..1e415eb3 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -32,8 +32,8 @@ See [Differences from OTP](#differences-from-otp). | `handle_call` returns `{noreply, State}` and keeps `From` | the handler receives a `PendingReply<'_, R>`; `defer()` gives a storable `ReplyTo` | | `gen_server:reply(From, R)` | `reply_to.reply(value)` | | `handle_cast` | the same handler, reached by `send`; the `PendingReply` is detached | -| delegating with `From` | `ctx.pipe_reply(token, other.request(msg), ..)`; the obligation stays here | -| `Task.Supervisor.async_nolink` + `handle_info` | `ctx.pipe(work, done)`; the result or a caught panic arrives as one message | +| delegating with `From` | `ctx.pipe_reply(opts, token, \|_\| other.request(msg), ..)`; the obligation stays here | +| `Task.Supervisor.async_nolink` + `handle_info` | `ctx.pipe(opts, work, done)`; the result or a caught panic arrives as one message | | `gen_server:call/3` timeout (default 5000, or `infinity`) | one deadline fixed at send (default 5 s, or `Deadline::Never`) | | a call times out; the server still handles it; the late reply is dropped | the same: expired requests are dispatched, and a late reply is a no-op | | call monitors the server and fails when it dies | the owner's death discharges every token it holds | @@ -472,7 +472,7 @@ There is no `ReplyTo::forward` in v1. ```rust let token = reply.defer()?; -ctx.pipe_reply(token, other.request(msg), |res, token| Delegated { res, token })?; +ctx.pipe_reply(JobOptions::new(), token, |_| other.request(msg), |res, token| Delegated { res, token })?; ``` The downstream request is an ordinary request with the destination's own @@ -550,32 +550,35 @@ otherwise `PipeReplyError::WrongOwner` hands the token back. ## 9. Pipe-to-self jobs ```rust -pub struct JobOptions { - pub deadline: Option, // none: no job deadline - pub key: Option, // cancel-previous - pub cancel_on_caller_gone: bool, // pipe_reply only; default false (OTP) - pub emit_credits: u16, // default 16 +#[non_exhaustive] +pub struct JobOptions { /* private */ } // OTP-style option list, passed first +impl JobOptions { + pub fn new() -> Self; // = default() + pub fn key(self, key: impl Into) -> Self; // cancel-previous + pub fn deadline(self, d: impl Into) -> Self; // Duration, Instant or Deadline + pub fn emit_credits(self, n: NonZeroU16) -> Self; // default 16 + pub fn cancel_on_caller_gone(self) -> Self; // pipe_reply* only; off by default (OTP) } -// Spawn at once; options are passed first, never chained afterwards. -ctx.pipe(work, done) -> Result -ctx.pipe_with(opts, work, done) -> Result -ctx.pipe_blocking_with(opts, |job| .., done) -> Result -ctx.pipe_reply(token, work, done) -> Result> -ctx.pipe_reply_with(opts, token, work, done) -> Result> -ctx.pipe_reply_blocking_with(opts, token, |job| .., done) -> Result> +// One call per shape; spawn at once. `work` receives the job's `JobCtx`. +ctx.pipe(opts, |job| async { .. }, done) -> Result +ctx.pipe_blocking(opts, |job| .., done) -> Result +ctx.pipe_reply(opts, token, |job| async { .. }, done) -> Result> +ctx.pipe_reply_blocking(opts, token, |job| .., done) -> Result> // Reserve-then-spawn: the admission checks (owner running, credit, native // slot) before `work`/`done` exist; spawning from the permit cannot fail. -ctx.reserve_job() -> Result, PipeError> // .pipe/.pipe_with/.pipe_with_ctx -ctx.reserve_blocking_job() -> Result, PipeError> // .pipe_blocking_with +ctx.reserve_job(opts) -> Result, PipeError> // .pipe(work, done) +ctx.reserve_blocking_job(opts) -> Result, PipeError> // .pipe_blocking(work, done) -pub enum PipeErrorKind { Overloaded, NativeBudget, SpawnFailed(io::Error), OwnerStopping } -pub struct PipeError { pub kind: PipeErrorKind } +#[non_exhaustive] +pub enum PipeError { Overloaded, NativeBudget, SpawnFailed(io::Error), OwnerStopping } +#[non_exhaustive] pub enum PipeReplyError { - Failed { token: ReplyTo, kind: PipeErrorKind }, + Failed { token: ReplyTo, error: PipeError }, Closed(ReplyTo), WrongOwner(ReplyTo), } -pub enum JobOutcome { Ok(T), Err(JobError) } +pub type JobOutcome = Result; +#[non_exhaustive] pub enum JobError { Panicked(String), DeadlineExceeded } impl JobCtx { pub fn is_cancelled(&self) -> bool; @@ -586,8 +589,6 @@ impl JobCtx { pub async fn emit_at(&self, at: Instant, msg: M) -> Result<(), JobGone> where A: Handler; pub fn emit_at_blocking(&self, at: Instant, msg: M) -> Result<(), JobGone> where A: Handler; } -// `JobOutcome`, `JobError` and `PipeErrorKind` are #[non_exhaustive]; -// `JobOptions::deadline_after(d)` sets `deadline` to now + d. ``` **Failure disposition.** On every error no job is spawned, and any credit @@ -660,7 +661,7 @@ the owner's `ActorId`. The run loop drops it before the handler runs if: `ActorId`, not a per-slot supervisor generation); or - the job is cancelled. -**Keyed replacement.** `pipe_keyed` / `JobOptions::key` cancels the previous +**Keyed replacement.** `JobOptions::key` cancels the previous job for the key **whatever its notification state**: pending, notified (its deadline fired) or finished with a queued terminal message. So none of its queued messages reach the handler. The key map holds only the current job. @@ -765,7 +766,7 @@ Aligned with OTP, so not differences: - **`stopped()` may reply**, and callers learn at death. - **Replies are immediate.** - **Completions are turns** (`handle_info`). -- **`pipe_keyed` is shorthand** for cancel then start. +- **A keyed job (`JobOptions::key`) is shorthand** for cancel then start. ## Decisions on rank 11a @@ -782,7 +783,7 @@ Aligned with OTP, so not differences: | Rank | Reuses | |---|---| -| 11b | `JobState`, the emit protocol, fencing, `done`, credits, `PipeErrorKind` | +| 11b | `JobState`, the emit protocol, fencing, `done`, credits, `PipeError` | | 13 | `stopped()` replies, the registry outstanding count, the unfinished report | | 15 | the per-instance job set, keyed cancel-previous, the bounded join | | 15a | a slot is a request scope; `JobOptions::deadline` is never later than its parent's | @@ -1017,9 +1018,9 @@ semantics above. during `ActorSystem::shutdown`, otherwise `DEFAULT_STOP_DEADLINE` (5 s). Jobs still running then appear in `ShutdownReport::unfinished_jobs` and as an `unfinished_at_stop` event. -- **Async emits.** `ctx.pipe_with_ctx(opts, |job| async move { .. }, done)` - hands async work its `JobCtx` (`emit`, `cancelled`); `pipe(work, done)` - keeps taking a plain future, as in the delegation example. +- **Async emits.** Every `pipe*` hands the work its `JobCtx` (`emit`, + `cancelled`): `ctx.pipe(opts, |job| async move { .. }, done)`; `|_| fut` + when the work needs neither. - **Destruction.** If an owner's lifecycle future is dropped (its runtime destroyed), exit begins before its state is dropped and the registry closes, so open tokens answer `ActorStopped`. @@ -1074,7 +1075,7 @@ changes). | 2.5.1 Attach after expiry | Attach CAS requires `OPEN`; otherwise `Closed` with the token (§8). | | 2.5.2 `TimedOut` does not trigger cancellation | The opt-in coupling triggers on `ABANDONED` and `TIMED_OUT` (§9). | | 2.5.3 `WrongOwner` shape; token lost on error | `PipeReplyError` returns the token in every variant (§9). | -| 2.5.4 No `pipe_reply_with` | `pipe_reply_with` and `pipe_reply_blocking_with` added (§9). | +| 2.5.4 No `pipe_reply_with` | `pipe_reply` and `pipe_reply_blocking` take `JobOptions` first (§9). | | 2.6.1 Keyed replacement after a deadline | Fixed (Codex 4). | | 2.6.2 Fenced payload drop panics | `DropFailures` path (§1 rule 5). | | 2.7.1 Emit credit deadlock on stop | Cancel and exit wake blocked emitters with `JobGone`; credits also return at mailbox disposal (§9). | diff --git a/examples/capabilities/src/bin/deferred_replies.rs b/examples/capabilities/src/bin/deferred_replies.rs index 445be929..ff06e89a 100644 --- a/examples/capabilities/src/bin/deferred_replies.rs +++ b/examples/capabilities/src/bin/deferred_replies.rs @@ -2,7 +2,7 @@ //! //! 1. a router delegates a slow request with `pipe_reply` and keeps serving //! its mailbox meanwhile; -//! 2. CPU work offloaded with a reserved `pipe_blocking_with` (Tokio's +//! 2. CPU work offloaded with a reserved `pipe_blocking` (Tokio's //! blocking pool) and a job deadline; //! 3. the ethrex `outgoing_request` pattern: request ids in a bounded //! `PendingReplies`, answered when the "peer" responds, late responses @@ -72,7 +72,12 @@ impl Router { .backend .request_raw_until(backend_protocol::Lookup { millis: msg.millis }, deadline); let work = async move { downstream?.await }; - if let Err(e) = ctx.pipe_reply(token, work, |res, token| Delegated { res, token }) { + if let Err(e) = ctx.pipe_reply( + JobOptions::new(), + token, + |_| work, + |res, token| Delegated { res, token }, + ) { e.into_token().overloaded(); } } @@ -85,7 +90,7 @@ impl Router { #[send_handler] async fn delegated(&mut self, msg: Delegated, _: &Context) { match msg.res { - JobOutcome::Ok(Ok(v)) => { + Ok(Ok(v)) => { msg.token.reply(v); } _ => msg.token.overloaded(), @@ -125,14 +130,14 @@ impl Verifier { async fn verify(&mut self, msg: Verify, ctx: &Context) { // Reserve first: a refusal still has `msg` to report, so every // object gets exactly one verdict. - let permit = match ctx.reserve_blocking_job() { + let opts = JobOptions::new().deadline(Duration::from_millis(100)); + let permit = match ctx.reserve_blocking_job(opts) { Ok(permit) => permit, Err(e) => return self.verdicts.push(format!("{}: rejected ({e})", msg.id)), }; let (id, spin) = (msg.id, msg.spin); // On Tokio's blocking pool, under the actor's native-job budget. - permit.pipe_blocking_with( - JobOptions::default().deadline_after(Duration::from_millis(100)), + permit.pipe_blocking( move |_job| { std::thread::sleep(spin); // stands in for a CPU-bound check true @@ -144,7 +149,7 @@ impl Verifier { #[send_handler] async fn verdict(&mut self, msg: Verdict, _: &Context) { // `JobOutcome` and `JobError` are `#[non_exhaustive]`. - self.verdicts.push(match msg.res.into_result() { + self.verdicts.push(match msg.res { Ok(valid) => format!("{}: valid={valid}", msg.id), Err(JobError::DeadlineExceeded) => format!("{}: timed out", msg.id), Err(JobError::Panicked(m)) => format!("{}: panicked {m}", msg.id), diff --git a/verification/tla/DeferredJobs.tla b/verification/tla/DeferredJobs.tla index 423393da..0d7b7c94 100644 --- a/verification/tla/DeferredJobs.tla +++ b/verification/tla/DeferredJobs.tla @@ -4,7 +4,7 @@ (* docs/design/deferred-replies.md. Part 1 is DeferredReplies.tla. *) (* *) (* An owner slot is restarted up to MaxGen times; each instance has a new *) -(* ActorId. Jobs are unlinked (ctx.pipe) or keyed (ctx.pipe_keyed), with *) +(* ActorId. Jobs are unlinked (ctx.pipe) or keyed (JobOptions::key), with *) (* or without a job deadline. Each job has three separate states: *) (* execution (running/finished; cancellation is cooperative, so cancelled *) (* or timed-out work keeps running and keeps its work permit), *) From 2021e22d35722543824bd686a7e990467077a826 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:03:38 -0300 Subject: [PATCH 145/208] test(fuzz): V5 oracle bounds cohort rounds by their grants, not rebuild times The smoke run reported "V5: 5 > 4 in 120ms" on a two-child OneForAll supervisor (max_restarts 2, window 120 ms). Instrumented, the supervisor took grants at 0, 75 and 150 ms and refused the fourth: within budget. A round is charged when it is selected, before the cohort stops and the backoff, so its rebuilds can land in a later window than its grant; the oracle counted rebuild times. For cohort supervisors the oracle now splits the retry log into rounds where a slot repeats. A round calls only while it is current, so each counted round's grant lies after the previous segment's first call and no later than its own. It flags max_restarts + 1 rounds forced into less than one window. Negative control: with charge_round ignoring the budget, the corpus plus 2000 generated inputs report 10 V5 violations; with the real code, the corpus plus 5000 generated inputs pass. OneForOne retries keep the exact check (a retry's grant is stamped at its factory call). Kept as regression cohort_rounds_charged_at_selection. --- .../cohort_rounds_charged_at_selection.input | Bin 0 -> 105 bytes fuzz/src/driver.rs | 77 +++++++++---- .../cohort_rounds_charged_at_selection.rs | 107 ++++++++++++++++++ 3 files changed, 161 insertions(+), 23 deletions(-) create mode 100644 fuzz/regressions/cohort_rounds_charged_at_selection.input create mode 100644 fuzz/tests/cohort_rounds_charged_at_selection.rs diff --git a/fuzz/regressions/cohort_rounds_charged_at_selection.input b/fuzz/regressions/cohort_rounds_charged_at_selection.input new file mode 100644 index 0000000000000000000000000000000000000000..74a07fdf1b1e7eb6c120e7f9314568cba3cfbb2f GIT binary patch literal 105 zcmaDG>`?X}1Q?e5k78g5db?-K9uSaR#Konm%;25CAYUAa25=~2{r_KnZ~X#&c{o63 F0|1ZUCdB{% literal 0 HcmV?d00001 diff --git a/fuzz/src/driver.rs b/fuzz/src/driver.rs index 5c4286c4..0922800e 100644 --- a/fuzz/src/driver.rs +++ b/fuzz/src/driver.rs @@ -1780,7 +1780,7 @@ fn budget_checks(world: &World) { let calls = s.calls.clone(); let mut found = Vec::new(); let mut seen: HashMap<(usize, String), usize> = HashMap::new(); - let mut retries: Vec> = vec![Vec::new(); s.sups.len()]; + let mut retries: Vec> = vec![Vec::new(); s.sups.len()]; for c in &calls { let n = seen.entry((c.inst, c.slot.clone())).or_default(); *n += 1; @@ -1791,38 +1791,69 @@ fn budget_checks(world: &World) { c.slot.clone(), )); } - retries[c.inst].push(c.at); + retries[c.inst].push((c.at, &c.slot)); } } - for (inst, times) in retries.iter().enumerate() { + for (inst, log) in retries.iter().enumerate() { let SupInst { max_restarts, window, cohort, .. } = s.sups[inst].clone(); - // V5 charges restart actions: a OneForOne retry is one, a cohort - // round is one however many children it rebuilds (at most all of - // them), so the factory log bounds a cohort supervisor's rebuilds by - // `max_restarts` times its children. - let children = calls - .iter() - .filter(|c| c.inst == inst) - .map(|c| &c.slot) - .collect::>() - .len() - .max(1); - let bound = if cohort { - max_restarts * children - } else { - max_restarts - }; - for (i, t) in times.iter().enumerate() { - let in_window = times[..=i].iter().filter(|u| *t - **u < window).count(); - if in_window > bound { + if !cohort { + // A OneForOne retry is one action, stamped at its admission: + // the factory call's time is the grant's. + for (i, (t, _)) in log.iter().enumerate() { + let in_window = log[..=i].iter().filter(|(u, _)| *t - *u < window).count(); + if in_window > max_restarts { + found.push(( + "V5: more restarts than the budget within one window", + format!("instance {inst}: {in_window} > {max_restarts} in {window:?}"), + )); + break; + } + } + continue; + } + // A cohort round is one action however many children it rebuilds, + // charged when it is selected, before the cohort stops and the + // backoff: its rebuilds come later by an unbounded delay, so their + // times are not the grants'. What the factory log does fix: a round + // calls each slot at most once and only while it is current, so the + // next round's grant follows its calls. Split the log where a slot + // repeats: each segment's first call starts a distinct round whose + // grant lies after the previous segment's first call and no later + // than its own first call (a merged segment only undercounts). + // `max_restarts + 1` rounds forced into less than one window break + // V5. (Retries before the tree is ready are per-child grants at + // their call, inside the same bounds.) + let mut firsts: Vec = Vec::new(); + let mut open: Vec<&str> = Vec::new(); + for (t, slot) in log { + if firsts.is_empty() || open.contains(slot) { + firsts.push(*t); + open.clear(); + } + open.push(slot); + } + for i in 0..firsts.len() { + let Some(last) = firsts.get(i + max_restarts) else { + break; + }; + let earliest = if i == 0 { + Duration::ZERO + } else { + firsts[i - 1] + }; + if *last - earliest < window { found.push(( "V5: more restarts than the budget within one window", - format!("instance {inst}: {in_window} > {bound} in {window:?}"), + format!( + "instance {inst}: {} rounds within {:?} < {window:?}", + max_restarts + 1, + *last - earliest + ), )); break; } diff --git a/fuzz/tests/cohort_rounds_charged_at_selection.rs b/fuzz/tests/cohort_rounds_charged_at_selection.rs new file mode 100644 index 00000000..403cb1d9 --- /dev/null +++ b/fuzz/tests/cohort_rounds_charged_at_selection.rs @@ -0,0 +1,107 @@ +//! Generated by `cargo run --example repro -- --emit cohort_rounds_charged_at_selection`. +//! +//! Failure: none (passing input) +//! Strategy dfs:bound=0, seed 18446744071398511597. +//! +//! ```text +//! Scenario { +//! seed: 18446744071398511597, +//! sched: Canonical, +//! ops: [ +//! TrapExit { +//! who: 237, +//! on: false, +//! }, +//! NewSystem, +//! Supervisor( +//! SupSpec { +//! system: 10, +//! policy: Policy { +//! strategy: 10, +//! max_restarts: 37, +//! window: 35, +//! backoff: 75, +//! jitter: false, +//! stable: 0, +//! }, +//! children: [ +//! Worker( +//! WorkerDef { +//! restart: 81, +//! group: 81, +//! fail_mask: 81, +//! panic_mask: 81, +//! hook_mask: 81, +//! hooks: Hooks { +//! flags: 81, +//! started_sleep: 81, +//! stopped_sleep: 81, +//! }, +//! startup_timeout: 81, +//! shutdown_timeout: 81, +//! }, +//! ), +//! Worker( +//! WorkerDef { +//! restart: 82, +//! group: 237, +//! fail_mask: 188, +//! panic_mask: 180, +//! hook_mask: 188, +//! hooks: Hooks { +//! flags: 188, +//! started_sleep: 188, +//! stopped_sleep: 188, +//! }, +//! startup_timeout: 188, +//! shutdown_timeout: 25, +//! }, +//! ), +//! ], +//! }, +//! ), +//! Link { +//! a: 0, +//! b: 75, +//! }, +//! Timer { +//! to: 5, +//! ms: 65535, +//! cmd: Monitor( +//! 47, +//! ), +//! interval: true, +//! }, +//! Request { +//! to: 31, +//! cmd: Panic, +//! timeout: None, +//! detached: true, +//! }, +//! Request { +//! to: 31, +//! cmd: Panic, +//! timeout: Some( +//! 0, +//! ), +//! detached: false, +//! }, +//! ], +//! } +//! ``` +#![cfg(spawned_sim)] + +const INPUT: &[u8] = include_bytes!("../regressions/cohort_rounds_charged_at_selection.input"); + +/// The input under its own schedule and under its recorded tape. +#[test] +fn cohort_rounds_charged_at_selection_tape() { + spawned_fuzz::assert_passes(INPUT, None); +} + +/// The same operations under other simulator schedules. +#[spawned_macros::sim_test(seeds = 32)] +async fn cohort_rounds_charged_at_selection_seeds(seed: u64) { + spawned_fuzz::init(); + spawned_fuzz::run_body(spawned_fuzz::decode(INPUT).unwrap(), seed).await; +} From a98ff83b97f07479757991568662f8a11648ebce Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:09:05 -0300 Subject: [PATCH 146/208] refactor!: send_request and one deadline type (0.8 API pass) request_raw and request_raw_until become send_request(msg, impl Into) (OTP's gen_server:send_request/3; Receiver takes a Deadline). 0.7's request_raw had no deadline: Deadline::Never keeps it. PendingReply, ReplyTo and ReplyReceiver::deadline return Deadline, so a deadline forwards as is; ReplyTo::with_deadline is shorten_deadline. Trimmed zero-user items: PendingReply::has_caller, PendingReplies::{purge, retained, live} (len/is_empty instead), the public DEFAULT_MAX_OUTSTANDING. PendingReply::detached is hidden (spawned_test::detached_reply). InsertError holds ReplyTo and is non_exhaustive. DEFAULT_REQUEST_TIMEOUT is re-exported at the root; macros name root paths. Duration parameters called deadline are named grace or limit. PendingReplies and deferred_outstanding are documented experimental. --- CHANGELOG.md | 24 +++- concurrency/benches/baseline.rs | 4 +- concurrency/benches/designs.rs | 12 +- concurrency/benches/runtime_isolation.rs | 18 ++- concurrency/benches/workload_research.rs | 121 ++++++++++-------- concurrency/src/lib.rs | 2 +- concurrency/src/reply/mod.rs | 62 ++++----- concurrency/src/reply/pending.rs | 47 ++++--- concurrency/src/reply/registry.rs | 2 +- concurrency/src/supervision.rs | 4 +- concurrency/src/system.rs | 10 +- concurrency/src/tasks/actor.rs | 79 +++++------- concurrency/src/tasks/mod.rs | 2 +- concurrency/src/threads/actor.rs | 79 +++++------- concurrency/src/threads/mod.rs | 2 +- concurrency/tests/actor_scenarios.rs | 10 +- concurrency/tests/cross_backend.rs | 2 +- concurrency/tests/deadline_service.rs | 2 +- concurrency/tests/death_boundary.rs | 55 ++++++-- concurrency/tests/deferred_api.rs | 16 ++- concurrency/tests/deferred_jobs.rs | 15 +-- concurrency/tests/deferred_replies.rs | 50 ++++---- concurrency/tests/deferred_sim.rs | 2 +- concurrency/tests/execution_groups.rs | 7 +- concurrency/tests/failpoints.rs | 8 +- concurrency/tests/foreign_panics.rs | 5 +- concurrency/tests/full_actor_shards.rs | 41 ++++-- concurrency/tests/lifecycle_trace.rs | 8 +- concurrency/tests/mailbox_drop_panic.rs | 18 ++- concurrency/tests/metrics.rs | 27 +++- concurrency/tests/metrics_deferred.rs | 26 +++- concurrency/tests/native_mailbox_drop.rs | 9 +- concurrency/tests/otp_exit_signals.rs | 54 ++++++-- concurrency/tests/panic_safety.rs | 12 +- concurrency/tests/reply_trace.rs | 22 +++- concurrency/tests/semantics_gaps.rs | 27 +++- concurrency/tests/shutdown_publication.rs | 14 +- concurrency/tests/sim_explore.rs | 5 +- concurrency/tests/sim_ordering.rs | 9 +- concurrency/tests/startup_admission.rs | 9 +- concurrency/tests/supervision.rs | 4 +- concurrency/tests/support/failure_actor.rs | 4 +- docs/GUIDE.md | 10 +- docs/MIGRATION.md | 30 +++-- docs/design/deferred-replies.md | 51 ++++---- docs/design/semantics.md | 9 +- .../capabilities/src/bin/deferred_replies.rs | 47 +++++-- .../capabilities/src/bin/timeout_outcomes.rs | 2 +- fuzz/src/driver.rs | 2 +- macros/src/lib.rs | 10 +- testing/src/lib.rs | 10 ++ testing/src/shared.rs | 2 +- testing/src/tasks.rs | 58 ++++----- testing/src/threads.rs | 12 +- testing/tests/tasks_kit.rs | 7 +- testing/tests/threads_kit.rs | 4 +- 56 files changed, 738 insertions(+), 445 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index badd4ef3..97d513ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,24 @@ `Failed` variant carries `error`. `Deadline` implements `From` and `From`. `DEFAULT_EMIT_CREDITS` and `default_max_native_jobs` are no longer public. + - Requests and deadlines: `request_raw` and `request_raw_until` become + `send_request(msg, impl Into)` (OTP's + `gen_server:send_request/3`; on `Receiver`, a `Deadline`). 0.7's + `request_raw` had no deadline: `Deadline::Never` keeps that. + `PendingReply::deadline`, `ReplyTo::deadline` and `ReplyReceiver::deadline` + return `Deadline`, so a deadline forwards as is; + `ReplyTo::with_deadline` is `shorten_deadline(impl Into)`. + Removed: `PendingReply::has_caller` (`defer()` returns `NoCaller`), + `PendingReplies::{purge, retained, live}` (now `len`/`is_empty`; purging + stays automatic), the public `DEFAULT_MAX_OUTSTANDING` (the default, + 1024, is documented at `Actor::MAX_OUTSTANDING`). + `PendingReply::detached` is hidden (`spawned_test::detached_reply()`). + `InsertError` holds `ReplyTo` and is `#[non_exhaustive]`. + `DEFAULT_REQUEST_TIMEOUT` is re-exported at the crate root. The + `Duration` parameters of `ActorSystem::shutdown`, `Supervisor::shutdown`, + `ShutdownReport::wait` and the test kit's waits are named `grace` or + `limit` (a deadline is an instant). `PendingReplies` and + `Context::deferred_outstanding` are documented as experimental. - Deferred replies and jobs, fixes from the 0.8 pre-release review: a job deadline still fires while the result waits for scheduled emits, and the job keeps its `MAX_OUTSTANDING` credit until its terminal message; a job @@ -175,8 +193,8 @@ obligations answer `ReplyDropped` (in a panicking turn or while exiting they are answered `ActorStopped` at death); at death every open token answers `ActorStopped`. Request - deadlines are fixed at send (`Deadline`, `request_raw_until`; `request_raw` - gains the 5 s default); a per-actor deadline heap and service expire stored + deadlines are fixed at send (`Deadline`, `send_request(msg, deadline)`, + formerly `request_raw`); a per-actor deadline heap and service expire stored tokens without a polling caller. `PendingReplies` is a bounded token table. Pipe-to-self jobs: `ctx.pipe`, `pipe_blocking`, `pipe_reply`, `pipe_reply_blocking` (threads mode: the blocking forms on owned native @@ -209,7 +227,7 @@ `spawned_test::tasks::in_actor` / `threads::in_actor` run a closure in a live actor's turn. GUIDE: hard job deadlines and the soft-deadline pattern, replies carrying a reason through `R = Result<..>`, the - `request_raw` rows. + `send_request` rows. - ROADMAP: a Path to 1.0 section (0.8, 0.9, 0.10 contents and what 1.0 requires: stable API rules, a complete semantics contract, production-safe defaults, introspection, determinism, release gates, production proof and diff --git a/concurrency/benches/baseline.rs b/concurrency/benches/baseline.rs index a2adff51..994c534d 100644 --- a/concurrency/benches/baseline.rs +++ b/concurrency/benches/baseline.rs @@ -135,12 +135,12 @@ impl Target { match self { Self::Tasks(actor) => Reply::Tasks( actor - .request_raw_until(work, Deadline::after(Duration::from_secs(10))) + .send_request(work, Deadline::after(Duration::from_secs(10))) .expect("actor stopped"), ), Self::Threads(actor) => Reply::Threads( actor - .request_raw_until(work, Deadline::after(Duration::from_secs(10))) + .send_request(work, Deadline::after(Duration::from_secs(10))) .expect("actor stopped"), ), Self::Tokio(tx) => { diff --git a/concurrency/benches/designs.rs b/concurrency/benches/designs.rs index c8a5cf7d..c03883d8 100644 --- a/concurrency/benches/designs.rs +++ b/concurrency/benches/designs.rs @@ -310,8 +310,16 @@ impl Target { return rx; } match self { - Self::Tasks(actor) => Reply::Tasks(actor.request_raw(work).expect("actor stopped")), - Self::Threads(actor) => Reply::Threads(actor.request_raw(work).expect("actor stopped")), + Self::Tasks(actor) => Reply::Tasks( + actor + .send_request(work, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .expect("actor stopped"), + ), + Self::Threads(actor) => Reply::Threads( + actor + .send_request(work, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .expect("actor stopped"), + ), Self::Tokio(tx) => { let (reply, rx) = tokio::sync::oneshot::channel(); tx.send(TokioMessage::Request(work, reply)) diff --git a/concurrency/benches/runtime_isolation.rs b/concurrency/benches/runtime_isolation.rs index 987c0f36..2b1491e2 100644 --- a/concurrency/benches/runtime_isolation.rs +++ b/concurrency/benches/runtime_isolation.rs @@ -55,7 +55,11 @@ mod linux { let __reply_value: (u64, u64, Instant) = async { self.handled += 1; if let Some(next) = &self.next { - return next.request_raw(m).unwrap().await.unwrap(); + return next + .send_request(m, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap() + .await + .unwrap(); } let mut value = m.0; for _ in 0..self.iterations { @@ -216,7 +220,11 @@ mod linux { barrier.wait().await; let mut n = 0u64; while Instant::now() < *finish.get().unwrap() { - let reply = actor.request_raw(Work(n)).unwrap().await.unwrap(); + let reply = actor + .send_request(Work(n), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap() + .await + .unwrap(); check(reply, n, ab, bb); n += 1; } @@ -236,7 +244,11 @@ mod linux { let mut n = 0; while Instant::now() < *cfinish.get().unwrap() { let start = Instant::now(); - let reply = endpoint.request_raw(Work(n)).unwrap().await.unwrap(); + let reply = endpoint + .send_request(Work(n), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap() + .await + .unwrap(); let end = Instant::now(); check(reply, n, ac, bc); latency.push(end.duration_since(start).as_nanos() as u64); diff --git a/concurrency/benches/workload_research.rs b/concurrency/benches/workload_research.rs index e9af06ef..0c2223b9 100644 --- a/concurrency/benches/workload_research.rs +++ b/concurrency/benches/workload_research.rs @@ -270,7 +270,10 @@ impl Handler for Node { let replies: Vec<_> = self .children .iter() - .map(|c| c.request_raw(msg.clone()).unwrap()) + .map(|c| { + c.send_request(msg.clone(), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap() + }) .collect(); let mut nodes = 1; let mut tag = self.id as u64 + 1; @@ -599,22 +602,28 @@ fn main() { runtime .block_on( nodes[id] - .request_raw(Wire { - children, - expected_shard: assignment[id], - }) + .send_request( + Wire { + children, + expected_shard: assignment[id], + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) .unwrap(), ) .unwrap(); } for (id, node) in nodes.iter().enumerate() { let reply = node - .request_raw(Route { - id: id as u64, - enqueued: Instant::now(), - iterations: c.iterations, - measured: false, - }) + .send_request( + Route { + id: id as u64, + enqueued: Instant::now(), + iterations: c.iterations, + measured: false, + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) .unwrap(); runtime.block_on(reply).unwrap(); } @@ -663,43 +672,45 @@ fn main() { epoch.changed().await.unwrap(); let start = epoch.borrow().unwrap(); wait_until(start); - let producers = (driver..c.producers) - .step_by(c.caller_threads) - .map(|producer| async move { - let mut samples = - Vec::with_capacity(c.messages.div_ceil(c.producers)); - for id in (producer..c.messages).step_by(c.producers) { - let intended = Instant::now(); - let enqueued = Instant::now(); - let (target, hot) = c.target(id); - let reply = nodes[target] - .request_raw(Route { + let producers = + (driver..c.producers).step_by(c.caller_threads).map( + |producer| async move { + let mut samples = Vec::with_capacity( + c.messages.div_ceil(c.producers), + ); + for id in (producer..c.messages).step_by(c.producers) { + let intended = Instant::now(); + let enqueued = Instant::now(); + let (target, hot) = c.target(id); + let reply = nodes[target] + .send_request(Route { id: id as u64, enqueued, iterations: c.iterations, measured: false, - }) + }, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) .unwrap(); - let ack = reply.await.unwrap(); - samples.push(sample( - Meta { - id, - target, - hot, - intended, - enqueued, - }, - Some(ack), - Instant::now(), - c, - assignment, - )); - if c.observation_counter { - observed.fetch_add(1, Ordering::Relaxed); + let ack = reply.await.unwrap(); + samples.push(sample( + Meta { + id, + target, + hot, + intended, + enqueued, + }, + Some(ack), + Instant::now(), + c, + assignment, + )); + if c.observation_counter { + observed.fetch_add(1, Ordering::Relaxed); + } } - } - samples - }); + samples + }, + ); futures::future::join_all(producers) .await .into_iter() @@ -788,12 +799,15 @@ fn main() { meters[target].submitted.fetch_add(1, Ordering::Relaxed); } let reply = nodes[target] - .request_raw(Route { - id: id as u64, - enqueued, - iterations, - measured: c.mode != "graph", - }) + .send_request( + Route { + id: id as u64, + enqueued, + iterations, + measured: c.mode != "graph", + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) .unwrap(); let meta = Meta { id, @@ -858,10 +872,13 @@ fn main() { for (id, node) in nodes.iter().enumerate() { runtime .block_on( - node.request_raw(Wire { - children: Vec::new(), - expected_shard: assignment[id], - }) + node.send_request( + Wire { + children: Vec::new(), + expected_shard: assignment[id], + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) .unwrap(), ) .unwrap(); diff --git a/concurrency/src/lib.rs b/concurrency/src/lib.rs index 8a2518bd..deefc428 100644 --- a/concurrency/src/lib.rs +++ b/concurrency/src/lib.rs @@ -114,7 +114,7 @@ pub use reply::{ Deadline, DeferError, Delivery, InsertError, PendingReplies, PendingReply, ReplyReceiver, ReplyTo, }; -pub use response::Response; +pub use response::{Response, DEFAULT_REQUEST_TIMEOUT}; pub use spawned_macros::{actor, protocol, send_handler}; /// The instant type of reply and job deadlines (`Deadline::At`, /// `JobOptions::deadline`, `ReplyTo::deadline`, `JobCtx::emit_at`): diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 0ba4cda4..626f96ff 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -15,7 +15,7 @@ pub(crate) mod slot; pub(crate) mod turn; pub use pending::{InsertError, PendingReplies}; -pub use registry::DEFAULT_MAX_OUTSTANDING; +pub(crate) use registry::DEFAULT_MAX_OUTSTANDING; pub use slot::Delivery; use crate::error::ActorError; @@ -86,9 +86,11 @@ pub(crate) fn now_ticks() -> u64 { /// A request's deadline, fixed when it is sent. /// /// `gen_server:call/3` takes a timeout or `infinity`; a request takes an -/// absolute instant or [`Deadline::Never`]. The default for `request` and -/// `request_raw` is [`DEFAULT_REQUEST_TIMEOUT`](crate::response::DEFAULT_REQUEST_TIMEOUT) -/// after the send. +/// absolute instant or [`Deadline::Never`]. `request` uses +/// [`DEFAULT_REQUEST_TIMEOUT`](crate::response::DEFAULT_REQUEST_TIMEOUT) +/// after the send; `send_request` takes the deadline. Every call that takes +/// a deadline takes `impl Into`: a `Duration` (from now), an +/// `Instant` or a `Deadline`. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Deadline { /// Expire at this instant (on the `spawned_rt::time` clock). @@ -113,8 +115,12 @@ impl Deadline { } } - fn from_ticks(ticks: u64) -> Option { - (ticks != NEVER).then(|| spawned_rt::time::from_ticks(ticks)) + fn of_ticks(ticks: u64) -> Self { + if ticks == NEVER { + Self::Never + } else { + Self::At(spawned_rt::time::from_ticks(ticks)) + } } } @@ -277,7 +283,9 @@ impl<'t, R: Send + 'static> PendingReply<'t, R> { } } - /// A `PendingReply` for a sent message: there is no caller. + /// A `PendingReply` for a sent message: there is no caller. For tests + /// calling a handler by hand, use `spawned_test::detached_reply()`. + #[doc(hidden)] pub fn detached() -> Self { Self { slot: None, @@ -331,11 +339,13 @@ impl<'t, R: Send + 'static> PendingReply<'t, R> { } } - /// The request's deadline, or `None` for [`Deadline::Never`] and sends. - pub fn deadline(&self) -> Option { + /// The request's deadline, fixed at send; [`Deadline::Never`] for an + /// unbounded request and for sends. Pass it on to forward the request + /// with the same deadline (`other.send_request(msg, reply.deadline())`). + pub fn deadline(&self) -> Deadline { self.slot .as_ref() - .and_then(|s| Deadline::from_ticks(s.deadline())) + .map_or(Deadline::Never, |s| Deadline::of_ticks(s.deadline())) } /// Whether the caller can no longer be answered (the deadline passed or @@ -352,11 +362,6 @@ impl<'t, R: Send + 'static> PendingReply<'t, R> { slot.terminate(Terminal::Stopped); } } - - /// Whether this is a request (has a caller) rather than a send. - pub fn has_caller(&self) -> bool { - self.slot.is_some() - } } impl Drop for PendingReply<'_, R> { @@ -427,19 +432,18 @@ impl ReplyTo { } } - /// The request's deadline, or `None` for [`Deadline::Never`]. - pub fn deadline(&self) -> Option { - Deadline::from_ticks(self.slot().deadline()) + /// The request's deadline (as shortened so far). + pub fn deadline(&self) -> Deadline { + Deadline::of_ticks(self.slot().deadline()) } - /// Shorten the deadline to `at` (never extends it). - pub fn with_deadline(&self, at: Instant) { - // Only a lowered deadline gets a heap entry, so repeated - // `with_deadline` calls with the same or a later instant store nothing. - if self - .slot() - .shorten(spawned_rt::time::to_ticks(at), &now_ticks) - { + /// Shorten the request's deadline to `deadline` (a `Duration` from now + /// or an `Instant`); a later deadline, or [`Deadline::Never`], leaves it + /// unchanged. The waiting caller is woken to re-arm its timer. + pub fn shorten_deadline(&self, deadline: impl Into) { + // Only a lowered deadline gets a heap entry, so repeated calls with + // the same or a later deadline store nothing. + if self.slot().shorten(deadline.into().ticks(), &now_ticks) { self.registry.track_deadline(self.slot()); } } @@ -568,9 +572,9 @@ impl ReplyReceiver { .expect("ReplyReceiver used after completion") } - /// The deadline fixed at send, or `None` for [`Deadline::Never`]. - pub fn deadline(&self) -> Option { - Deadline::from_ticks(self.slot().deadline()) + /// The deadline fixed at send (as the owner shortened it so far). + pub fn deadline(&self) -> Deadline { + Deadline::of_ticks(self.slot().deadline()) } /// Block the calling thread until the reply, the deadline or the owner's diff --git a/concurrency/src/reply/pending.rs b/concurrency/src/reply/pending.rs index 2881d1c0..ccbaaf56 100644 --- a/concurrency/src/reply/pending.rs +++ b/concurrency/src/reply/pending.rs @@ -9,15 +9,17 @@ use std::hash::Hash; /// the answer (typically `token.overloaded()` or a domain error) and still /// owns the token. #[derive(Debug)] -pub enum InsertError { +#[non_exhaustive] +pub enum InsertError { /// A live token is already stored under this key; it is kept. - Duplicate(T), + Duplicate(ReplyTo), /// The table holds `cap` live tokens. - Full(T), + Full(ReplyTo), } -impl InsertError { - pub fn into_inner(self) -> T { +impl InsertError { + /// The token handed back. + pub fn into_inner(self) -> ReplyTo { match self { Self::Duplicate(token) | Self::Full(token) => token, } @@ -27,6 +29,9 @@ impl InsertError { /// Deferred tokens by key, capped: retained entries never exceed `cap`, /// however fast tokens expire. Terminal tokens (answered, expired, abandoned) /// are purged on every insert and skipped by `take`. +/// +/// Experimental in 0.8: a utility shaped by one client (requests by id +/// answered when a peer responds); it may change before 1.0. pub struct PendingReplies { cap: usize, tokens: HashMap>, @@ -51,7 +56,7 @@ impl PendingReplies { /// Purge terminal tokens, then insert. A duplicate key (with a live /// token) or a full table hands the new token back. - pub fn insert(&mut self, key: K, token: ReplyTo) -> Result<(), InsertError>> { + pub fn insert(&mut self, key: K, token: ReplyTo) -> Result<(), InsertError> { self.purge(); if self.tokens.contains_key(&key) { return Err(InsertError::Duplicate(token)); @@ -70,21 +75,20 @@ impl PendingReplies { (!token.is_closed()).then_some(token) } - /// Remove every terminal token; returns how many were removed. - pub fn purge(&mut self) -> usize { - let before = self.tokens.len(); + /// Remove every terminal token. + fn purge(&mut self) { self.tokens.retain(|_, token| !token.is_closed()); - before - self.tokens.len() } - /// Entries held, without purging. - pub fn retained(&self) -> usize { - self.tokens.len() + /// Live (non-terminal) tokens held: a scan, so a token whose caller + /// just left no longer counts. + pub fn len(&self) -> usize { + self.tokens.values().filter(|t| !t.is_closed()).count() } - /// Non-terminal entries (a scan). - pub fn live(&self) -> usize { - self.tokens.values().filter(|t| !t.is_closed()).count() + /// Whether no live token is held. + pub fn is_empty(&self) -> bool { + self.len() == 0 } /// Take every live token (for example to answer them in `stopped()`). @@ -123,11 +127,16 @@ mod tests { assert!(matches!(rdup.recv(), Err(ActorError::ReplyDropped))); // A caller that left frees its entry on the next insert. drop(rb); - assert_eq!(table.retained(), 2); - assert_eq!(table.live(), 1); + assert_eq!( + table.tokens.len(), + 2, + "terminal tokens stay until an insert" + ); + assert_eq!(table.len(), 1); let (d, rd) = token(); table.insert(4, d).unwrap(); - assert_eq!(table.retained(), 2); + assert_eq!(table.tokens.len(), 2); + assert_eq!(table.len(), 2); assert!(table.take(&2).is_none()); table.take(&1).unwrap().reply(10); table.take(&4).unwrap().reply(40); diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index d7c0a501..ea2c2067 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -31,7 +31,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, OnceLock, Weak}; /// Default `Actor::MAX_OUTSTANDING`: deferred credits per actor instance. -pub const DEFAULT_MAX_OUTSTANDING: usize = 1024; +pub(crate) const DEFAULT_MAX_OUTSTANDING: usize = 1024; use super::turn::{Outcome, TurnPhase, DEAD, EXITING}; diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs index bfa858b3..750c8109 100644 --- a/concurrency/src/supervision.rs +++ b/concurrency/src/supervision.rs @@ -1410,9 +1410,9 @@ impl Supervisor { pub async fn join(&self) { let _ = system::wait_true(self.core.done.subscribe()).await; } - pub async fn shutdown(&self, deadline: Duration) -> SupervisorReport { + pub async fn shutdown(&self, grace: Duration) -> SupervisorReport { self.stop(); - let complete = rt::timeout(deadline, system::wait_true(self.core.done.subscribe())) + let complete = rt::timeout(grace, system::wait_true(self.core.done.subscribe())) .await .unwrap_or(false); if !complete { diff --git a/concurrency/src/system.rs b/concurrency/src/system.rs index 43b41f2d..9d1b43b1 100644 --- a/concurrency/src/system.rs +++ b/concurrency/src/system.rs @@ -413,8 +413,8 @@ impl ActorSystem { /// Close admission and request ordered shutdown. At expiry the report owns /// pending shutdown state, which can be awaited again. Blocking Rust is not /// forcibly killed and runtime destruction never blocks this caller thread. - pub async fn shutdown(&self, deadline: Duration) -> ShutdownReport { - self.inner.begin_shutdown(deadline); + pub async fn shutdown(&self, grace: Duration) -> ShutdownReport { + self.inner.begin_shutdown(grace); ShutdownReport { pending: Some(self.inner.clone()), complete: false, @@ -427,7 +427,7 @@ impl ActorSystem { coordinator_failures: Vec::new(), unfinished_jobs: Vec::new(), } - .wait(deadline) + .wait(grace) .await } @@ -699,11 +699,11 @@ pub struct ActorDiagnostic { pub turn_age: Option, } impl ShutdownReport { - pub async fn wait(mut self, deadline: Duration) -> Self { + pub async fn wait(mut self, grace: Duration) -> Self { let Some(owner) = self.pending.as_ref() else { return self; }; - self.complete = rt::timeout(deadline, wait_true(owner.control.done.clone())) + self.complete = rt::timeout(grace, wait_true(owner.control.done.clone())) .await .unwrap_or(false); let state = lock(&owner.state); diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index a3696e9b..eff91c43 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -75,7 +75,7 @@ pub enum Backend { pub trait Actor: Send + Sized + 'static { /// Deferred-reply credits per instance (live `ReplyTo` tokens plus running /// linked jobs). At the bound `defer()` answers the caller `Overloaded`. - /// Capped at 65535. + /// Default 1024; capped at 65535. const MAX_OUTSTANDING: usize = DEFAULT_MAX_OUTSTANDING; /// Native threads this instance's blocking jobs may run at once @@ -565,6 +565,8 @@ impl Context { /// Deferred-reply credits in use by this instance: live deferred /// tokens plus running linked jobs (bounded by `Actor::MAX_OUTSTANDING`). + /// Experimental in 0.8 (for tests and diagnostics; the `metrics` + /// feature exports the same count): it may change before 1.0. pub fn deferred_outstanding(&self) -> usize { self.shared.replies.outstanding() } @@ -636,28 +638,21 @@ impl Context { ) } - /// Send a request and get its [`ReplyReceiver`], with the default - /// deadline ([`DEFAULT_REQUEST_TIMEOUT`] after the send, as - /// `gen_server:call/2`). - pub fn request_raw(&self, msg: M) -> Result, ActorError> - where - A: Handler, - M: Message, - { - self.request_raw_until(msg, Deadline::after(DEFAULT_REQUEST_TIMEOUT)) - } - - /// Send a request whose deadline is `deadline` ([`Deadline::Never`] for - /// an unbounded wait). - pub fn request_raw_until( + /// Send a request without waiting (`gen_server:send_request/3`) and get + /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from + /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait + /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) + /// uses). The handler sees it through `PendingReply::deadline`. + pub fn send_request( &self, msg: M, - deadline: Deadline, + deadline: impl Into, ) -> Result, ActorError> where A: Handler, M: Message, { + let deadline = deadline.into(); let (slot, rx) = reply_slot(deadline, Some(&self.shared.replies)); enqueue( &self.shared.sender, @@ -693,7 +688,7 @@ impl Context { // Bind the receiver before awaiting it: awaiting inside a `match` on // the `Result` keeps the scrutinee's storage alive beside the awaited // receiver and doubles this future's size. - let rx = match self.request_raw_until(msg, Deadline::after(duration)) { + let rx = match self.send_request(msg, Deadline::after(duration)) { Ok(rx) => rx, Err(error) => { let result = Err(error); @@ -1046,12 +1041,12 @@ where Context::send(self, msg) } - fn request_raw_until( + fn send_request( &self, msg: M, deadline: Deadline, ) -> Result, ActorError> { - Context::request_raw_until(self, msg, deadline) + Context::send_request(self, msg, deadline) } } @@ -1065,17 +1060,14 @@ where /// message type that `A` handles. pub trait Receiver: Send + Sync { fn send(&self, msg: M) -> Result<(), ActorError>; - /// Send a request whose deadline is fixed now (the slot carries it). - fn request_raw_until( + /// Send a request without waiting (`gen_server:send_request/3`); its + /// deadline is fixed now (the slot carries it). On a [`Recipient`], + /// pass a `Deadline` (`Deadline::after(d)`, or `d.into()`). + fn send_request( &self, msg: M, deadline: Deadline, ) -> Result, ActorError>; - /// Send a request with the default deadline - /// ([`DEFAULT_REQUEST_TIMEOUT`] after the send). - fn request_raw(&self, msg: M) -> Result, ActorError> { - self.request_raw_until(msg, Deadline::after(DEFAULT_REQUEST_TIMEOUT)) - } } /// Type-erased reference for sending a single message type. @@ -1092,7 +1084,7 @@ pub async fn request( let observation = crate::instrumentation::RequestObservation::new(); // The deadline is computed before enqueueing (send time, as OTP's call). // Bind the receiver before awaiting it (see `ActorRef::request_with_timeout`). - let rx = match recipient.request_raw_until(msg, Deadline::after(timeout_duration)) { + let rx = match recipient.send_request(msg, Deadline::after(timeout_duration)) { Ok(rx) => rx, Err(error) => { let result = Err(error); @@ -1168,28 +1160,21 @@ impl ActorRef { ) } - /// Send a request and get its [`ReplyReceiver`], with the default - /// deadline ([`DEFAULT_REQUEST_TIMEOUT`] after the send, as - /// `gen_server:call/2`). - pub fn request_raw(&self, msg: M) -> Result, ActorError> - where - A: Handler, - M: Message, - { - self.request_raw_until(msg, Deadline::after(DEFAULT_REQUEST_TIMEOUT)) - } - - /// Send a request whose deadline is `deadline` ([`Deadline::Never`] for - /// an unbounded wait). - pub fn request_raw_until( + /// Send a request without waiting (`gen_server:send_request/3`) and get + /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from + /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait + /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) + /// uses). The handler sees it through `PendingReply::deadline`. + pub fn send_request( &self, msg: M, - deadline: Deadline, + deadline: impl Into, ) -> Result, ActorError> where A: Handler, M: Message, { + let deadline = deadline.into(); let (slot, rx) = reply_slot(deadline, Some(&self.shared.replies)); enqueue( &self.shared.sender, @@ -1249,7 +1234,7 @@ impl ActorRef { // Bind the receiver before awaiting it: awaiting inside a `match` on // the `Result` keeps the scrutinee's storage alive beside the awaited // receiver and doubles this future's size. - let rx = match self.request_raw_until(msg, Deadline::after(duration)) { + let rx = match self.send_request(msg, Deadline::after(duration)) { Ok(rx) => rx, Err(error) => { let result = Err(error); @@ -1361,12 +1346,12 @@ where ActorRef::send(self, msg) } - fn request_raw_until( + fn send_request( &self, msg: M, deadline: Deadline, ) -> Result, ActorError> { - ActorRef::request_raw_until(self, msg, deadline) + ActorRef::send_request(self, msg, deadline) } } @@ -2449,7 +2434,9 @@ mod tests { let counter = Counter { count: 42 }.start(); let recipient: Recipient = counter.recipient(); - let rx = recipient.request_raw(GetCount).unwrap(); + let rx = recipient + .send_request(GetCount, Deadline::after(DEFAULT_REQUEST_TIMEOUT)) + .unwrap(); let result = rx.await.unwrap(); assert_eq!(result, 42); diff --git a/concurrency/src/tasks/mod.rs b/concurrency/src/tasks/mod.rs index 5decdded..dd73d3db 100644 --- a/concurrency/src/tasks/mod.rs +++ b/concurrency/src/tasks/mod.rs @@ -12,7 +12,7 @@ pub use crate::reply::jobs::{ }; pub use crate::reply::{ Deadline, DeferError, Delivery, InsertError, PendingReplies, PendingReply, ReplyFuture, - ReplyReceiver, ReplyTo, DEFAULT_MAX_OUTSTANDING, + ReplyReceiver, ReplyTo, }; pub use crate::response::Response; pub use crate::Instant; diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 67df8676..718ef2fc 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -43,7 +43,7 @@ pub use jobs::{BlockingJobPermit, JobCtx}; pub trait Actor: Send + Sized + 'static { /// Deferred-reply credits per instance (live `ReplyTo` tokens plus running /// linked jobs). At the bound `defer()` answers the caller `Overloaded`. - /// Capped at 65535. + /// Default 1024; capped at 65535. const MAX_OUTSTANDING: usize = DEFAULT_MAX_OUTSTANDING; /// Native threads this instance's jobs may run at once. `None`: @@ -446,6 +446,8 @@ impl Context { /// Deferred-reply credits in use by this instance: live deferred /// tokens plus running linked jobs (bounded by `Actor::MAX_OUTSTANDING`). + /// Experimental in 0.8 (for tests and diagnostics; the `metrics` + /// feature exports the same count): it may change before 1.0. pub fn deferred_outstanding(&self) -> usize { self.replies.outstanding() } @@ -516,27 +518,21 @@ impl Context { ) } - /// Send a request and get its [`ReplyReceiver`], with the default - /// deadline ([`DEFAULT_REQUEST_TIMEOUT`] after the send). - pub fn request_raw(&self, msg: M) -> Result, ActorError> - where - A: Handler, - M: Message, - { - self.request_raw_until(msg, Deadline::after(DEFAULT_REQUEST_TIMEOUT)) - } - - /// Send a request whose deadline is `deadline` ([`Deadline::Never`] for - /// an unbounded wait). - pub fn request_raw_until( + /// Send a request without waiting (`gen_server:send_request/3`) and get + /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from + /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait + /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) + /// uses). The handler sees it through `PendingReply::deadline`. + pub fn send_request( &self, msg: M, - deadline: Deadline, + deadline: impl Into, ) -> Result, ActorError> where A: Handler, M: Message, { + let deadline = deadline.into(); let (slot, rx) = reply_slot(deadline, Some(&self.replies)); enqueue(&self.sender, &self.mailbox, &self.replies, msg, slot)?; Ok(rx) @@ -562,7 +558,7 @@ impl Context { M: Message, { let observation = crate::instrumentation::RequestObservation::new(); - let result = match self.request_raw_until(msg, Deadline::after(duration)) { + let result = match self.send_request(msg, Deadline::after(duration)) { Err(error) => Err(error), Ok(rx) => rx.recv(), }; @@ -845,12 +841,12 @@ where Context::send(self, msg) } - fn request_raw_until( + fn send_request( &self, msg: M, deadline: Deadline, ) -> Result, ActorError> { - Context::request_raw_until(self, msg, deadline) + Context::send_request(self, msg, deadline) } } @@ -864,17 +860,14 @@ where /// message type that `A` handles. pub trait Receiver: Send + Sync { fn send(&self, msg: M) -> Result<(), ActorError>; - /// Send a request whose deadline is fixed now (the slot carries it). - fn request_raw_until( + /// Send a request without waiting (`gen_server:send_request/3`); its + /// deadline is fixed now (the slot carries it). On a [`Recipient`], + /// pass a `Deadline` (`Deadline::after(d)`, or `d.into()`). + fn send_request( &self, msg: M, deadline: Deadline, ) -> Result, ActorError>; - /// Send a request with the default deadline - /// ([`DEFAULT_REQUEST_TIMEOUT`] after the send). - fn request_raw(&self, msg: M) -> Result, ActorError> { - self.request_raw_until(msg, Deadline::after(DEFAULT_REQUEST_TIMEOUT)) - } } /// Type-erased reference for sending a single message type. @@ -887,7 +880,7 @@ pub fn request( timeout: Duration, ) -> Result { let observation = crate::instrumentation::RequestObservation::new(); - let result = match recipient.request_raw_until(msg, Deadline::after(timeout)) { + let result = match recipient.send_request(msg, Deadline::after(timeout)) { Err(error) => Err(error), Ok(rx) => rx.recv(), }; @@ -969,27 +962,21 @@ impl ActorRef { ) } - /// Send a request and get its [`ReplyReceiver`], with the default - /// deadline ([`DEFAULT_REQUEST_TIMEOUT`] after the send). - pub fn request_raw(&self, msg: M) -> Result, ActorError> - where - A: Handler, - M: Message, - { - self.request_raw_until(msg, Deadline::after(DEFAULT_REQUEST_TIMEOUT)) - } - - /// Send a request whose deadline is `deadline` ([`Deadline::Never`] for - /// an unbounded wait). - pub fn request_raw_until( + /// Send a request without waiting (`gen_server:send_request/3`) and get + /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from + /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait + /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) + /// uses). The handler sees it through `PendingReply::deadline`. + pub fn send_request( &self, msg: M, - deadline: Deadline, + deadline: impl Into, ) -> Result, ActorError> where A: Handler, M: Message, { + let deadline = deadline.into(); let (slot, rx) = reply_slot(deadline, Some(&self.replies)); enqueue(&self.sender, &self.mailbox, &self.replies, msg, slot)?; Ok(rx) @@ -1038,7 +1025,7 @@ impl ActorRef { M: Message, { let observation = crate::instrumentation::RequestObservation::new(); - let result = match self.request_raw_until(msg, Deadline::after(duration)) { + let result = match self.send_request(msg, Deadline::after(duration)) { Err(error) => Err(error), Ok(rx) => rx.recv(), }; @@ -1123,12 +1110,12 @@ where ActorRef::send(self, msg) } - fn request_raw_until( + fn send_request( &self, msg: M, deadline: Deadline, ) -> Result, ActorError> { - ActorRef::request_raw_until(self, msg, deadline) + ActorRef::send_request(self, msg, deadline) } } @@ -2613,9 +2600,11 @@ mod stop_wakeup_regression { handled: handled.clone(), } .start(); - let first = actor.request_raw(Work).unwrap(); + let first = actor.send_request(Work, DEFAULT_REQUEST_TIMEOUT).unwrap(); entered.recv_timeout(Duration::from_secs(2)).unwrap(); - let replies: Vec<_> = (0..256).map(|_| actor.request_raw(Work).unwrap()).collect(); + let replies: Vec<_> = (0..256) + .map(|_| actor.send_request(Work, DEFAULT_REQUEST_TIMEOUT).unwrap()) + .collect(); std::thread::scope(|scope| { for _ in 0..16 { let child = actor.child_handle(); diff --git a/concurrency/src/threads/mod.rs b/concurrency/src/threads/mod.rs index 3eff7b52..4bdd6644 100644 --- a/concurrency/src/threads/mod.rs +++ b/concurrency/src/threads/mod.rs @@ -13,7 +13,7 @@ pub use crate::reply::jobs::{ }; pub use crate::reply::{ Deadline, DeferError, Delivery, InsertError, PendingReplies, PendingReply, ReplyReceiver, - ReplyTo, DEFAULT_MAX_OUTSTANDING, + ReplyTo, }; pub use crate::response::Response; pub use crate::Instant; diff --git a/concurrency/tests/actor_scenarios.rs b/concurrency/tests/actor_scenarios.rs index 60fc4977..33345c1b 100644 --- a/concurrency/tests/actor_scenarios.rs +++ b/concurrency/tests/actor_scenarios.rs @@ -164,8 +164,14 @@ impl Handle { fn abandon(&self, cmd: Command) { match self { #[cfg(feature = "async")] - Self::Tasks(a) => drop(a.request_raw(cmd).expect("live actor")), - Self::Threads(a) => drop(a.request_raw(cmd).expect("live actor")), + Self::Tasks(a) => drop( + a.send_request(cmd, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .expect("live actor"), + ), + Self::Threads(a) => drop( + a.send_request(cmd, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .expect("live actor"), + ), } } } diff --git a/concurrency/tests/cross_backend.rs b/concurrency/tests/cross_backend.rs index e894a417..90bf9afa 100644 --- a/concurrency/tests/cross_backend.rs +++ b/concurrency/tests/cross_backend.rs @@ -350,7 +350,7 @@ mod threads_node { .iter() .enumerate() .map(|(i, peer)| { - peer.request_raw_until( + peer.send_request( Cmd::Echo(v + i as u64), spawned_concurrency::Deadline::after(LIMIT), ) diff --git a/concurrency/tests/deadline_service.rs b/concurrency/tests/deadline_service.rs index 22234437..d54fd817 100644 --- a/concurrency/tests/deadline_service.rs +++ b/concurrency/tests/deadline_service.rs @@ -97,7 +97,7 @@ fn deadline_service_child() { // Another actor's deadlines, serviced by the same thread. let other = Owner::default().start(); let parked = other - .request_raw_until(Park, Deadline::after(Duration::from_millis(50))) + .send_request(Park, Deadline::after(Duration::from_millis(50))) .unwrap(); match tokio::time::timeout(Duration::from_secs(5), parked).await { Ok(Err(ActorError::RequestTimeout)) => {} diff --git a/concurrency/tests/death_boundary.rs b/concurrency/tests/death_boundary.rs index ee32b17b..88ecac82 100644 --- a/concurrency/tests/death_boundary.rs +++ b/concurrency/tests/death_boundary.rs @@ -213,7 +213,7 @@ mod tasks_mode { actor.send(Block(blocked, entered_tx)).unwrap(); entered.recv_timeout(LIMIT).unwrap(); let queued = actor - .request_raw_until(Queued(inert()), spawned_concurrency::Deadline::Never) + .send_request(Queued(inert()), spawned_concurrency::Deadline::Never) .unwrap(); *caller.lock().unwrap() = Some(queued); actor.context().stop(); @@ -261,9 +261,9 @@ mod tasks_mode { } .start(); let turn_caller = actor - .request_raw_until(Hang(payload, entered_tx), never) + .send_request(Hang(payload, entered_tx), never) .unwrap(); - let queued = actor.request_raw_until(Queued(inert()), never).unwrap(); + let queued = actor.send_request(Queued(inert()), never).unwrap(); while entered.try_recv().is_err() { tokio::time::sleep(Duration::from_millis(1)).await; } @@ -326,8 +326,8 @@ mod tasks_mode { actor.send(Block(blocked, entered_tx)).unwrap(); entered.recv_timeout(LIMIT).unwrap(); let never = spawned_concurrency::Deadline::Never; - let first = actor.request_raw_until(Queued(inert()), never).unwrap(); - let second = actor.request_raw_until(Queued(inert()), never).unwrap(); + let first = actor.send_request(Queued(inert()), never).unwrap(); + let second = actor.send_request(Queued(inert()), never).unwrap(); let waiter = Arc::new(WaitForSecond { second: Mutex::new(second), answered: AtomicBool::new(false), @@ -386,7 +386,7 @@ mod tasks_mode { .start(); let (tokens, token) = mpsc::channel(); let receiver = actor - .request_raw_until(HandOut(tokens), spawned_concurrency::Deadline::Never) + .send_request(HandOut(tokens), spawned_concurrency::Deadline::Never) .unwrap(); let token = token.recv_timeout(LIMIT).unwrap(); let exited = Arc::new(AtomicBool::new(false)); @@ -438,8 +438,18 @@ mod tasks_mode { actor.send(Block(blocked, entered_tx)).unwrap(); entered.recv_timeout(LIMIT).unwrap(); let (payload, destroying, release) = gated(); - let mut first = actor.request_raw(Queued(payload)).unwrap(); - let mut second = actor.request_raw(Queued(inert())).unwrap(); + let mut first = actor + .send_request( + Queued(payload), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); + let mut second = actor + .send_request( + Queued(inert()), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); actor.context().stop(); unblock.send(()).unwrap(); destroying.recv_timeout(LIMIT).unwrap(); @@ -474,7 +484,12 @@ mod tasks_mode { } .start() }; - let mut queued = actor.request_raw(Queued(inert())).unwrap(); + let mut queued = actor + .send_request( + Queued(inert()), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); let dropper = std::thread::spawn(move || drop(runtime)); destroying.recv_timeout(LIMIT).unwrap(); eventually("a queued caller waited for the state destructor", || { @@ -502,7 +517,9 @@ mod tasks_mode { parked: Vec::new(), } .start(); - let parked = actor.request_raw(Park).unwrap(); + let parked = actor + .send_request(Park, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); // Let the actor defer it. actor.request(Queued(inert())).await.unwrap(); (actor, parked) @@ -578,7 +595,7 @@ mod threads_mode { actor.send(Block(blocked, entered_tx)).unwrap(); entered.recv_timeout(LIMIT).unwrap(); let queued = actor - .request_raw_until(Queued(inert()), spawned_concurrency::Deadline::Never) + .send_request(Queued(inert()), spawned_concurrency::Deadline::Never) .unwrap(); *caller.lock().unwrap() = Some(queued); actor.context().stop(); @@ -599,8 +616,18 @@ mod threads_mode { actor.send(Block(blocked, entered_tx)).unwrap(); entered.recv_timeout(LIMIT).unwrap(); let (payload, destroying, release) = gated(); - let mut first = actor.request_raw(Queued(payload)).unwrap(); - let mut second = actor.request_raw(Queued(inert())).unwrap(); + let mut first = actor + .send_request( + Queued(payload), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); + let mut second = actor + .send_request( + Queued(inert()), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); actor.context().stop(); unblock.send(()).unwrap(); destroying.recv_timeout(LIMIT).unwrap(); @@ -703,7 +730,7 @@ mod subscriber_after_discharge { Idle.start() }; let queued = actor - .request_raw_until(Queued(inert()), spawned_concurrency::Deadline::Never) + .send_request(Queued(inert()), spawned_concurrency::Deadline::Never) .unwrap(); *QUEUED.lock().unwrap() = Some(queued); drop(runtime); diff --git a/concurrency/tests/deferred_api.rs b/concurrency/tests/deferred_api.rs index d97c7c2a..4aeff771 100644 --- a/concurrency/tests/deferred_api.rs +++ b/concurrency/tests/deferred_api.rs @@ -618,7 +618,7 @@ mod threads_mode { mod caller_gone_vs_shortening_tasks { use spawned_concurrency::message::Message; use spawned_concurrency::tasks::{ - Actor, ActorStart, Context, Handler, Instant, JobError, JobHandle, JobOptions, JobOutcome, + Actor, ActorStart, Context, Handler, JobError, JobHandle, JobOptions, JobOutcome, PendingReply, ReplyTo, }; use std::time::Duration; @@ -706,7 +706,7 @@ mod caller_gone_vs_shortening_tasks { async fn handle(&mut self, msg: Delegated, _: PendingReply<'_, ()>, _: &Context) { let Delegated(outcome, token) = msg; let deadline = matches!(outcome, Err(JobError::DeadlineExceeded)); - token.with_deadline(Instant::now() + Duration::from_secs(30)); + token.shorten_deadline(Duration::from_secs(30)); let cancelled = self.job.as_ref().unwrap().is_cancelled(); token.reply((deadline, cancelled)); } @@ -722,7 +722,9 @@ mod caller_gone_vs_shortening_tasks { assert!(deadline, "the job deadline handed the token back"); assert!(!cancelled, "a shortened deadline cancelled the job"); // The caller leaving still cancels a blocking linked job. - let left = actor.request_raw(Leave).unwrap(); + let left = actor + .send_request(Leave, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); assert!( !actor.request(Probe).await.unwrap(), "cancelled before the caller left" @@ -746,7 +748,7 @@ mod caller_gone_vs_shortening_tasks { mod caller_gone_vs_shortening_threads { use spawned_concurrency::message::Message; use spawned_concurrency::threads::{ - Actor, ActorStart, Context, Handler, Instant, JobError, JobHandle, JobOptions, JobOutcome, + Actor, ActorStart, Context, Handler, JobError, JobHandle, JobOptions, JobOutcome, PendingReply, ReplyTo, }; use std::time::Duration; @@ -829,7 +831,7 @@ mod caller_gone_vs_shortening_threads { fn handle(&mut self, msg: Delegated, _: PendingReply<'_, ()>, _: &Context) { let Delegated(outcome, token) = msg; let deadline = matches!(outcome, Err(JobError::DeadlineExceeded)); - token.with_deadline(Instant::now() + Duration::from_secs(30)); + token.shorten_deadline(Duration::from_secs(30)); let cancelled = self.job.as_ref().unwrap().is_cancelled(); token.reply((deadline, cancelled)); } @@ -844,7 +846,9 @@ mod caller_gone_vs_shortening_threads { assert!(deadline, "the job deadline handed the token back"); assert!(!cancelled, "a shortened deadline cancelled the job"); // The caller leaving still cancels a blocking linked job. - let left = actor.request_raw(Leave).unwrap(); + let left = actor + .send_request(Leave, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); assert!( !actor.request(Probe).unwrap(), "cancelled before the caller left" diff --git a/concurrency/tests/deferred_jobs.rs b/concurrency/tests/deferred_jobs.rs index d98a2bbf..9c493c5c 100644 --- a/concurrency/tests/deferred_jobs.rs +++ b/concurrency/tests/deferred_jobs.rs @@ -481,10 +481,7 @@ mod tasks_mode { impl Handler for Router { async fn handle(&mut self, msg: Route, reply: PendingReply<'_, u64>, ctx: &Context) { let Ok(token) = reply.defer() else { return }; - let downstream = self.backend.request_raw_until( - Lookup(msg.0), - token.deadline().map_or(Deadline::Never, Deadline::At), - ); + let downstream = self.backend.send_request(Lookup(msg.0), token.deadline()); let work = async move { match downstream { Ok(rx) => rx.await, @@ -580,7 +577,9 @@ mod tasks_mode { backend: backend.clone(), } .start(); - let slow = router.request_raw(Route(200)).unwrap(); + let slow = router + .send_request(Route(200), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); // The router's mailbox is free while the slow request is delegated. assert_eq!(router.request(Ping).await.unwrap(), "pong"); assert_eq!(router.request(Route(1)).await.unwrap(), 10); @@ -599,7 +598,7 @@ mod tasks_mode { } .start(); let holder = Holder(None).start(); - let rx = holder.request_raw_until(Hold, Deadline::Never).unwrap(); + let rx = holder.send_request(Hold, Deadline::Never).unwrap(); let token = holder.request(Give).await.unwrap().unwrap(); assert!(router.request(Foreign(token)).await.unwrap()); assert_eq!( @@ -688,9 +687,7 @@ mod threads_mode { fn native_jobs_answer_deferred_requests() { let log = Arc::new(Log::default()); let actor: ActorRef = Owner { log: log.clone() }.start(); - let rx = actor - .request_raw_until(Compute(7), Deadline::Never) - .unwrap(); + let rx = actor.send_request(Compute(7), Deadline::Never).unwrap(); assert_eq!(rx.recv().unwrap(), 49); assert!(matches!( actor.request(Compute(0)), diff --git a/concurrency/tests/deferred_replies.rs b/concurrency/tests/deferred_replies.rs index 621dec17..d73e4eda 100644 --- a/concurrency/tests/deferred_replies.rs +++ b/concurrency/tests/deferred_replies.rs @@ -171,8 +171,8 @@ mod tasks_mode { #[spawned_macros::sim_test] async fn deferred_reply_answers_from_a_later_turn() { let (actor, _) = start(); - let a = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); - let b = actor.request_raw_until(Park(2), Deadline::Never).unwrap(); + let a = actor.send_request(Park(1), Deadline::Never).unwrap(); + let b = actor.send_request(Park(2), Deadline::Never).unwrap(); assert_eq!(settle(&actor).await, 2); // The mailbox kept serving while both requests were parked. assert_eq!(actor.request(Release(10)).await.unwrap(), 2); @@ -186,7 +186,7 @@ mod tasks_mode { #[spawned_macros::sim_test] async fn dropped_token_between_turns_answers_reply_dropped() { let (actor, _) = start(); - let rx = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); + let rx = actor.send_request(Park(1), Deadline::Never).unwrap(); actor.request(DropAll).await.unwrap(); assert!(matches!(rx.await, Err(ActorError::ReplyDropped))); assert_eq!(settle(&actor).await, 0); @@ -222,8 +222,8 @@ mod tasks_mode { #[spawned_macros::sim_test] async fn defer_at_the_bound_answers_overloaded() { let (actor, seen) = start(); - let a = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); - let b = actor.request_raw_until(Park(2), Deadline::Never).unwrap(); + let a = actor.send_request(Park(1), Deadline::Never).unwrap(); + let b = actor.send_request(Park(2), Deadline::Never).unwrap(); // MAX_OUTSTANDING is 2: the third deferral is refused in the same step. assert!(matches!( actor.request(Park(3)).await, @@ -242,7 +242,7 @@ mod tasks_mode { #[spawned_macros::sim_test] async fn abandoned_caller_returns_the_credit_and_drops_the_late_reply() { let (actor, seen) = start(); - let rx = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); + let rx = actor.send_request(Park(1), Deadline::Never).unwrap(); assert_eq!(settle(&actor).await, 1); drop(rx); assert_eq!( @@ -261,7 +261,7 @@ mod tasks_mode { let (actor, seen) = start(); actor.send(TryDefer).unwrap(); let expired = actor - .request_raw_until(TryDefer, Deadline::after(Duration::ZERO)) + .send_request(TryDefer, Deadline::after(Duration::ZERO)) .unwrap(); assert!(matches!(expired.await, Err(ActorError::RequestTimeout))); settle(&actor).await; @@ -277,7 +277,7 @@ mod tasks_mode { async fn late_reply_after_the_deadline_is_expired() { let (actor, seen) = start(); let rx = actor - .request_raw_until(Park(1), Deadline::after(Duration::from_millis(50))) + .send_request(Park(1), Deadline::after(Duration::from_millis(50))) .unwrap(); assert!(matches!(rx.await, Err(ActorError::RequestTimeout))); actor.request(Release(0)).await.unwrap(); @@ -293,7 +293,7 @@ mod tasks_mode { // Nobody polls this receiver: only the owner's deadline heap can // expire the token and return its credit. let rx = actor - .request_raw_until(Park(1), Deadline::after(Duration::from_millis(20))) + .send_request(Park(1), Deadline::after(Duration::from_millis(20))) .unwrap(); assert_eq!(settle(&actor).await, 1); let mut waited = Duration::ZERO; @@ -321,7 +321,7 @@ mod tasks_mode { .unwrap(); let (actor, rx) = rt.block_on(async { let (actor, _) = start(); - let rx = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); + let rx = actor.send_request(Park(1), Deadline::Never).unwrap(); assert_eq!(settle(&actor).await, 1); (actor, rx) }); @@ -344,7 +344,7 @@ mod tasks_mode { #[spawned_macros::sim_test] async fn stopped_may_answer_stored_tokens() { let (actor, _) = start(); - let rx = actor.request_raw_until(Park(7), Deadline::Never).unwrap(); + let rx = actor.send_request(Park(7), Deadline::Never).unwrap(); actor.request(Stop { answer: true }).await.unwrap(); assert_eq!(rx.await.unwrap(), 1007); actor.join().await; @@ -353,10 +353,10 @@ mod tasks_mode { #[spawned_macros::sim_test] async fn death_answers_stored_tokens_actor_stopped() { let (actor, _) = start(); - let rx = actor.request_raw_until(Park(7), Deadline::Never).unwrap(); + let rx = actor.send_request(Park(7), Deadline::Never).unwrap(); let queued_behind_stop = { actor.request(Stop { answer: false }).await.unwrap(); - actor.request_raw_until(Park(8), Deadline::Never) + actor.send_request(Park(8), Deadline::Never) }; assert!(matches!( rx.await, @@ -414,7 +414,7 @@ mod tasks_mode { } } let actor: ActorRef = Watcher { closed: None }.start(); - let rx = actor.request_raw_until(Watch, Deadline::Never).unwrap(); + let rx = actor.send_request(Watch, Deadline::Never).unwrap(); actor.request(Ping).await.unwrap(); // `Watch` was handled (FIFO) drop(rx); assert!(actor.request(Done).await.unwrap()); @@ -433,7 +433,7 @@ mod tasks_mode { spawned_rt::tasks::spawn(async move { token.closed().await }) }; sleep(Duration::from_millis(5)).await; - token.with_deadline(spawned_rt::time::Instant::now() + Duration::from_millis(20)); + token.shorten_deadline(Duration::from_millis(20)); spawned_rt::tasks::timeout(Duration::from_secs(5), watcher) .await .expect("closed() missed the shortened deadline") @@ -458,7 +458,7 @@ mod tasks_mode { impl Handler for Shortener { async fn handle(&mut self, _: Shorten, reply: PendingReply<'_, ()>, _: &Context) { let token = reply.defer().unwrap(); - token.with_deadline(spawned_rt::time::Instant::now() + Duration::from_millis(50)); + token.shorten_deadline(Duration::from_millis(50)); self.0 = Some(token); } } @@ -513,7 +513,7 @@ mod tasks_mode { let (actor, _) = start(); // Sent now with a short deadline, polled only after it passed. let response = actor - .request_raw_until(Park(1), Deadline::after(Duration::from_millis(10))) + .send_request(Park(1), Deadline::after(Duration::from_millis(10))) .unwrap(); spawned_rt::tasks::sleep(Duration::from_millis(50)).await; assert!(matches!(response.await, Err(ActorError::RequestTimeout))); @@ -626,12 +626,12 @@ mod threads_mode { fn every_discharge_path() { // Later turn. let (actor, seen) = start(); - let a = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); + let a = actor.send_request(Park(1), Deadline::Never).unwrap(); assert_eq!(actor.request(Outstanding).unwrap(), 1); assert_eq!(actor.request(Release(10)).unwrap(), 1); assert_eq!(a.recv().unwrap(), 11); // Dropped between turns. - let b = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); + let b = actor.send_request(Park(1), Deadline::Never).unwrap(); actor.request(DropAll).unwrap(); assert!(matches!(b.recv(), Err(ActorError::ReplyDropped))); // Unanswered turn. @@ -640,8 +640,8 @@ mod threads_mode { Err(ActorError::ReplyDropped) )); // Bound. - let c = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); - let d = actor.request_raw_until(Park(2), Deadline::Never).unwrap(); + let c = actor.send_request(Park(1), Deadline::Never).unwrap(); + let d = actor.send_request(Park(2), Deadline::Never).unwrap(); assert!(matches!( actor.request(Park(3)), Err(ActorError::Overloaded) @@ -653,7 +653,7 @@ mod threads_mode { assert_eq!(*seen.defer_errors.lock().unwrap(), [DeferError::Overloaded]); // Expired deadline: the receiver expires the slot; the late reply is dropped. let e = actor - .request_raw_until(Park(4), Deadline::after(Duration::from_millis(20))) + .send_request(Park(4), Deadline::after(Duration::from_millis(20))) .unwrap(); assert!(matches!(e.recv(), Err(ActorError::RequestTimeout))); actor.request(Release(0)).unwrap(); @@ -664,7 +664,7 @@ mod threads_mode { ); assert_eq!(actor.request(Outstanding).unwrap(), 0); // stopped() may answer; then death closes the rest. - let f = actor.request_raw_until(Park(7), Deadline::Never).unwrap(); + let f = actor.send_request(Park(7), Deadline::Never).unwrap(); actor.request(Stop { answer: true }).unwrap(); assert_eq!(f.recv().unwrap(), 1007); actor.join(); @@ -679,7 +679,7 @@ mod threads_mode { fn unpolled_deferred_token_expires_by_the_owner_heap() { let (actor, seen) = start(); let rx = actor - .request_raw_until(Park(1), Deadline::after(Duration::from_millis(20))) + .send_request(Park(1), Deadline::after(Duration::from_millis(20))) .unwrap(); let began = std::time::Instant::now(); while actor.request(Outstanding).unwrap() != 0 { @@ -707,7 +707,7 @@ mod threads_mode { actor.join(); } let (actor, _) = start(); - let parked = actor.request_raw_until(Park(1), Deadline::Never).unwrap(); + let parked = actor.send_request(Park(1), Deadline::Never).unwrap(); actor.request(Stop { answer: false }).unwrap(); assert!(matches!( parked.recv(), diff --git a/concurrency/tests/deferred_sim.rs b/concurrency/tests/deferred_sim.rs index 87d035b2..a6967974 100644 --- a/concurrency/tests/deferred_sim.rs +++ b/concurrency/tests/deferred_sim.rs @@ -180,7 +180,7 @@ async fn reply_vs_expiry_vs_death_vs_drop(seed: u64) { let actor = owner(&log); let d = Duration::from_millis(10); let receiver = actor - .request_raw_until(Hold, Deadline::At(spawned_rt::time::Instant::now() + d)) + .send_request(Hold, Deadline::At(spawned_rt::time::Instant::now() + d)) .unwrap(); actor.request(Sync).await.unwrap(); let replier = actor.clone(); diff --git a/concurrency/tests/execution_groups.rs b/concurrency/tests/execution_groups.rs index edfd7566..fe5eab65 100644 --- a/concurrency/tests/execution_groups.rs +++ b/concurrency/tests/execution_groups.rs @@ -393,7 +393,12 @@ async fn failed_startup_discards_queued_payloads_and_reply_channels_without_disp .await .unwrap() .unwrap(); - let reply = actor.request_raw(Queued(Probe(dropped.clone()))).unwrap(); + let reply = actor + .send_request( + Queued(Probe(dropped.clone())), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); actor.send(Queued(Probe(dropped.clone()))).unwrap(); assert_eq!(handled.load(Ordering::SeqCst), 0); release.send(()).unwrap(); diff --git a/concurrency/tests/failpoints.rs b/concurrency/tests/failpoints.rs index 4e69bee2..dbf17be8 100644 --- a/concurrency/tests/failpoints.rs +++ b/concurrency/tests/failpoints.rs @@ -969,7 +969,9 @@ async fn no_reply_succeeds_after_noproc_is_observable() { let _scenario = FailScenario::setup(); let holder = Holder.start(); let (tokens, token) = std::sync::mpsc::channel(); - let caller = holder.request_raw(Hold(tokens)).unwrap(); + let caller = holder + .send_request(Hold(tokens), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); let token = token.recv_timeout(LIMIT).unwrap(); failpoints::configure( "actor.at_death", @@ -1136,7 +1138,9 @@ mod native { let _scenario = FailScenario::setup(); let holder = NativeHolder.start(); let (tokens, token) = std::sync::mpsc::channel(); - let caller = holder.request_raw(Hold(tokens)).unwrap(); + let caller = holder + .send_request(Hold(tokens), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); let token = token.recv_timeout(LIMIT).unwrap(); failpoints::configure( "actor.at_death", diff --git a/concurrency/tests/foreign_panics.rs b/concurrency/tests/foreign_panics.rs index fdef017c..aba3f359 100644 --- a/concurrency/tests/foreign_panics.rs +++ b/concurrency/tests/foreign_panics.rs @@ -376,7 +376,10 @@ mod env { task.abort(); let _ = env.bounded(task); } - Started::Native(a) => drop(a.request_raw(Gift).unwrap()), + Started::Native(a) => drop( + a.send_request(Gift, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(), + ), } gate.release(); } diff --git a/concurrency/tests/full_actor_shards.rs b/concurrency/tests/full_actor_shards.rs index 60e90467..3b2bc898 100644 --- a/concurrency/tests/full_actor_shards.rs +++ b/concurrency/tests/full_actor_shards.rs @@ -151,13 +151,18 @@ fn stop_finishes_in_flight_handler_and_rejects_queued_work() { let (started, rx) = tokio::sync::oneshot::channel(); let (release, gate) = tokio::sync::oneshot::channel(); let reply = actor - .request_raw(Hold { - started, - release: gate, - }) + .send_request( + Hold { + started, + release: gate, + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) .unwrap(); tokio::time::timeout(LIMIT, rx).await.unwrap().unwrap(); - let queued = actor.request_raw(Thread).unwrap(); + let queued = actor + .send_request(Thread, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); actor.child_handle().stop(); assert!(actor.exit_reason().is_none()); release.send(()).unwrap(); @@ -205,10 +210,13 @@ fn suspended_actor_does_not_block_another_actor_on_its_shard() { let (started, rx) = tokio::sync::oneshot::channel(); let (release, gate) = tokio::sync::oneshot::channel(); let held = actors[a] - .request_raw(Hold { - started, - release: gate, - }) + .send_request( + Hold { + started, + release: gate, + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) .unwrap(); tokio::time::timeout(LIMIT, rx).await.unwrap().unwrap(); assert_eq!( @@ -302,13 +310,18 @@ fn blocking_handler_stalls_its_shard_while_another_shard_progresses() { let (started, rx) = tokio::sync::oneshot::channel(); let (release, gate) = std::sync::mpsc::channel(); let frozen = actors[a] - .request_raw(Freeze { - started, - release: gate, - }) + .send_request( + Freeze { + started, + release: gate, + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) .unwrap(); tokio::time::timeout(LIMIT, rx).await.unwrap().unwrap(); - let mut queued = actors[b].request_raw(Thread).unwrap(); + let mut queued = actors[b] + .send_request(Thread, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); assert!(queued.try_recv().is_none()); if let Some(other) = (0..actors.len()).find(|&id| names[id] != names[a]) { assert_eq!( diff --git a/concurrency/tests/lifecycle_trace.rs b/concurrency/tests/lifecycle_trace.rs index 2e665b7e..b032b722 100644 --- a/concurrency/tests/lifecycle_trace.rs +++ b/concurrency/tests/lifecycle_trace.rs @@ -232,9 +232,13 @@ async fn kill_answers_queued_request_at_death() { let log = Log::default(); let a1 = Node::new(&log).start(); let (release, gate) = tokio::sync::oneshot::channel(); - let blocked = a1.request_raw(Block(gate)).unwrap(); + let blocked = a1 + .send_request(Block(gate), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); wait_for(&log, "blocked").await; - let queued = a1.request_raw(Sync()).unwrap(); + let queued = a1 + .send_request(Sync(), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); a1.child_handle().kill(); release.send(()).unwrap(); assert!(blocked.await.is_ok()); diff --git a/concurrency/tests/mailbox_drop_panic.rs b/concurrency/tests/mailbox_drop_panic.rs index 5677bbfc..2fce7ed1 100644 --- a/concurrency/tests/mailbox_drop_panic.rs +++ b/concurrency/tests/mailbox_drop_panic.rs @@ -151,7 +151,14 @@ fn mailbox_drop_child() { victim.send(Hold(entered, gate, handler_panics)).unwrap(); tokio::time::timeout(LIMIT, arrival).await.unwrap().unwrap(); let replies: Vec<_> = (0..bombs) - .map(|_| victim.request_raw(Bomb(dropped.clone())).unwrap()) + .map(|_| { + victim + .send_request( + Bomb(dropped.clone()), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap() + }) .collect(); victim.context().stop(); release.send(()).unwrap(); @@ -265,7 +272,14 @@ fn abandoned_receiver_child() { } let dropped = Arc::new(AtomicUsize::new(0)); let replies: Vec<_> = (0..2) - .map(|_| actor.request_raw(Bomb(dropped.clone())).unwrap()) + .map(|_| { + actor + .send_request( + Bomb(dropped.clone()), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap() + }) .collect(); shutdown.send(()).unwrap(); let reason = tokio::time::timeout(LIMIT, actor.wait_exit()) diff --git a/concurrency/tests/metrics.rs b/concurrency/tests/metrics.rs index 032cfef8..7a5424a8 100644 --- a/concurrency/tests/metrics.rs +++ b/concurrency/tests/metrics.rs @@ -98,7 +98,13 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { runtime().block_on(async { let (release, gate) = oneshot::channel(); let actor = Probe { gate: Some(gate) }.start(); - let replies: Vec<_> = (0..10).map(|_| actor.request_raw(Ping).unwrap()).collect(); + let replies: Vec<_> = (0..10) + .map(|_| { + actor + .send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap() + }) + .collect(); assert_eq!(actor.mailbox_depth(), 10); assert_eq!(actor.context().mailbox_depth(), 10); assert!(matches!( @@ -107,13 +113,16 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { )); assert!(matches!( Response::from( - actor.request_raw_until(Ping, spawned_concurrency::Deadline::after(Duration::ZERO)) + actor.send_request(Ping, spawned_concurrency::Deadline::after(Duration::ZERO)) ) .await, Err(ActorError::RequestTimeout) )); // Dropping either kind of caller wrapper does not cancel actor work. - drop(Response::from(actor.request_raw(Ping))); + drop(Response::from(actor.send_request( + Ping, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ))); let mut cancelled = Box::pin(actor.request(Ping)); assert!(futures::poll!(cancelled.as_mut()).is_pending()); drop(cancelled); @@ -130,8 +139,10 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { spawned_concurrency::tasks::request(recipient.as_ref(), Ping, Duration::from_secs(2)) .await .unwrap(); - Response::from(actor.request_raw(Ping)).await.unwrap(); - Response::from(actor.request_raw_until( + Response::from(actor.send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT)) + .await + .unwrap(); + Response::from(actor.send_request( Ping, spawned_concurrency::Deadline::after(Duration::from_secs(2)), )) @@ -142,7 +153,7 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { assert_eq!(actor.mailbox_depth(), 0); assert!(actor.send(Ping).is_err()); assert!(matches!( - Response::from(actor.request_raw_until( + Response::from(actor.send_request( Ping, spawned_concurrency::Deadline::after(Duration::from_secs(2)) )) @@ -152,7 +163,9 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { let (release, gate) = oneshot::channel(); let actor = Probe { gate: Some(gate) }.start(); - let raw = actor.request_raw(Ping).unwrap(); + let raw = actor + .send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); actor.send(Ping).unwrap(); let mut pending = Box::pin(actor.request(Ping)); assert!(futures::poll!(pending.as_mut()).is_pending()); diff --git a/concurrency/tests/metrics_deferred.rs b/concurrency/tests/metrics_deferred.rs index d9ee05df..3ebb48a9 100644 --- a/concurrency/tests/metrics_deferred.rs +++ b/concurrency/tests/metrics_deferred.rs @@ -177,8 +177,12 @@ async fn deferred_and_job_families_count_each_outcome_once() { let actor = Owner::default().start(); // Replied, and Overloaded at the bound. - let first = actor.request_raw(Hold).unwrap(); - let second = actor.request_raw(Hold).unwrap(); + let first = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + let second = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); assert!(matches!( actor.request(Hold).await, Err(ActorError::Overloaded) @@ -188,12 +192,16 @@ async fn deferred_and_job_families_count_each_outcome_once() { assert!(matches!(second.await, Ok(1))); // Expired, dropped, abandoned. let deadline = Deadline::At(spawned_rt::time::Instant::now() + Duration::from_millis(20)); - let expiring = actor.request_raw_until(Hold, deadline).unwrap(); + let expiring = actor.send_request(Hold, deadline).unwrap(); assert!(matches!(expiring.await, Err(ActorError::RequestTimeout))); - let dropped = actor.request_raw(Hold).unwrap(); + let dropped = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); actor.request(Flush::Drop).await.unwrap(); assert!(matches!(dropped.await, Err(ActorError::ReplyDropped))); - let abandoned = actor.request_raw(Hold).unwrap(); + let abandoned = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); actor.request(Sync).await.unwrap(); drop(abandoned); actor.request(Flush::Reply).await.unwrap(); @@ -213,9 +221,13 @@ async fn deferred_and_job_families_count_each_outcome_once() { } // Stopped: a token still stored at death. - let stopped = actor.request_raw(Hold).unwrap(); + let stopped = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); actor.request(Flush::Reply).await.unwrap(); - let late = actor.request_raw(Hold).unwrap(); + let late = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); sleep(Duration::from_millis(20)).await; actor.context().stop(); assert!(matches!( diff --git a/concurrency/tests/native_mailbox_drop.rs b/concurrency/tests/native_mailbox_drop.rs index 0e0c62b4..0d472dc9 100644 --- a/concurrency/tests/native_mailbox_drop.rs +++ b/concurrency/tests/native_mailbox_drop.rs @@ -97,7 +97,14 @@ fn native_drop_child() { arrival.recv_timeout(LIMIT).unwrap(); let dropped = Arc::new(AtomicUsize::new(0)); let replies: Vec<_> = (0..2) - .map(|_| actor.request_raw(Bomb(dropped.clone())).unwrap()) + .map(|_| { + actor + .send_request( + Bomb(dropped.clone()), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap() + }) .collect(); actor.context().stop_with_reason(reason.clone()); release.send(()).unwrap(); diff --git a/concurrency/tests/otp_exit_signals.rs b/concurrency/tests/otp_exit_signals.rs index 83630f10..980fab9f 100644 --- a/concurrency/tests/otp_exit_signals.rs +++ b/concurrency/tests/otp_exit_signals.rs @@ -904,10 +904,14 @@ mod tasks_mode { async fn queued_and_stored_requests_get_killed_after_a_kill() { let seen = Seen::default(); let actor = Node::new(&seen, false).start(); - let stored = actor.request_raw(Hold).unwrap(); + let stored = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); settle(&actor).await; let release = block(&actor).await; - let queued = actor.request_raw(Ping).unwrap(); + let queued = actor + .send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); let child = actor.child_handle(); child.kill(); release.send(()).unwrap(); @@ -923,7 +927,9 @@ mod tasks_mode { let actor = Node::new(&seen, false).start(); settle(&actor).await; let release = block(&actor).await; - let queued = actor.request_raw(Ping).unwrap(); + let queued = actor + .send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); let child = actor.child_handle(); let reason = ExitReason::Panic("signalled".into()); child.exit(reason.clone()); @@ -945,7 +951,9 @@ mod tasks_mode { node.in_stopped = Some((entered_tx, release_rx)); let actor = node.start(); settle(&actor).await; - let mut crashed = actor.request_raw(Boom).unwrap(); + let mut crashed = actor + .send_request(Boom, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); entered.await.unwrap(); tokio::time::sleep(Duration::from_millis(50)).await; assert!( @@ -978,7 +986,9 @@ mod tasks_mode { node.in_stopped = Some((entered_tx, release_rx)); node.drop_held_in_stopped = true; let actor = node.start(); - let mut stored = actor.request_raw(Hold).unwrap(); + let mut stored = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); settle(&actor).await; let child = actor.child_handle(); child.stop_with_reason(ExitReason::Shutdown); @@ -1008,14 +1018,19 @@ mod tasks_mode { let child = actor.child_handle(); child.stop_with_reason(ExitReason::Shutdown); entered.await.unwrap(); - let admitted = actor.request_raw(Ping).expect("admitted while exiting"); + let admitted = actor + .send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .expect("admitted while exiting"); actor.send(Ping).expect("a send while exiting is admitted"); release.send(()).unwrap(); assert_eq!(ended(&child).await, ExitReason::Shutdown); stopped_with(answered(admitted).await, &ExitReason::Shutdown); assert!(matches!(actor.send(Ping), Err(ActorError::NoProc))); assert!(matches!(actor.request(Ping).await, Err(ActorError::NoProc))); - assert!(matches!(actor.request_raw(Ping), Err(ActorError::NoProc))); + assert!(matches!( + actor.send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT), + Err(ActorError::NoProc) + )); let recipient = actor.recipient::(); assert!(matches!(recipient.send(Ping), Err(ActorError::NoProc))); } @@ -1385,11 +1400,15 @@ mod threads_mode { #[test] fn queued_and_stored_requests_get_killed_after_a_kill() { let actor = Server::default().start(); - let stored = actor.request_raw(Hold).unwrap(); + let stored = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); assert_eq!(actor.request(Ping).unwrap(), 7); let (release, gate) = mpsc::channel(); actor.send(Block(gate)).unwrap(); - let queued = actor.request_raw(Ping).unwrap(); + let queued = actor + .send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); actor.child_handle().kill(); release.send(()).unwrap(); assert_eq!(ended(&actor.child_handle()), ExitReason::Killed); @@ -1401,8 +1420,12 @@ mod threads_mode { fn panicked_request_and_dropped_token_wait_for_death() { let (server, entered, release) = gated(); let actor = server.start(); - let mut stored = actor.request_raw(Hold).unwrap(); - let mut crashed = actor.request_raw(Boom).unwrap(); + let mut stored = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + let mut crashed = actor + .send_request(Boom, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); entered.recv_timeout(LIMIT).unwrap(); std::thread::sleep(Duration::from_millis(50)); assert!( @@ -1427,14 +1450,19 @@ mod threads_mode { assert_eq!(actor.request(Ping).unwrap(), 7); actor.child_handle().stop_with_reason(ExitReason::Shutdown); entered.recv_timeout(LIMIT).unwrap(); - let admitted = actor.request_raw(Ping).expect("admitted while exiting"); + let admitted = actor + .send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .expect("admitted while exiting"); actor.send(Ping).expect("a send while exiting is admitted"); release.send(()).unwrap(); assert_eq!(ended(&actor.child_handle()), ExitReason::Shutdown); stopped_with(admitted, &ExitReason::Shutdown); assert!(matches!(actor.send(Ping), Err(ActorError::NoProc))); assert!(matches!(actor.request(Ping), Err(ActorError::NoProc))); - assert!(matches!(actor.request_raw(Ping), Err(ActorError::NoProc))); + assert!(matches!( + actor.send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT), + Err(ActorError::NoProc) + )); } } diff --git a/concurrency/tests/panic_safety.rs b/concurrency/tests/panic_safety.rs index 14800018..44da247a 100644 --- a/concurrency/tests/panic_safety.rs +++ b/concurrency/tests/panic_safety.rs @@ -614,7 +614,11 @@ mod threads_mode { self.victim.request_with_timeout(Ask, LIMIT).map_err(|_| ()) } fn victim_abandoned_gift(&self) { - drop(self.victim.request_raw(Gift).unwrap()); + drop( + self.victim + .send_request(Gift, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(), + ); } fn victim_setup(&self, setup: Setup) -> Option { self.victim.request_with_timeout(setup, LIMIT).unwrap() @@ -788,7 +792,11 @@ mod tasks_mode { .map_err(|_| ()) } fn victim_abandoned_gift(&self) { - drop(self.victim.request_raw(Gift).unwrap()); + drop( + self.victim + .send_request(Gift, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(), + ); } fn victim_setup(&self, setup: Setup) -> Option { self.env diff --git a/concurrency/tests/reply_trace.rs b/concurrency/tests/reply_trace.rs index 9f869705..657dbbb5 100644 --- a/concurrency/tests/reply_trace.rs +++ b/concurrency/tests/reply_trace.rs @@ -179,7 +179,9 @@ async fn reply_in_turn() { #[tokio::test] async fn stored_token_answered_later() { let actor = start(); - let held = actor.request_raw(Hold).unwrap(); + let held = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); actor.request(Sync).await.unwrap(); actor.send(Flush::Reply).unwrap(); let got = held.await; @@ -190,7 +192,9 @@ async fn stored_token_answered_later() { #[tokio::test] async fn stored_token_dropped_later() { let actor = start(); - let held = actor.request_raw(Hold).unwrap(); + let held = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); actor.request(Sync).await.unwrap(); actor.send(Flush::Drop).unwrap(); let got = held.await; @@ -209,7 +213,9 @@ async fn unused_obligation_answers_dropped() { #[tokio::test] async fn defer_at_the_bound_is_overloaded() { let actor = start(); - let first = actor.request_raw(Hold).unwrap(); + let first = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); let got = actor.request(Hold).await; assert!(matches!(got, Err(ActorError::Overloaded)), "{got:?}"); actor.send(Flush::Reply).unwrap(); @@ -222,7 +228,7 @@ async fn defer_at_the_bound_is_overloaded() { async fn stored_token_expires_at_its_deadline() { let actor = start(); let deadline = Deadline::At(spawned_rt::time::Instant::now() + Duration::from_millis(30)); - let held = actor.request_raw_until(Hold, deadline).unwrap(); + let held = actor.send_request(Hold, deadline).unwrap(); let got = held.await; assert!(matches!(got, Err(ActorError::RequestTimeout)), "{got:?}"); actor.send(Flush::Reply).unwrap(); @@ -233,7 +239,9 @@ async fn stored_token_expires_at_its_deadline() { #[tokio::test] async fn caller_gives_up() { let actor = start(); - let held = actor.request_raw(Hold).unwrap(); + let held = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); actor.request(Sync).await.unwrap(); drop(held); actor.send(Flush::Reply).unwrap(); @@ -244,7 +252,9 @@ async fn caller_gives_up() { #[tokio::test] async fn death_closes_stored_tokens() { let actor = start(); - let held = actor.request_raw(Hold).unwrap(); + let held = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); actor.request(Sync).await.unwrap(); let owner = id(&actor); actor.context().stop(); diff --git a/concurrency/tests/semantics_gaps.rs b/concurrency/tests/semantics_gaps.rs index 34665c6e..5932738a 100644 --- a/concurrency/tests/semantics_gaps.rs +++ b/concurrency/tests/semantics_gaps.rs @@ -222,7 +222,7 @@ mod native { let raw = Panicker.start(); let receiver = raw - .request_raw_until(Boom, spawned_concurrency::Deadline::after(LIMIT)) + .send_request(Boom, spawned_concurrency::Deadline::after(LIMIT)) .unwrap(); assert!(matches!( receiver.recv(), @@ -584,7 +584,9 @@ mod tasks { ); let raw = Panicker.start(); - let receiver = raw.request_raw(Boom).unwrap(); + let receiver = raw + .send_request(Boom, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); assert!(timeout(LIMIT, receiver).await.unwrap().is_err()); assert!(system.shutdown(LIMIT).await.complete); } @@ -790,7 +792,10 @@ mod native_contract { actor.send(Queued(Payload(drops.clone()))).expect("D3"); replies.push( actor - .request_raw(Asked(Payload(drops.clone()))) + .send_request( + Asked(Payload(drops.clone())), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) .expect("D3"), ); } @@ -1194,7 +1199,10 @@ mod tasks_contract { actor.send(Queued(Payload(drops.clone()))).expect("D3"); replies.push( actor - .request_raw(Asked(Payload(drops.clone()))) + .send_request( + Asked(Payload(drops.clone())), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) .expect("D3"), ); } @@ -1627,7 +1635,11 @@ mod tasks_contract { place, ); let replies: Vec<_> = (0..4) - .map(|i| actor.request_raw(Work(i)).unwrap()) + .map(|i| { + actor + .send_request(Work(i), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap() + }) .collect(); tokio::task::yield_now().await; assert!( @@ -2056,7 +2068,10 @@ mod supervision_contract { let mut replies = Vec::new(); for _ in 0..5 { old.send(Incr).unwrap(); - replies.push(old.request_raw(Next).unwrap()); + replies.push( + old.send_request(Next, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(), + ); } release.send(()).unwrap(); let new = next_generation(&child, generation).await; diff --git a/concurrency/tests/shutdown_publication.rs b/concurrency/tests/shutdown_publication.rs index 8147acb4..c8751c1d 100644 --- a/concurrency/tests/shutdown_publication.rs +++ b/concurrency/tests/shutdown_publication.rs @@ -106,10 +106,13 @@ fn admitted_request_racing_shutdown_drops_its_reply_before_join_completes() { let (entered, entered_rx) = tokio::sync::oneshot::channel(); let (release, gate) = tokio::sync::oneshot::channel(); let _hold_reply = actor - .request_raw(Command::Hold { - entered, - release: gate, - }) + .send_request( + Command::Hold { + entered, + release: gate, + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) .unwrap(); tokio::time::timeout(LIMIT, entered_rx) .await @@ -120,7 +123,8 @@ fn admitted_request_racing_shutdown_drops_its_reply_before_join_completes() { let producer = std::thread::spawn(move || { PRODUCER.with(|p| p.set(true)); for _ in 0..1024 { - let reply = sender.request_raw(Command::Work); + let reply = sender + .send_request(Command::Work, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT); if PAUSED.load(Ordering::SeqCst) { return reply; } diff --git a/concurrency/tests/sim_explore.rs b/concurrency/tests/sim_explore.rs index 7f47c2b4..b05062e9 100644 --- a/concurrency/tests/sim_explore.rs +++ b/concurrency/tests/sim_explore.rs @@ -141,7 +141,10 @@ async fn bump_all(noises: &[u32]) -> Vec { .collect(); let replies: Vec<_> = workers .iter() - .map(|w| w.request_raw(Bump).unwrap()) + .map(|w| { + w.send_request(Bump, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap() + }) .collect(); for reply in replies { reply.await.unwrap(); diff --git a/concurrency/tests/sim_ordering.rs b/concurrency/tests/sim_ordering.rs index c08115e1..0ad77bd6 100644 --- a/concurrency/tests/sim_ordering.rs +++ b/concurrency/tests/sim_ordering.rs @@ -450,7 +450,14 @@ async fn stop_discards_body() { // D3: accepted even after the stop request. actor.send(Queued(Payload(drops.clone()))).unwrap(); } else { - replies.push(actor.request_raw(Asked(Payload(drops.clone()))).unwrap()); + replies.push( + actor + .send_request( + Asked(Payload(drops.clone())), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(), + ); } yield_now().await; } diff --git a/concurrency/tests/startup_admission.rs b/concurrency/tests/startup_admission.rs index c83877f4..20cc4c02 100644 --- a/concurrency/tests/startup_admission.rs +++ b/concurrency/tests/startup_admission.rs @@ -89,7 +89,14 @@ async fn managed_actor_stopped_before_first_poll_runs_no_hooks_but_drops_state() let actor = Probe(counts.clone()).start_on(&group).unwrap(); let dropped = Arc::new(AtomicUsize::new(0)); let replies: Vec<_> = (0..2) - .map(|_| actor.request_raw(Queued(dropped.clone())).unwrap()) + .map(|_| { + actor + .send_request( + Queued(dropped.clone()), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap() + }) .collect(); actor.child_handle().shutdown(); release.send(()).unwrap(); diff --git a/concurrency/tests/supervision.rs b/concurrency/tests/supervision.rs index 7ee71153..038faedf 100644 --- a/concurrency/tests/supervision.rs +++ b/concurrency/tests/supervision.rs @@ -435,7 +435,9 @@ async fn stuck_stopped_hook_triggers_cleanup_deadline_before_terminal_publicatio .unwrap(); let supervisor = tree.start().await.unwrap(); let old = child.current().unwrap(); - let response = old.request_raw(Crash).unwrap(); + let response = old + .send_request(Crash, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); spawned_rt::tasks::timeout(LIMIT, entered) .await .unwrap() diff --git a/concurrency/tests/support/failure_actor.rs b/concurrency/tests/support/failure_actor.rs index 63845a4d..9e080ed1 100644 --- a/concurrency/tests/support/failure_actor.rs +++ b/concurrency/tests/support/failure_actor.rs @@ -336,10 +336,10 @@ impl Handle { match self { #[cfg(feature = "async")] Self::Async(actor) => actor - .request_raw_until(command, spawned_concurrency::Deadline::Never) + .send_request(command, spawned_concurrency::Deadline::Never) .map(Response::Async), Self::Native(actor) => actor - .request_raw_until(command, spawned_concurrency::Deadline::Never) + .send_request(command, spawned_concurrency::Deadline::Never) .map(Response::Native), } } diff --git a/docs/GUIDE.md b/docs/GUIDE.md index ac34de50..a9ee8d33 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -124,7 +124,7 @@ State that must be released on every exit, including kills and runtime drops, be | `ctx.send(msg)` | Send a fire-and-forget message to this actor (self-send) | | `ctx.request(msg)` | Send a request and wait for the reply (tasks: async, threads: blocking) | | `ctx.request_with_timeout(msg, duration)` | Send a request with a custom timeout | -| `ctx.request_raw(msg)` / `ctx.request_raw_until(msg, deadline)` | Send a request and get its `ReplyReceiver` (tasks: a future, threads: `recv()`/`try_recv()`) yielding `Result`; the deadline (5 s default, or `Deadline::At`/`Never`) is fixed at send | +| `ctx.send_request(msg, deadline)` | Send a request without waiting (`gen_server:send_request/3`) and get its `ReplyReceiver` (tasks: a future, threads: `recv()`/`try_recv()`) yielding `Result`; the deadline (a `Duration`, an `Instant` or `Deadline::Never`) is fixed at send | | `ctx.recipient::()` | Get a type-erased `Recipient` for this actor | | `ctx.actor_ref()` | Get an `ActorRef` from the context | | `ctx.id()` | This actor's `ActorId` | @@ -165,7 +165,7 @@ async fn handle_tick(&mut self, _msg: Tick, ctx: &Context) { | `actor_ref.request(msg)` | Request with default 5s timeout. Tasks: `.await`, Threads: blocking | | `actor_ref.request_with_timeout(msg, duration)` | Request with custom timeout | | `actor_ref.with_request_timeout(duration)` | Clone a configured handle for generated protocol methods (v0.7) | -| `actor_ref.request_raw(msg)` / `actor_ref.request_raw_until(msg, deadline)` | Returns the request's `ReplyReceiver` (as `ctx.request_raw`) | +| `actor_ref.send_request(msg, deadline)` | Returns the request's `ReplyReceiver` (as `ctx.send_request`) | | `actor_ref.recipient::()` | Get a type-erased `Recipient` | | `actor_ref.context()` | Get a `Context` (for timer setup, etc.) | | `actor_ref.join()` | Wait until the actor has fully stopped (tasks: `.await`, threads: blocking) | @@ -412,7 +412,7 @@ When you need to send a specific message type to an actor without knowing its co pub type Recipient = Arc>; ``` -`Receiver` is the object-safe trait that provides `send()` and `request_raw()` for a single message type. +`Receiver` is the object-safe trait that provides `send()` and `send_request()` for a single message type (on a `Recipient`, `send_request` takes a `Deadline`: `Deadline::after(d)` or `d.into()`). ### Getting a Recipient @@ -661,7 +661,7 @@ async fn on_checked(&mut self, msg: Checked, _ctx: &Context) { // Delegation: the obligation and its credit stay here. let token = reply.defer()?; -let downstream = other.request_raw_until(Lookup(key), token.deadline().map_or(Deadline::Never, Deadline::At)); +let downstream = other.send_request(Lookup(key), token.deadline()); // same deadline as our caller ctx.pipe_reply(JobOptions::new(), token, |_| async move { downstream?.await }, |res, token| Delegated { res, token })?; ``` @@ -1500,7 +1500,7 @@ waits retain their existing first-poll timeout behavior. Synchronous protocol requests are observed by their blocking request; `Response::ready` does not count that result a second time. Manually constructed ready results are not requests. -`request_raw` returns a `ReplyReceiver`; it does not observe how the caller +`send_request` returns a `ReplyReceiver`; it does not observe how the caller waits on it, so it is not a caller-observed request. Its deadline clock is part of the contract: awaited directly, a raw receiver is expired by the owner's deadline heap on the real clock (the simulated one under diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 629250dc..f1e61dc9 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -30,6 +30,18 @@ every change; this guide covers what you need to edit when upgrading. | `PipeError { kind: PipeErrorKind::Overloaded }`, `e.kind` *(0.8 pre-release)* | `PipeError::Overloaded`, `e` | | `PipeReplyError::Failed { token, kind }` *(0.8 pre-release)* | `PipeReplyError::Failed { token, error }` (`#[non_exhaustive]`) | | `DEFAULT_EMIT_CREDITS`, `default_max_native_jobs()` *(0.8 pre-release)* | removed from the public API (the defaults, 16 and `min(4, cores)`, are documented) | +| `actor.request_raw(msg)` (0.7: no deadline) | `actor.send_request(msg, Deadline::Never)` (or a `Duration` / `Instant`) | +| `actor.request_raw_until(msg, deadline)` *(0.8 pre-release)* | `actor.send_request(msg, deadline)` | +| `Response::from_with_timeout(actor.request_raw(msg), t)` | `Response::from(actor.send_request(msg, t))` | +| `impl Receiver`: `fn request_raw(&self, msg)` | `fn send_request(&self, msg, deadline: Deadline)` | +| `reply.deadline()` / `token.deadline()` / `rx.deadline()` returning `Option` *(0.8 pre-release)* | return `Deadline` (`Deadline::Never` for none); forward with `other.send_request(msg, token.deadline())` | +| `token.with_deadline(at)` *(0.8 pre-release)* | `token.shorten_deadline(at)` (a `Duration` or `Instant`) | +| `reply.has_caller()` *(0.8 pre-release)* | removed: `reply.defer()` returns `Err(DeferError::NoCaller)` for a send | +| `PendingReply::detached()` *(0.8 pre-release)* | `spawned_test::detached_reply()` | +| `PendingReplies::{purge, retained, live}` *(0.8 pre-release)* | `len()` / `is_empty()` (live tokens); purging is automatic on `insert` | +| `InsertError>` *(0.8 pre-release)* | `InsertError` (`#[non_exhaustive]`) | +| `DEFAULT_MAX_OUTSTANDING` *(0.8 pre-release)* | removed from the public API (`Actor::MAX_OUTSTANDING` defaults to 1024) | +| `system.shutdown(deadline)`, `supervisor.shutdown(deadline)`, `report.wait(deadline)` | unchanged calls; the `Duration` parameter is named `grace` (a deadline is an instant in 0.8) | ### Exit signals, startup and runtime drops as in OTP (ranks 5e, 5f, 5g, targets 0.8, in progress) @@ -202,16 +214,18 @@ the actor's death point). Sends reach the same a test, use `spawned_test::reply_pair()`. **Timeouts.** The request timeout starts at send, not at first poll. -`request_raw` gains the 5 s default; use `request_raw_until(msg, -Deadline::Never)` for the old unbounded wait. -**Raw API.** `request_raw` returns a `ReplyReceiver` (a future in tasks +**Raw API.** `request_raw(msg)` is now `send_request(msg, deadline)` +(OTP's `gen_server:send_request/3`). It takes the deadline explicitly: a +`Duration` from now, an `Instant`, or `Deadline::Never`. **0.7's +`request_raw` had no deadline**: `send_request(msg, Deadline::Never)` keeps +that unbounded wait, and `send_request(msg, DEFAULT_REQUEST_TIMEOUT)` gives +the 5 s `request` uses. It returns a `ReplyReceiver` (a future in tasks mode, `recv()` in threads mode) yielding `Result`. -`Response::from_with_timeout` is removed: build the deadline first, -`Response::from(actor.request_raw_until(msg, Deadline::after(t)))`. -`Receiver` implementors implement `request_raw_until` (`request_raw` is -provided). A `recv_timeout` on the old native receiver has no counterpart: -the deadline is fixed when the request is sent. +`Response::from_with_timeout` is removed: `Response::from(actor.send_request(msg, t))`. +`Receiver` implementors implement `send_request(msg, Deadline)`. A +`recv_timeout` on the old native receiver has no counterpart: the deadline +is fixed when the request is sent. **Metrics.** `metrics::Snapshot` gains `requests_reply_dropped_total` and `requests_overloaded_total`, the `deferred_*` and `jobs_*` fields (see the diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index 1e415eb3..aa1fae90 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -133,7 +133,7 @@ Rules: Only `REPLIED` publishes the value; a receiver that sees `WRITING` waits. A registry close that finds `WRITING` waits for it to resolve. The window is a value write; no user code runs in it. -2. **Shortening.** `with_deadline(at)` is `fetch_min(deadline, at)`, then the +2. **Shortening.** `shorten_deadline(at)` is `fetch_min(deadline, at)`, then the `[deadline]` check. The linearization point is the reply's `WRITING` CAS: - a shortening before it is seen by the reply; - a shortening after it finds the word not `OPEN` and does nothing. @@ -172,15 +172,15 @@ There is one timeout reason, as in OTP. `DeadlineExceeded` exists only on | Today | Proposed | |---|---| -| `request_raw(msg) -> Result, _>`, no timeout | `request_raw(msg) -> Result, _>`, default 5 s deadline (as `gen_server:call/2`); `request_raw_until(msg, Deadline)` with `Deadline::At(t)` or `Deadline::Never` | -| `Receiver::request_raw` | `request_raw_until(msg, Deadline)` (required) plus a provided `request_raw` | -| erased `request(recipient, msg, timeout)` | computes the deadline before enqueueing, then `request_raw_until` | +| `request_raw(msg) -> Result, _>`, no timeout | `send_request(msg, impl Into) -> Result, _>` (OTP's `gen_server:send_request/3`): a `Duration`, an `Instant` or `Deadline::Never` | +| `Receiver::request_raw` | `send_request(msg, Deadline)` (required) | +| erased `request(recipient, msg, timeout)` | computes the deadline before enqueueing, then `send_request` | | `Response::from(Result, _>)`, `Response::from_with_timeout` | `Response::from(Result, _>)`; `from_with_timeout` removed (the deadline is already in the slot) | -| `#[protocol]` tasks code: `from_with_timeout(target.request_raw(m), t)` | the timeout is chosen first (per-call override, method, protocol, default), then `Response::from(target.request_raw_until(m, now + t))` | +| `#[protocol]` tasks code: `from_with_timeout(target.request_raw(m), t)` | the timeout is chosen first (per-call override, method, protocol, default), then `Response::from(target.send_request(m, Deadline::after(t)))` | | `#[protocol]` threads code: `Response::ready(target.request_with_timeout(m, t))` | unchanged: threads `Response` stays eager and blocks in the call | | `ActorError`, exhaustive | `#[non_exhaustive]`, plus `ReplyDropped` and `Overloaded` | | `metrics::Snapshot` request counters | add `requests_reply_dropped_total` and `requests_overloaded_total` (`RequestObservation::finish`) | -| `spawned-test` `Mock`, `Probe` | `Mock` implements the new `Handler` (below); `Probe` implements `request_raw_until` and discharges unexpected requests as `Stopped`, keeping its contract | +| `spawned-test` `Mock`, `Probe` | `Mock` implements the new `Handler` (below); `Probe` implements `send_request` and discharges unexpected requests as `Stopped`, keeping its contract | `ReplyReceiver` is a future in `tasks` mode and has `recv()` in `threads` mode. Dropping it is `CAS OPEN -> ABANDONED`, which returns the token's credit @@ -196,7 +196,7 @@ then expires the slot; there is one deadline, not two timers. The behaviour changes in two places, both in the migration notes: - a `#[protocol]` request's timeout used to start at first poll; -- `request_raw` used to have no timeout. +- `request_raw` used to have no timeout (`send_request` takes one explicitly). We use send time because the deadline is a property of the request, the server cannot learn when the caller polls, propagated deadlines (rank 19) need an @@ -207,10 +207,10 @@ absolute instant, and OTP's `call` starts waiting when it sends. - **A polling receiver.** `request`, `request_with_timeout` and `Response::from` sleep on the runtime's timer until the slot's deadline and then `CAS OPEN -> TIMED_OUT` (so they follow paused Tokio clocks). A - raw receiver awaited directly (`request_raw(..)?.await`) instead puts its + raw receiver awaited directly (`send_request(..)?.await`) instead puts its deadline once in the owner's heap, which expires it on the service's clock (real time, or the simulated clock under `spawned_sim`), saving a timer - per request. A shortened deadline (`ReplyTo::with_deadline`) wakes the + per request. A shortened deadline (`ReplyTo::shorten_deadline`) wakes the awaiting or parked receiver, which re-arms for it. - **`defer()`, the `pipe_reply` attach and `is_closed()`.** They check the deadline first, so an unpolled, expired request reports `Closed`. @@ -235,7 +235,7 @@ handles a call even after the caller timed out, and the late reply is dropped. - **Stale entries.** An entry is stale when its slot or job is terminal, or when its deadline is later than the current one (superseded by - `with_deadline`). + `shorten_deadline`). - **Compaction.** The heap is rebuilt when stale entries exceed live ones, per actor. So storage is at most about 2 × (live deferred tokens + live jobs with deadlines + awaited raw receivers); the first two are bounded by @@ -318,7 +318,7 @@ pub struct PendingReply<'t, R> { .. } impl PendingReply<'_, R> { pub fn reply(self, value: R) -> Delivery; // Delivered | CallerGone | Expired pub fn defer(self) -> Result, DeferError>; // Overloaded | Closed | NoCaller - pub fn deadline(&self) -> Option; + pub fn deadline(&self) -> Deadline; // Never for sends pub fn is_closed(&self) -> bool; // checks the deadline first pub fn overloaded(self); // answer Err(Overloaded) } @@ -327,8 +327,8 @@ impl PendingReply<'_, R> { pub struct ReplyTo { .. } // Send + 'static, not Clone impl ReplyTo { pub fn reply(self, value: R) -> Delivery; // .. | OwnerExited - pub fn deadline(&self) -> Option; - pub fn with_deadline(&self, at: Instant); // shorten only + pub fn deadline(&self) -> Deadline; + pub fn shorten_deadline(&self, d: impl Into); // shorten only pub fn is_closed(&self) -> bool; pub async fn closed(&self); // tasks mode pub fn owner(&self) -> ActorId; @@ -476,7 +476,8 @@ ctx.pipe_reply(JobOptions::new(), token, |_| other.request(msg), |res, token| De ``` The downstream request is an ordinary request with the destination's own -admission and deadline; it should use `token.deadline()`, which rank 19 +admission and deadline; it should pass `token.deadline()` on +(`other.send_request(msg, token.deadline())`), which rank 19 automates. The obligation and credit stay in this actor's registry. When the linked job ends or is cancelled, its future, and with it the downstream `ReplyReceiver`, is dropped. That abandons the downstream slot and returns @@ -489,12 +490,14 @@ impl PendingReplies { pub fn with_capacity(cap: usize) -> Self; /// Purges terminal tokens, then inserts; a duplicate key or a full table /// hands the new token back. - pub fn insert(&mut self, key: K, token: ReplyTo) -> Result<(), InsertError>>; + pub fn insert(&mut self, key: K, token: ReplyTo) -> Result<(), InsertError>; pub fn take(&mut self, key: &K) -> Option>; // skips (and removes) terminal entries - pub fn purge(&mut self) -> usize; - pub fn retained(&self) -> usize; // entries held, without purging - pub fn live(&self) -> usize; // non-terminal entries (scan) + pub fn len(&self) -> usize; // live entries (scan) + pub fn is_empty(&self) -> bool; + pub fn drain(&mut self) -> impl Iterator)>; } +#[non_exhaustive] +pub enum InsertError { Duplicate(ReplyTo), Full(ReplyTo) } // into_inner() ``` Retained entries never exceed `cap`, however fast tokens expire. The handler @@ -807,16 +810,16 @@ Aligned with OTP, so not differences: **Timeouts.** - The request timeout starts at send, not at first poll. -- `request_raw` gains a 5 s default; use `request_raw_until(msg, - Deadline::Never)` for the old unbounded wait. +- `request_raw(msg)` becomes `send_request(msg, deadline)`; 0.7's + unbounded wait is `send_request(msg, Deadline::Never)`. **Raw API.** -- `request_raw` returns `ReplyReceiver`, which yields +- `send_request` returns `ReplyReceiver`, which yields `Result`. - `Response::from_with_timeout` is removed. - `Receiver` implementors (including `Probe`) implement - `request_raw_until`. + `send_request`. **Lifecycle.** @@ -1064,7 +1067,7 @@ changes). | 2.1.1 Read before publish | `WRITING` state; `REPLIED` stored after the value (§1). | | 2.1.2 Permit install racing a terminal slot | `TOKEN` bit set by a CAS that requires `OPEN` (§4). | | 2.1.3 Undispatched envelope drop | Mailbox disposal discharges `STOPPED` (§5 step 3.2); only a turn's `PendingReply` drop gives `Dropped`. | -| 2.2.1 `request_raw` gains a timeout | Stated; `Deadline::Never` keeps the unbounded wait (§1, migration). | +| 2.2.1 `request_raw` gains a timeout | `send_request(msg, deadline)` takes it explicitly; `Deadline::Never` keeps the unbounded wait (§1, migration). | | 2.2.2 Stale doc comments in code | Implementation task: `macros/src/lib.rs` "timers start on poll" and `tasks/actor.rs` request docs. | | 2.3.1 Unpolled expired requests dispatched | Dispatched by design (OTP); `defer()` on an expired slot returns `Closed`; skipping is opt-in (rank 19). | | 2.3.2 Process-wide service on the first runtime | Dedicated service thread for unmanaged actors; per-sim-instance under sim (§2). | @@ -1082,7 +1085,7 @@ changes). | 2.7.2 Emits after the deadline | Emit check and terminal enqueue are serialized under the job lock (§9). | | 2.8 `R: From`, local-slot deadlock | Removed; explicit trait migration (§3). | | 2.9.1 Threads `Response` stays eager | Stated (§1 API table). | -| 2.9.2 `Probe` | Implements `request_raw_until`; unexpected requests answer `Stopped` (§1). | +| 2.9.2 `Probe` | Implements `send_request`; unexpected requests answer `Stopped` (§1). | | 2.9.3 `PendingReplies::len(&self)` | Replaced by non-mutating `retained` and `live`; purging is in `&mut` methods (§7). | | 2.9.4 Metrics type name | `metrics::Snapshot` and `RequestObservation::finish` (§1). | | 3.A model divergences | `HeldA` replaced by `Credits` with two holders; expiry abstraction stated; `pipe_reply` notification is a turn (`JobNotify`); cancellation is opt-in (`JobCancel` any closed slot). | diff --git a/docs/design/semantics.md b/docs/design/semantics.md index 31af79ec..a269be64 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -133,8 +133,9 @@ only the caller that sent that request. Checked by: [`actor_scenarios::concurrent_producers_preserve_order_and_reply_identity`][actor_scenarios], [`sim_ordering::per_sender_fifo_and_reply_identity_{pct,dfs}`][sim_ordering]. -**R3. Timeouts.** Default 5 s, as `gen_server:call/2`; `request_raw` has the -same default and `request_raw_until(msg, Deadline::Never)` waits without one. +**R3. Timeouts.** Default 5 s, as `gen_server:call/2`; `send_request(msg, +deadline)` takes its deadline explicitly (`Deadline::Never` waits without +one). Precedence: `with_request_timeout` handle, `#[request_timeout]` method attribute, `#[protocol(timeout)]`, 5 s. Send-only methods have no timeout. The deadline is fixed when the request is sent (send time plus the timeout) @@ -146,12 +147,12 @@ a late reply is dropped (`Delivery::Expired`). Which clock expires a waiting caller is part of the contract: `request`, `request_with_timeout` and `Response::from(rx)` arm the runtime's timer, so they follow paused Tokio time; a `ReplyReceiver` awaited directly -(`request_raw(..)?.await`) puts its deadline in the owner's deadline heap +(`send_request(..)?.await`) puts its deadline in the owner's deadline heap instead, serviced on the real clock (the simulated one under `spawned_sim`), not paused Tokio time. The reason is cost: a raw receiver is the building block for fan-out and needs no runtime timer per request. Tests that pause Tokio time wait through `request` or `Response::from`. Either way a -shortened deadline (`ReplyTo::with_deadline`) wakes the waiting caller. +shortened deadline (`ReplyTo::shorten_deadline`) wakes the waiting caller. Checked by: [`protocol_timeouts`][protocol_timeouts], [`cancellation_safety::protocol_calls_are_cancel_safe_and_send_eagerly`][cancellation_safety], [`protocol_timeouts`][ex_protocol_timeouts] example (both modes), diff --git a/examples/capabilities/src/bin/deferred_replies.rs b/examples/capabilities/src/bin/deferred_replies.rs index ff06e89a..11049e0a 100644 --- a/examples/capabilities/src/bin/deferred_replies.rs +++ b/examples/capabilities/src/bin/deferred_replies.rs @@ -67,10 +67,10 @@ impl Router { ) { let Ok(token) = reply.defer() else { return }; // The downstream request inherits the caller's deadline. - let deadline = token.deadline().map_or(Deadline::Never, Deadline::At); + let deadline = token.deadline(); let downstream = self .backend - .request_raw_until(backend_protocol::Lookup { millis: msg.millis }, deadline); + .send_request(backend_protocol::Lookup { millis: msg.millis }, deadline); let work = async move { downstream?.await }; if let Err(e) = ctx.pipe_reply( JobOptions::new(), @@ -197,7 +197,8 @@ impl Connection { match self.current_requests.insert(msg.id, token) { Ok(()) => {} // the request goes out on the socket here Err(InsertError::Duplicate(token)) => drop(token), // ReplyDropped; the first caller is kept - Err(InsertError::Full(token)) => token.overloaded(), + // `Full` (InsertError is non_exhaustive): answer Overloaded. + Err(error) => error.into_inner().overloaded(), } } @@ -361,10 +362,30 @@ async fn main() { current_requests: PendingReplies::with_capacity(2), } .start(); - let a = conn.request_raw(Outgoing { id: 1 }).unwrap(); - let dup = conn.request_raw(Outgoing { id: 1 }).unwrap(); - let b = conn.request_raw(Outgoing { id: 2 }).unwrap(); - let full = conn.request_raw(Outgoing { id: 3 }).unwrap(); + let a = conn + .send_request( + Outgoing { id: 1 }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); + let dup = conn + .send_request( + Outgoing { id: 1 }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); + let b = conn + .send_request( + Outgoing { id: 2 }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); + let full = conn + .send_request( + Outgoing { id: 3 }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); assert!(matches!(dup.await, Err(ActorError::ReplyDropped))); assert!(matches!(full.await, Err(ActorError::Overloaded))); drop(b); // the caller gives up: its entry is purged on the next insert @@ -391,8 +412,12 @@ async fn main() { waiters: VecDeque::new(), } .start(); - let first = table.request_raw(WaitForPeer).unwrap(); - let second = table.request_raw(WaitForPeer).unwrap(); + let first = table + .send_request(WaitForPeer, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + let second = table + .send_request(WaitForPeer, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); assert!(matches!( table.request(WaitForPeer).await, Err(ActorError::Overloaded) @@ -407,8 +432,8 @@ async fn main() { answer_first: 0, } .start(); - let one = parker.request_raw_until(Park, Deadline::Never).unwrap(); - let two = parker.request_raw_until(Park, Deadline::Never).unwrap(); + let one = parker.send_request(Park, Deadline::Never).unwrap(); + let two = parker.send_request(Park, Deadline::Never).unwrap(); parker.request(StopAnswering(1)).await.unwrap(); assert_eq!(one.await.unwrap(), "answered in stopped()"); assert!(matches!( diff --git a/examples/capabilities/src/bin/timeout_outcomes.rs b/examples/capabilities/src/bin/timeout_outcomes.rs index 4cd0215f..a6d08d56 100644 --- a/examples/capabilities/src/bin/timeout_outcomes.rs +++ b/examples/capabilities/src/bin/timeout_outcomes.rs @@ -53,7 +53,7 @@ async fn main() { let (release_tx, release_rx) = oneshot::channel(); // The deadline is fixed at send (OTP's call/3): already due here. An // expired request is still dispatched, as in OTP; its late reply is dropped. - let reply = counter.request_raw_until( + let reply = counter.send_request( Apply { entered: entered_tx, release: release_rx, diff --git a/fuzz/src/driver.rs b/fuzz/src/driver.rs index 0922800e..b843fdf4 100644 --- a/fuzz/src/driver.rs +++ b/fuzz/src/driver.rs @@ -731,7 +731,7 @@ impl Driver { lane: 0, act, }; - drop(aref.request_raw(msg)); + drop(aref.send_request(msg, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT)); } Op::WaitGeneration { sup, child, ms } => self.wait_generation(sup, child, ms).await, Op::Fence => { diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 9f10af67..7bfda59b 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -491,7 +491,7 @@ fn generate_target_impl( MethodKind::Request(_) => match mode { // The timeout was chosen before the message was built; the // deadline is fixed now, at send, and travels in the slot. - RuntimeMode::Tasks => quote! { spawned_concurrency::Response::from(target.request_raw_until(message, spawned_concurrency::reply::Deadline::after(timeout))) }, + RuntimeMode::Tasks => quote! { spawned_concurrency::Response::from(target.send_request(message, spawned_concurrency::Deadline::after(timeout))) }, RuntimeMode::Threads => quote! { spawned_concurrency::Response::ready(target.request_with_timeout(message, timeout)) }, }, }; @@ -1576,7 +1576,7 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { // request's reply obligation and answers it with the method's // result (a deferred handler receives the obligation itself). let reply_ty = quote! { - spawned_concurrency::reply::PendingReply<'_, <#msg_ty as spawned_concurrency::message::Message>::Result> + spawned_concurrency::PendingReply<'_, <#msg_ty as spawned_concurrency::message::Message>::Result> }; // The parameter list carries the user's parentheses span. let mut params = proc_macro2::Group::new( @@ -1736,7 +1736,7 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { /// The reply type is named through the `Handler` path when it is qualified /// (`impl my_spawned::tasks::Handler for A` uses /// `my_spawned::tasks::PendingReply`), so a renamed dependency works. -/// With a bare `Handler`, it is `spawned_concurrency::reply::PendingReply`; +/// With a bare `Handler`, it is `spawned_concurrency::PendingReply`; /// a crate that renames `spawned-concurrency` names it: /// `#[send_handler(crate = my_spawned)]`. #[proc_macro_attribute] @@ -1818,9 +1818,9 @@ pub fn send_handler(attr: TokenStream, item: TokenStream) -> TokenStream { }) }); let reply_ty = match (krate, handler_module) { - (Some(krate), _) => quote! { #krate::reply::PendingReply<'_, ()> }, + (Some(krate), _) => quote! { #krate::PendingReply<'_, ()> }, (None, Some(module)) => quote! { #module::PendingReply<'_, ()> }, - (None, None) => quote! { spawned_concurrency::reply::PendingReply<'_, ()> }, + (None, None) => quote! { spawned_concurrency::PendingReply<'_, ()> }, }; let reply_param: FnArg = syn::parse_quote! { #reply: #reply_ty diff --git a/testing/src/lib.rs b/testing/src/lib.rs index 547a8cc0..e0b98c97 100644 --- a/testing/src/lib.rs +++ b/testing/src/lib.rs @@ -72,3 +72,13 @@ pub fn reply_pair() -> ( ) { spawned_concurrency::reply::reply_pair(spawned_concurrency::reply::Deadline::Never) } + +/// The `PendingReply` a handler receives for a sent message (no caller), +/// for calling a manual `Handler::handle` directly as a send: +/// +/// ```ignore +/// actor.handle(Tick, spawned_test::detached_reply(), &ctx).await; +/// ``` +pub fn detached_reply() -> spawned_concurrency::PendingReply<'static, R> { + spawned_concurrency::PendingReply::detached() +} diff --git a/testing/src/shared.rs b/testing/src/shared.rs index 0b684db9..2a6657ef 100644 --- a/testing/src/shared.rs +++ b/testing/src/shared.rs @@ -170,7 +170,7 @@ macro_rules! probe_common { fn send(&self, msg: M) -> Result<(), ActorError> { ProbeCore::send(self, msg) } - fn request_raw_until( + fn send_request( &self, msg: M, deadline: Deadline, diff --git a/testing/src/tasks.rs b/testing/src/tasks.rs index 320ca8a4..1db211a0 100644 --- a/testing/src/tasks.rs +++ b/testing/src/tasks.rs @@ -285,22 +285,22 @@ where } } -/// Wait for `handle` to exit, or panic after `deadline` instead of hanging. +/// Wait for `handle` to exit, or panic after `limit` instead of hanging. /// Works for actors of either mode. #[track_caller] -pub fn exit_within(handle: &ChildHandle, deadline: Duration) -> impl Future { - exit_within_at(handle.clone(), deadline, Location::caller()) +pub fn exit_within(handle: &ChildHandle, limit: Duration) -> impl Future { + exit_within_at(handle.clone(), limit, Location::caller()) } async fn exit_within_at( handle: ChildHandle, - deadline: Duration, + limit: Duration, at: &'static Location<'static>, ) -> ExitReason { - match timeout(deadline, handle.wait_exit_async()).await { + match timeout(limit, handle.wait_exit_async()).await { Ok(reason) => reason, Err(_) => panic!( - "actor {:?} did not exit within {deadline:?}; a handler may be blocked or the \ + "actor {:?} did not exit within {limit:?}; a handler may be blocked or the \ actor was never asked to stop (exit_within called at {at})", handle.id() ), @@ -311,9 +311,9 @@ async fn exit_within_at( #[track_caller] pub fn join_within( actor: &ActorRef, - deadline: Duration, + limit: Duration, ) -> impl Future { - exit_within_at(actor.child_handle(), deadline, Location::caller()) + exit_within_at(actor.child_handle(), limit, Location::caller()) } /// Poll `condition` every millisecond until it holds; panic naming `what` after `timeout`. @@ -338,16 +338,16 @@ pub fn wait_until<'a>( } /// Wait for a ready generation strictly greater than `after`, or panic with -/// the child's snapshot after `deadline`. +/// the child's snapshot after `limit`. #[track_caller] pub fn wait_for_generation( child: &SupervisedChild, after: u64, - deadline: Duration, + limit: Duration, ) -> impl Future> + '_ { let at = Location::caller(); async move { - match timeout(deadline, child.wait_for_generation(after)).await { + match timeout(limit, child.wait_for_generation(after)).await { Ok(Ok(actor)) => actor, Ok(Err(unavailable)) => panic!( "supervised child will not reach a generation after {after}: {unavailable}; \ @@ -356,7 +356,7 @@ pub fn wait_for_generation( ), Err(_) => panic!( "supervised child did not reach a generation after {after} within \ - {deadline:?}; snapshot {:?} (wait_for_generation called at {at})", + {limit:?}; snapshot {:?} (wait_for_generation called at {at})", child.snapshot() ), } @@ -365,8 +365,8 @@ pub fn wait_for_generation( /// Fault: make the current instance of `child` exit with `reason` (see /// [`faults::exit`]) and return its replacement once the supervisor has started -/// it, or panic after `deadline` (for example when the restart mode does not -/// restart that reason). `deadline` bounds the whole call: finding a running +/// it, or panic after `limit` (for example when the restart mode does not +/// restart that reason). `limit` bounds the whole call: finding a running /// instance, injecting the fault and waiting for the replacement. /// /// The instance targeted and the generation waited past always belong @@ -380,7 +380,7 @@ pub fn wait_for_generation( pub fn restart_child( child: &SupervisedChild, reason: ExitReason, - deadline: Duration, + limit: Duration, ) -> impl Future> + '_ { let at = Location::caller(); async move { @@ -392,7 +392,7 @@ pub fn restart_child( faulted.set(Some(generation)); child.wait_for_generation(generation).await }; - let outcome = timeout(deadline, run).await; + let outcome = timeout(limit, run).await; match (outcome, faulted.get()) { (Ok(Ok(actor)), _) => actor, (Ok(Err(unavailable)), generation) => panic!( @@ -401,13 +401,13 @@ pub fn restart_child( child.snapshot() ), (Err(_), None) => panic!( - "supervised child never had a stable instance to fault within {deadline:?}; \ + "supervised child never had a stable instance to fault within {limit:?}; \ snapshot {:?} (restart_child called at {at})", child.snapshot() ), (Err(_), Some(generation)) => panic!( "supervised child did not reach a generation after {generation} within \ - {deadline:?} (one deadline for the whole call); snapshot {:?} \ + {limit:?} (one limit for the whole call); snapshot {:?} \ (restart_child called at {at})", child.snapshot() ), @@ -445,24 +445,24 @@ async fn faultable_instance(child: &SupervisedChild) -> (ActorRef SupervisorReport { - let report = supervisor.shutdown(deadline).await; +/// Shut a supervisor down within `grace`, or panic with its report. +pub async fn shutdown_supervisor(supervisor: &Supervisor, grace: Duration) -> SupervisorReport { + let report = supervisor.shutdown(grace).await; assert!( report.complete, - "supervisor did not shut down within {deadline:?}: failure {:?}, children {:#?}", + "supervisor did not shut down within {grace:?}: failure {:?}, children {:#?}", report.failure, report.children ); report } -/// Shut an [`ActorSystem`] down within `deadline`, or panic with a report of +/// Shut an [`ActorSystem`] down within `grace`, or panic with a report of /// what is still running instead of hanging the test. -pub async fn shutdown_system(system: &ActorSystem, deadline: Duration) -> ShutdownReport { - let report = system.shutdown(deadline).await; +pub async fn shutdown_system(system: &ActorSystem, grace: Duration) -> ShutdownReport { + let report = system.shutdown(grace).await; assert!( report.complete, - "ActorSystem did not shut down within {deadline:?}\n unfinished groups: {:?}\n \ + "ActorSystem did not shut down within {grace:?}\n unfinished groups: {:?}\n \ unfinished actors: {:?}\n exits: {:?}\n supervisor failures: {:?}\n \ group failures: {:?}", report.unfinished_groups, @@ -514,9 +514,9 @@ impl TestSystem { .expect("start mock on test group") } - /// [`shutdown_system`]: panics with a report if not complete by `deadline`. - pub async fn shutdown(self, deadline: Duration) -> ShutdownReport { - shutdown_system(&self.system, deadline).await + /// [`shutdown_system`]: panics with a report if not complete by `grace`. + pub async fn shutdown(self, grace: Duration) -> ShutdownReport { + shutdown_system(&self.system, grace).await } } diff --git a/testing/src/threads.rs b/testing/src/threads.rs index edc25747..fa850edd 100644 --- a/testing/src/threads.rs +++ b/testing/src/threads.rs @@ -192,20 +192,20 @@ where }) } -/// Block until `handle` exits, or panic after `deadline` instead of hanging. +/// Block until `handle` exits, or panic after `limit` instead of hanging. /// /// Polls every millisecond: `ChildHandle` has no blocking wait with a /// timeout, and a waiter thread per call would leak on timeout. #[track_caller] -pub fn exit_within(handle: &ChildHandle, deadline: Duration) -> ExitReason { - let end = Instant::now() + deadline; +pub fn exit_within(handle: &ChildHandle, limit: Duration) -> ExitReason { + let end = Instant::now() + limit; loop { if let Some(reason) = handle.exit_reason() { return reason; } if Instant::now() >= end { panic!( - "actor {:?} did not exit within {deadline:?}; a handler may be blocked or the \ + "actor {:?} did not exit within {limit:?}; a handler may be blocked or the \ actor was never asked to stop", handle.id() ); @@ -216,8 +216,8 @@ pub fn exit_within(handle: &ChildHandle, deadline: Duration) -> ExitReason { /// [`exit_within`] for a native `ActorRef`. #[track_caller] -pub fn join_within(actor: &ActorRef, deadline: Duration) -> ExitReason { - exit_within(&actor.child_handle(), deadline) +pub fn join_within(actor: &ActorRef, limit: Duration) -> ExitReason { + exit_within(&actor.child_handle(), limit) } /// Poll `condition` every millisecond until it holds; panic naming `what` after `timeout`. diff --git a/testing/tests/tasks_kit.rs b/testing/tests/tasks_kit.rs index c19e6747..48a9eb6b 100644 --- a/testing/tests/tasks_kit.rs +++ b/testing/tests/tasks_kit.rs @@ -419,9 +419,10 @@ async fn settle_within_uses_the_given_bound() { .delay::(Duration::from_millis(500)) .start(); // Occupy the mock; the fence queues behind the slow reply. - let _pending = mock - .actor_ref() - .request_raw(inventory_protocol::Reserve { item: "x".into() }); + let _pending = mock.actor_ref().send_request( + inventory_protocol::Reserve { item: "x".into() }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ); mock.settle_within(Duration::from_millis(20)).await; } diff --git a/testing/tests/threads_kit.rs b/testing/tests/threads_kit.rs index dd7bc58f..02f1256b 100644 --- a/testing/tests/threads_kit.rs +++ b/testing/tests/threads_kit.rs @@ -235,7 +235,9 @@ fn native_expect_no_message_reports_in_flight_messages() { q.0 }) .start(); - let _reply = mock.request_raw(Quote(1)).unwrap(); + let _reply = mock + .send_request(Quote(1), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); wait_until(WAIT, "the quote arrived", || mock.count::() == 1); mock.expect_no_message(Duration::from_millis(5)); } From 6994f777b553694d9dee036c9a544f6eb9546bce Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:13:54 -0300 Subject: [PATCH 147/208] fix(reply): PendingReplies drains in insertion order (#261) `PendingReplies::drain` walked a `HashMap` with a per-process `RandomState`, so deferred replies were answered in a different order in each process and a same-seed simulator run did not reproduce its schedule. The table is now a Vec in insertion order (it is capped, and insert already scanned it to purge): drain yields live tokens oldest first, take is a linear scan, and the public bounds are unchanged. No new dependency. Tests (both fail on the HashMap table): `drain_follows_insertion_order` drains two identically built tables and expects insertion order; `tests/sim.rs::pending_replies_drain_is_reproducible` parks 16 deferred callers, drains them in one handler, and asserts the same drain order, caller log, scheduler trace and tape across two same-seed runs. --- CHANGELOG.md | 5 ++ concurrency/src/reply/pending.rs | 73 ++++++++++++++++--- concurrency/tests/sim.rs | 116 +++++++++++++++++++++++++++++++ docs/design/deferred-replies.md | 9 ++- 4 files changed, 191 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97d513ea..89d98fcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- Fix: `PendingReplies::drain` yields live tokens in insertion order. It + walked a `HashMap` with a per-process `RandomState`, so deferred replies + were answered in a different order in each process, and a same-seed + simulator run did not reproduce its schedule (#261). `take` is now a + linear scan of the capped table. - **Breaking (0.8), API pass before publishing** (each change is a row of the [0.7 to 0.8 call-site table](docs/MIGRATION.md#07-to-08-api-call-site-table)): - Jobs: one call per shape, options first (`pipe`, `pipe_blocking`, diff --git a/concurrency/src/reply/pending.rs b/concurrency/src/reply/pending.rs index ccbaaf56..2a07bfe2 100644 --- a/concurrency/src/reply/pending.rs +++ b/concurrency/src/reply/pending.rs @@ -2,7 +2,6 @@ //! (docs/design/deferred-replies.md §7). use super::ReplyTo; -use std::collections::HashMap; use std::hash::Hash; /// Why [`PendingReplies::insert`] handed the token back. The handler decides @@ -32,9 +31,15 @@ impl InsertError { /// /// Experimental in 0.8: a utility shaped by one client (requests by id /// answered when a peer responds); it may change before 1.0. +/// +/// Entries keep insertion order: [`drain`](Self::drain) yields them oldest +/// first, the same in every process, so answering a drained table is +/// reproducible under the simulator (no per-process hash seed decides it). +/// The table is a scan (insert already purges the whole table, and `cap` +/// bounds it): `take` and `insert` are linear in the held entries. pub struct PendingReplies { cap: usize, - tokens: HashMap>, + tokens: Vec<(K, ReplyTo)>, } impl std::fmt::Debug for PendingReplies { @@ -50,7 +55,7 @@ impl PendingReplies { pub fn with_capacity(cap: usize) -> Self { Self { cap, - tokens: HashMap::new(), + tokens: Vec::new(), } } @@ -58,32 +63,33 @@ impl PendingReplies { /// token) or a full table hands the new token back. pub fn insert(&mut self, key: K, token: ReplyTo) -> Result<(), InsertError> { self.purge(); - if self.tokens.contains_key(&key) { + if self.tokens.iter().any(|(k, _)| *k == key) { return Err(InsertError::Duplicate(token)); } if self.tokens.len() >= self.cap { return Err(InsertError::Full(token)); } - self.tokens.insert(key, token); + self.tokens.push((key, token)); Ok(()) } /// Remove and return the live token under `key`; a terminal one is /// removed and skipped. pub fn take(&mut self, key: &K) -> Option> { - let token = self.tokens.remove(key)?; + let at = self.tokens.iter().position(|(k, _)| k == key)?; + let (_, token) = self.tokens.remove(at); (!token.is_closed()).then_some(token) } /// Remove every terminal token. fn purge(&mut self) { - self.tokens.retain(|_, token| !token.is_closed()); + self.tokens.retain(|(_, token)| !token.is_closed()); } /// Live (non-terminal) tokens held: a scan, so a token whose caller /// just left no longer counts. pub fn len(&self) -> usize { - self.tokens.values().filter(|t| !t.is_closed()).count() + self.tokens.iter().filter(|(_, t)| !t.is_closed()).count() } /// Whether no live token is held. @@ -91,9 +97,12 @@ impl PendingReplies { self.len() == 0 } - /// Take every live token (for example to answer them in `stopped()`). + /// Take every live token, oldest first (for example to answer them in + /// `stopped()`). pub fn drain(&mut self) -> impl Iterator)> + '_ { - self.tokens.drain().filter(|(_, token)| !token.is_closed()) + self.tokens + .drain(..) + .filter(|(_, token)| !token.is_closed()) } } @@ -143,4 +152,48 @@ mod tests { assert_eq!(ra.recv().unwrap(), 10); assert_eq!(rd.recv().unwrap(), 40); } + /// #261: draining follows insertion order, so two tables built the + /// same way (in the old `HashMap`, two different `RandomState` seeds) + /// drain identically, oldest first, with taken and terminal entries + /// skipped. + #[test] + fn drain_follows_insertion_order() { + let keys: Vec = (0..64).map(|i| (i * 7_919) % 1_000).collect(); + let build = || { + let mut table = PendingReplies::with_capacity(keys.len()); + let mut receivers = Vec::new(); + for &k in &keys { + let (t, rx) = token(); + table.insert(k, t).unwrap(); + receivers.push(rx); + } + table.take(&keys[3]).unwrap().reply(0); + drop(receivers.remove(10)); // abandoned: terminal, skipped + (table, receivers) + }; + let (mut first, _a) = build(); + let (mut second, _b) = build(); + let once: Vec = first + .drain() + .map(|(k, t)| { + drop(t); + k + }) + .collect(); + let twice: Vec = second + .drain() + .map(|(k, t)| { + drop(t); + k + }) + .collect(); + let expected: Vec = keys + .iter() + .enumerate() + .filter(|(i, _)| *i != 3 && *i != 10) + .map(|(_, k)| *k) + .collect(); + assert_eq!(once, expected); + assert_eq!(twice, expected); + } } diff --git a/concurrency/tests/sim.rs b/concurrency/tests/sim.rs index 3f35b946..1bdddaaa 100644 --- a/concurrency/tests/sim.rs +++ b/concurrency/tests/sim.rs @@ -947,3 +947,119 @@ fn shrink_stays_within_its_run_budget() { assert!(runs <= budget, "budget {budget}: spent {runs}"); } } + +// --------------------------------------------------------------------------- +// Workload: deferred replies parked in a `PendingReplies` and drained at +// once (#261). The drain order decides which caller is answered first, so +// it must not depend on a per-process hash seed. +// --------------------------------------------------------------------------- + +struct Park(u32); +impl Message for Park { + type Result = u32; +} +struct Release; +impl Message for Release { + type Result = Vec; +} +struct Parked; +impl Message for Parked { + type Result = usize; +} + +struct Parker { + pending: spawned_concurrency::tasks::PendingReplies, +} +impl Actor for Parker {} +impl Handler for Parker { + async fn handle( + &mut self, + Park(id): Park, + reply: spawned_concurrency::reply::PendingReply<'_, u32>, + _: &Context, + ) { + if let Ok(token) = reply.defer() { + assert!(self.pending.insert(id, token).is_ok()); + } + } +} +impl Handler for Parker { + async fn handle( + &mut self, + _: Parked, + reply: spawned_concurrency::reply::PendingReply<'_, usize>, + _: &Context, + ) { + reply.reply(self.pending.len()); + } +} +impl Handler for Parker { + async fn handle( + &mut self, + _: Release, + reply: spawned_concurrency::reply::PendingReply<'_, Vec>, + _: &Context, + ) { + let mut order = Vec::new(); + for (id, token) in self.pending.drain() { + token.reply(id); + order.push(id); + } + reply.reply(order); + } +} + +/// Sixteen callers park in an order the scheduler picks; one `Release` +/// drains them. Returns the drain order; callers log their answers. +async fn drain_pending(log: Log) -> Vec { + const CALLERS: u32 = 16; + let parker = Parker { + pending: spawned_concurrency::tasks::PendingReplies::with_capacity(CALLERS as usize), + } + .start(); + let callers: Vec<_> = (0..CALLERS) + .map(|id| { + let (parker, log) = (parker.clone(), log.clone()); + spawned_rt::tasks::spawn(async move { + let answer = parker.request(Park(id)).await.unwrap(); + push(&log, format!("answered:{answer}")); + }) + }) + .collect(); + while parker.request(Parked).await.unwrap() < CALLERS as usize { + sim::yield_now().await; + } + let order = parker.request(Release).await.unwrap(); + for caller in callers { + caller.await.unwrap(); + } + parker.context().stop(); + parker.join().await; + order +} + +/// #261: the same seed gives the same drain order, caller log, scheduler +/// trace and tape. With a `HashMap` table each run drew its own +/// `RandomState`, so the order (and with it the schedule) differed. +#[test] +fn pending_replies_drain_is_reproducible() { + let record = || { + let rt = Runtime::with_seed(7); + rt.record_trace(); + rt.record_tape(); + let log: Log = Arc::default(); + let order = rt.block_on(drain_pending(log.clone())); + let log = log.lock().unwrap_or_else(|p| p.into_inner()).clone(); + (order, log, rt.trace(), rt.tape()) + }; + let (order, log, trace, tape) = record(); + let again = record(); + assert_eq!(order.len(), 16); + assert_eq!(again.0, order, "drain order differs between same-seed runs"); + assert_eq!(again.1, log, "caller answers differ between same-seed runs"); + assert_eq!( + again.2, trace, + "scheduler trace differs between same-seed runs" + ); + assert_eq!(again.3, tape, "tape differs between same-seed runs"); +} diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index aa1fae90..572ca597 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -494,7 +494,7 @@ impl PendingReplies { pub fn take(&mut self, key: &K) -> Option>; // skips (and removes) terminal entries pub fn len(&self) -> usize; // live entries (scan) pub fn is_empty(&self) -> bool; - pub fn drain(&mut self) -> impl Iterator)>; + pub fn drain(&mut self) -> impl Iterator)>; // live tokens, oldest first } #[non_exhaustive] pub enum InsertError { Duplicate(ReplyTo), Full(ReplyTo) } // into_inner() @@ -502,7 +502,12 @@ pub enum InsertError { Duplicate(ReplyTo), Full(ReplyTo) } // into_in Retained entries never exceed `cap`, however fast tokens expire. The handler decides what to answer on `InsertError` (typically `Overloaded` or a domain -error) and still owns the token. +error) and still owns the token. Entries keep insertion order, so `drain` +answers callers in the same order in every process: the order never comes +from a per-process hash seed, and a same-seed simulator run reproduces it +(#261; `reply::pending::tests::drain_follows_insertion_order`, +`tests/sim.rs::pending_replies_drain_is_reproducible`). The table is a scan +bounded by `cap`: `insert` already purges it whole, and `take` is linear. ## 8. Bounds From f5e93e9e91414ecb8e87360ddb694351eccb5240 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:48:03 +0000 Subject: [PATCH 148/208] verification: model per-round restart charging in Cohorts (RoundCharged, PerChildCharge control) --- verification/tla/Cohorts.tla | 23 ++++++++++++++++++----- verification/tla/CohortsInd.tla | 2 +- verification/tla/Supervision.tla | 6 +++++- verification/tla/apalache.sh | 2 +- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/verification/tla/Cohorts.tla b/verification/tla/Cohorts.tla index d4235420..cf72658d 100644 --- a/verification/tla/Cohorts.tla +++ b/verification/tla/Cohorts.tla @@ -1,6 +1,12 @@ ----------------------------- MODULE Cohorts ----------------------------- -(* Coordinator policy and actual generation ownership. Backoff durations, *) -(* handlers and retry budgets are abstracted; Supervision covers policy. *) +(* Coordinator policy and actual generation ownership. Backoff durations *) +(* and handlers are abstracted; Supervision covers the per-child policy. *) +(* The restart budget is modelled as its charge (rank 5f): a OneForAll or *) +(* RestForOne round takes one grant when the coordinator selects it, its *) +(* rebuilds take none, and a failed rebuild asks for a new round, charged *) +(* again when selected (supervision.rs charge_round). Exhaustion itself is *) +(* Supervision's (BudgetBounded); here RoundCharged pins the charge to the *) +(* rounds. Fault "PerChildCharge" restores v0.7's charge per rebuilt child.*) EXTENDS Naturals, FiniteSets, Sequences, TLC (* `@type:` comments are Apalache annotations (CohortsInd.tla); TLC *) (* ignores them. Integer ranges with a state-dependent bound are written *) @@ -32,7 +38,7 @@ ASSUME Mode \in [Children -> {"Permanent", "Transient", "Temporary"}] \* scanned: Set(Int), affected: Set(Int), stopped: Set(Int), \* clean: Set(Int), resumed: Set(Int), ack: Set(Int), terminal: Set(Int), \* gen: Int -> Int, live: Int -> Set(Int), published: Int -> Int, -\* begun: Set(Int), +\* begun: Set(Int), charged: Int, \* badBarrier: Bool, badClosed: Bool, badTerminal: Bool, badRetry: Bool }; CohortTypeAliases == TRUE VARIABLE @@ -48,7 +54,7 @@ Init == affected |-> {}, stopped |-> {}, clean |-> {}, resumed |-> {}, ack |-> {}, terminal |-> {}, gen |-> [c \in Children |-> 0], live |-> [c \in Children |-> {}], published |-> [c \in Children |-> 0], - begun |-> {}, + begun |-> {}, charged |-> 0, badBarrier |-> FALSE, badClosed |-> FALSE, badTerminal |-> FALSE, badRetry |-> FALSE] Range == {j \in 0..N : s.start <= j} \* = s.start..N, since start \in 0..N @@ -64,6 +70,8 @@ CSelect(c,r,n) == /\ r = s.epoch + 1 /\ r <= MaxRounds /\ n = IF Fault = "Selection" THEN c ELSE ExpectedStart(c) /\ s' = [s EXCEPT !.epoch = r, !.origin = c, !.start = n, + \* One restart grant for the whole round, taken at selection. + !.charged = IF Fault = "PerChildCharge" THEN @ ELSE @ + 1, !.covered = [j \in Children |-> IF j >= n THEN r ELSE s.covered[j]], !.pending = {j \in s.pending : j < n}, !.scanned = {}, !.affected = {}, !.stopped = {}, !.clean = {}, @@ -112,6 +120,7 @@ CBegin(c,r,n) == /\ Mode[c] # "Temporary" \/ s.gen[c] = 0 /\ s' = [s EXCEPT !.gen[c] = n, !.live[c] = @ \cup {n}, !.begun = IF r > 0 THEN @ \cup {c} ELSE @, + !.charged = IF Fault = "PerChildCharge" /\ r > 0 THEN @ + 1 ELSE @, !.badBarrier = @ \/ (r > 0 /\ s.clean # s.affected), !.badClosed = @ \/ ~s.open, !.badRetry = @ \/ (r > 0 /\ c \in s.begun)] @@ -159,6 +168,7 @@ TypeOK == /\ s.gen \in [Children -> 0..MaxGen] /\ s.live \in [Children -> SUBSET (1..MaxGen)] /\ s.published \in [Children -> 0..MaxGen] + /\ s.charged \in 0..(MaxRounds * N) AtMostOneGeneration == \A c \in Children : Cardinality(s.live[c]) <= 1 PublishedIsLive == \A c \in Children : s.published[c] # 0 => s.published[c] \in s.live[c] StrategySelection == s.epoch = 0 \/ s.start = ExpectedStart(s.origin) @@ -170,10 +180,13 @@ TemporaryStartsOnce == \A c \in Children : Mode[c] = "Temporary" => s.gen[c] <= \* S1: a child that never completed its first start had one attempt. FirstStartOnce == \A c \in Children : c \notin s.initial => s.gen[c] <= 1 DoneMeansQuiescent == s.phase = "Done" => \A c \in Children : s.live[c] = {} +\* Rank 5f: one restart grant per selected round (superseded ones included), +\* none per rebuilt child. +RoundCharged == s.charged = s.epoch Safety == TypeOK /\ AtMostOneGeneration /\ PublishedIsLive /\ StrategySelection /\ WholeCohortBarrier /\ NoStartAfterClose /\ NoRetiredParticipant /\ TemporaryStartsOnce /\ DoneMeansQuiescent /\ OneStartPerRound - /\ FirstStartOnce + /\ FirstStartOnce /\ RoundCharged Next == \/ \E c \in Children, r \in Epochs, x \in {"true","false"} : CRequest(c,r,x) \/ \E c \in Children, r \in Epochs, n \in Children : CSelect(c,r,n) diff --git a/verification/tla/CohortsInd.tla b/verification/tla/CohortsInd.tla index 88ee83b5..97ffcf00 100644 --- a/verification/tla/CohortsInd.tla +++ b/verification/tla/CohortsInd.tla @@ -20,7 +20,7 @@ TypeInd == ack: SUBSET Children, terminal: SUBSET Children, gen: [Children -> 0..MaxGen], live: [Children -> SUBSET (1..MaxGen)], published: [Children -> 0..MaxGen], - begun: SUBSET Children, + begun: SUBSET Children, charged: 0..(MaxRounds * N), badBarrier: BOOLEAN, badClosed: BOOLEAN, badTerminal: BOOLEAN, badRetry: BOOLEAN] diff --git a/verification/tla/Supervision.tla b/verification/tla/Supervision.tla index cfc34a87..9f5e93f0 100644 --- a/verification/tla/Supervision.tla +++ b/verification/tla/Supervision.tla @@ -190,7 +190,11 @@ LoopTop(c) == /\ UNCHANGED <> \* Reserve capacity before waiting. Cancellation/supersession refunds it; -\* only a committed grant enters the rolling time window. +\* only a committed grant enters the rolling time window. One grant per +\* recovery round (rank 5f, supervision.rs charge_round): under OneForOne a +\* round restarts one child, so it is one grant per restart; the cohort +\* strategies' charge (one per round, none per rebuilt child) is +\* Cohorts.tla's RoundCharged. RetryReserve(c) == /\ pc[c] = "backoff" /\ open /\ c \notin reserved /\ budget < MaxRestarts /\ reserved' = reserved \cup {c} /\ budget' = budget + 1 diff --git a/verification/tla/apalache.sh b/verification/tla/apalache.sh index 4c695713..fd2a97f0 100755 --- a/verification/tla/apalache.sh +++ b/verification/tla/apalache.sh @@ -161,7 +161,7 @@ for model in $models; do Cohorts|Nested) module="MC${model}Ind" if [[ "$model" == Cohorts ]]; then - probes=NoMidRoundState faults="Selection Cleanup Retirement Closed InPlaceRetry FirstStartRetry" + probes=NoMidRoundState faults="Selection Cleanup Retirement Closed InPlaceRetry FirstStartRetry PerChildCharge" else probes=NoFinishedSubtree faults="Join Closed" fi From 763232f96762c5ea6835f57ded6e42c1484b2c54 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:48:03 +0000 Subject: [PATCH 149/208] verification: model job admission fence and exit cancelling held results (ExitCancelsJobs) --- verification/tla/DeferredJobs.tla | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/verification/tla/DeferredJobs.tla b/verification/tla/DeferredJobs.tla index 0d7b7c94..56cd4aab 100644 --- a/verification/tla/DeferredJobs.tla +++ b/verification/tla/DeferredJobs.tla @@ -38,6 +38,11 @@ (* when the work ends *) (* HeldDeadlineLost (DeadlineArmedWhileHeld) the deadline cannot take *) (* a held result's notification *) +(* ExitKeepsHeld (ExitCancelsJobs) owner exit cancels running jobs *) +(* only, not held results or scheduled emits *) +(* NoAdmissionFence (ExitCancelsJobs) a job is admitted after the exit *) +(* cancelled the job set (admission and cancel_all are *) +(* not serialized under the jobs lock) *) (* *) (* Delayed emits (emit_at) and held results: a running job may schedule an *) (* emit (it takes an emit credit at once and is enqueued, fenced like any *) @@ -55,7 +60,7 @@ CONSTANTS J, Keyed, JobBound, MaxGen, MaxEmit, Credits, Fault ASSUME Fault \in {"None", "NoInstanceCheck", "NoCancelFence", "DoubleTerminal", "ReleaseOnTimeout", "KeyNoCancel", "ReplacePendingOnly", "EmitAfterTerminal", "NoEmitCredit", "EarlyCredit", - "HeldDeadlineLost"} + "HeldDeadlineLost", "ExitKeepsHeld", "NoAdmissionFence"} Jobs == 1..J ASSUME Keyed \subseteq Jobs @@ -114,8 +119,11 @@ CreditAfterCancel(S) == [j \in Jobs |-> IF j \in S /\ exec[j] = "finished" THEN FALSE ELSE credit[j]] CancelSet(S) == CancelCore(S) /\ credit' = CreditAfterCancel(S) +\* Admission and the exit's cancel_all are serialized (one jobs lock and +\* its closed flag): no job is admitted once the exit cancelled the set. Start(j) == - /\ alive /\ exec[j] = "none" /\ Cardinality(Permits) < JobBound + /\ alive \/ Fault = "NoAdmissionFence" + /\ exec[j] = "none" /\ Cardinality(Permits) < JobBound /\ exec' = [exec EXCEPT ![j] = "running"] /\ owner' = [owner EXCEPT ![j] = inst] /\ \E d \in BOOLEAN : hasDl' = [hasDl EXCEPT ![j] = d] /\ IF j \in Keyed /\ Fault # "KeyNoCancel" @@ -219,7 +227,8 @@ Cancel(j) == Exit == /\ alive /\ alive' = FALSE /\ CancelSet({j \in Jobs : owner[j] = inst - /\ (exec[j] = "running" \/ sched[j] > 0 \/ held[j])}) + /\ (exec[j] = "running" + \/ ((sched[j] > 0 \/ held[j]) /\ Fault # "ExitKeepsHeld"))}) /\ UNCHANGED <> Restart == @@ -285,11 +294,18 @@ TerminalLast == \A a, b \in 1..Len(mbox) : (a < b /\ mbox[a].j = mbox[b].j /\ IsTerm(mbox[a])) => FALSE EmitCreditsBounded == \A j \in Jobs : inflight[j] + sched[j] <= Credits +\* After the owner's exit nothing of its instance is live: running work, +\* scheduled emits and held results are all cancelled (their notification +\* suppressed, held results dropped). +ExitCancelsJobs == + ~alive => \A j \in Jobs : + (owner[j] = inst /\ (exec[j] = "running" \/ sched[j] > 0 \/ held[j])) => cancelled[j] Safety == /\ TypeOK /\ NoLateResult /\ NoStaleAfterCancel /\ NoStaleKeyed /\ AtMostOneTerminal /\ PermitsBounded /\ OneLivePerKey /\ TerminalLast /\ EmitCreditsBounded /\ HeldCounted /\ DeadlineArmedWhileHeld + /\ ExitCancelsJobs \* A job started with a deadline and never cancelled is reported to its \* live owner instance even if its work never finishes. From 49c4fe272afb9745159b1c2f289432714e2361e3 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 17:25:56 +0000 Subject: [PATCH 150/208] verification: model the death boundary in DeferredReplies (reply fence, answer batch, wake-ups after it) --- docs/design/deferred-replies.md | 16 +++- verification/tla/DeferredReplies.tla | 112 +++++++++++++++++++++------ verification/tla/TraceReplies.tla | 13 +++- 3 files changed, 112 insertions(+), 29 deletions(-) diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index 572ca597..7df2e4eb 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -891,6 +891,13 @@ It establishes, within its bounds: owner (cleanup deadline passed while `stopped()` runs) closes nothing early: no token is answered `Stopped` before death (`NoEarlyClose`). `stopped()` may still reply while stuck, and death may never follow. +- **Death boundary.** Death is the three steps of `actor_cleanup.rs`: the + death point fences the reply gate and seals the registry, then the batch + answers every held token and queued request one at a time (callers and the + deadline service interleave), then the collected wake-ups run. No reply is + admitted after the death point (`NoReplyAfterDeath`, control + `NoReplyFence`), and a caller woken with `Stopped` finds the whole death + answered (`WokenAfterDischarge`, control `CallbackBeforeDischarge`). - **Downstream cleanup.** A downstream request ends with its last job (`DownstreamOwned`). - **Liveness without handler fairness:** `EveryAcceptedDischarged` and @@ -913,6 +920,12 @@ It establishes: - **Terminal messages.** `AtMostOneTerminal` and `TerminalLast`. - **Bounds.** `PermitsBounded` (running work keeps its permit), `OneLivePerKey` and `EmitCreditsBounded`. +- **Held results.** A held result keeps its credit until its terminal + message (`HeldCounted`) and its deadline stays armed + (`DeadlineArmedWhileHeld`). +- **Owner exit.** Admission and `cancel_all` are serialized, and the exit + cancels running work, scheduled emits and held results alike + (`ExitCancelsJobs`, controls `NoAdmissionFence` and `ExitKeepsHeld`). - **Liveness.** `JobReported` for jobs with a deadline. Abstraction: credits return at dispatch (fenced or not); mailbox disposal at @@ -931,7 +944,8 @@ Requests and timing: Tokens and synchronization: - token drops by other tasks during an unrelated turn; -- `WRITING` versus registry close; +- `WRITING` versus registry close (the reply permit is modelled as the + reply gate closing at the death point; the CAS race is Loom territory); - the packed-word counter deltas; - memory orderings and wakeups. diff --git a/verification/tla/DeferredReplies.tla b/verification/tla/DeferredReplies.tla index 33e493b1..47441a27 100644 --- a/verification/tla/DeferredReplies.tla +++ b/verification/tla/DeferredReplies.tla @@ -31,8 +31,14 @@ (* pending" (pendA): the registry keeps it open and death answers it with *) (* the final reason (docs/design/otp-exit-signals.md, "Required changes *) (* to rank 11"); only the caller's deadline or abandonment comes first. *) -(* Death closes the registry (every token A holds, wherever it is, and *) -(* every exit-pending slot) and discards the mailbox. The model has no *) +(* Death follows actor_cleanup.rs. Death (the point D) fences the reply *) +(* gate (a reply already past it has discharged; none is admitted after) *) +(* and seals the registry ("sealed": no new request is admitted); callers,*) +(* the deadline service and stored replies race it. Answer then closes *) +(* every token A holds (wherever it is), every exit-pending slot and the *) +(* mailbox with Stopped, as one batch of terminal transitions, and runs *) +(* the collected wake-ups after the last answer ("dead"): no waker, *) +(* callback or destructor runs in between. The model has no *) (* reasons: the reason is fixed at death (Lifecycle.tla), so StoppedAtDeath*) (* (Stopped only in the death step) is what makes callers see it. *) (* If stopped() (or a destructor) is still *) @@ -58,6 +64,9 @@ (* registry before death *) (* EarlyStopped (StoppedAtDeath) exit errors answered Stopped at *) (* once (rank 11 before 5e) *) +(* NoReplyFence (NoReplyAfterDeath) a reply is admitted after D *) +(* CallbackBeforeDischarge (WokenAfterDischarge) an answer's wake-up *) +(* runs before the last answer *) (***************************************************************************) EXTENDS Naturals, FiniteSets CONSTANTS N, BoundA, BoundB, MaxTime, MaxWait, Fault @@ -65,7 +74,7 @@ ASSUME Fault \in {"None", "NoAutoReply", "NoAtomicExpiry", "LateSuccess", "LoopDeadline", "Lengthen", "Unbounded", "NoDestAdmission", "ExitStateOnly", "UnwindDropped", "ShareRunning", "AttachClosed", "LeakDownstream", "EarlyClose", - "EarlyStopped"} + "EarlyStopped", "NoReplyFence", "CallbackBeforeDischarge"} Req == 1..N \* qA queued; turnA PendingReply in a turn; localA ReplyTo in a turn; @@ -78,13 +87,15 @@ Terminal == {"ok", "timeout", "overloaded", "stopped", "dropped", "abandoned"} VARIABLES now, caller, sent, dl, slot, holder, discharges, okAt, jobs, \* running linked jobs per request (0..2) down, \* downstream request to B: none, qB, stateB, done, abandoned - phase, \* A: run, exiting, stuck (cleanup deadline passed), dead + phase, \* A: run, exiting, stuck (cleanup deadline passed), + \* sealed (past D, answering), dead (answered, woken) turnA, \* request in A's turn, 0 if idle panicked, \* history: requests whose token was in a panicking turn - earlyClosed \* history: tokens answered Stopped by a close before death + earlyClosed, \* history: tokens answered Stopped by a close before death + lateOk \* history: requests answered Ok by a reply admitted after D vars == <> + phase, turnA, panicked, earlyClosed, lateOk>> Init == /\ now = 0 /\ caller = [r \in Req |-> "idle"] @@ -92,7 +103,7 @@ Init == /\ slot = [r \in Req |-> "none"] /\ holder = [r \in Req |-> "none"] /\ discharges = [r \in Req |-> 0] /\ okAt = [r \in Req |-> 0] /\ jobs = [r \in Req |-> 0] /\ down = [r \in Req |-> "none"] - /\ phase = "run" /\ turnA = 0 /\ panicked = {} /\ earlyClosed = {} + /\ phase = "run" /\ turnA = 0 /\ panicked = {} /\ earlyClosed = {} /\ lateOk = {} Open(r) == slot[r] = "open" TokenHeld(r) == Open(r) /\ holder[r] \in {"localA", "dropA", "stateA", "doneA"} @@ -139,7 +150,7 @@ U(v) == UNCHANGED v \* Callers Send(r) == - /\ caller[r] = "idle" /\ phase # "dead" + /\ caller[r] = "idle" /\ phase \notin {"sealed", "dead"} /\ \E d \in 1..MaxWait : now + d <= MaxTime /\ sent' = [sent EXCEPT ![r] = now + d] /\ dl' = [dl EXCEPT ![r] = now + d] /\ slot' = [slot EXCEPT ![r] = "open"] /\ holder' = [holder EXCEPT ![r] = "qA"] @@ -238,9 +249,14 @@ Panic == /\ U(<>) \* A later turn, or stopped() while exiting, answers or drops a stored token. +\* The reply gate admits it only before D (the reply permit; Death waits +\* out a reply already admitted). +ReplyGateOpen == phase \in {"run", "exiting", "stuck"} + \/ (Fault = "NoReplyFence" /\ phase = "sealed") LaterReply(r) == - /\ phase \in {"run", "exiting", "stuck"} /\ turnA = 0 /\ holder[r] = "stateA" + /\ ReplyGateOpen /\ turnA = 0 /\ holder[r] = "stateA" /\ holder' = [holder EXCEPT ![r] = "gone"] /\ TryReply(r) + /\ lateOk' = IF phase = "sealed" /\ Open(r) /\ now < dl[r] THEN lateOk \cup {r} ELSE lateOk /\ U(<>) LaterDrop(r) == @@ -264,12 +280,49 @@ CleanupDeadline == ELSE Keep /\ UNCHANGED earlyClosed /\ U(<>) +\* The death point D: the reply gate is fenced and the registry sealed. Death == - /\ phase \in {"exiting", "stuck"} /\ phase' = "dead" - /\ DischargeSet(OpenOf({r \in Req : holder[r] \in - (IF Fault = "ExitStateOnly" THEN {"qA", "stateA", "pendA"} - ELSE {"qA", "stateA", "doneA", "pendA"})}), "stopped") - /\ U(<>) + /\ phase \in {"exiting", "stuck"} /\ phase' = "sealed" + /\ Keep /\ U(<>) + +\* What death answers: every token A holds and every exit-pending slot. +DeathHeld == OpenOf({r \in Req : holder[r] \in + (IF Fault = "ExitStateOnly" THEN {"qA", "stateA", "pendA"} + ELSE {"qA", "stateA", "doneA", "pendA"})}) + +\* The batch of answers and then its collected wake-ups. Each slot's +\* transition is its own CAS, and the only other steps on a held slot +\* (expiry, abandonment) are terminal transitions of that slot, so an +\* interleaving of them with the batch reaches the same states as one taken +\* before it; the wake-ups follow the batch with no user code in between. +\* So the batch and its wake-ups are one step (Answer); what the model keeps +\* apart is D (Death) and the batch, with callers, the deadline service and +\* replies racing D in between. CallbackBeforeDischarge answers one slot at +\* a time, waking its caller at once, and ends with Discharged. +Answer(r) == + /\ phase = "sealed" + /\ IF Fault = "CallbackBeforeDischarge" + THEN /\ r \in DeathHeld + /\ slot' = [slot EXCEPT ![r] = "stopped"] + /\ discharges' = [discharges EXCEPT ![r] = @ + 1] + /\ caller' = IF caller[r] = "waiting" THEN [caller EXCEPT ![r] = "err"] ELSE caller + /\ UNCHANGED phase + ELSE /\ r = 1 + /\ slot' = [q \in Req |-> IF q \in DeathHeld THEN "stopped" ELSE slot[q]] + /\ discharges' = [q \in Req |-> IF q \in DeathHeld THEN discharges[q] + 1 + ELSE discharges[q]] + /\ caller' = [q \in Req |-> IF q \in DeathHeld /\ caller[q] = "waiting" + THEN "err" ELSE caller[q]] + /\ phase' = "dead" + /\ U(<>) + +\* The batch is done: the collected wake-ups run. +Discharged == + /\ Fault = "CallbackBeforeDischarge" + /\ phase = "sealed" /\ DeathHeld = {} /\ phase' = "dead" + /\ caller' = [r \in Req |-> IF slot[r] = "stopped" /\ caller[r] = "waiting" + THEN "err" ELSE caller[r]] + /\ U(<>) ------------------------------------------------------------------------------ \* Linked jobs and the downstream actor B @@ -343,21 +396,26 @@ Tick == ------------------------------------------------------------------------------ NextOther == - \/ Tick \/ EndTurn \/ Panic \/ ExitBegin \/ Death + \/ Tick \/ EndTurn \/ Panic \/ ExitBegin \/ Death \/ Discharged \/ \E r \in Req : + \/ Answer(r) \/ Send(r) \/ CallerGivesUp(r) \/ DispatchA(r) \/ ReplyNow(r) \/ Defer(r) - \/ Store(r) \/ DropInTurn(r) \/ PipeReply(r) \/ LaterReply(r) \/ LaterDrop(r) + \/ Store(r) \/ DropInTurn(r) \/ PipeReply(r) \/ LaterDrop(r) \/ JobNotify(r) \/ JobEnds(r) \/ JobCancel(r) \/ JobCallB(r) \/ BDispatch(r) \/ BReply(r) \/ Shorten(r) \/ Expire(r) Next == - \/ CleanupDeadline - \/ (UNCHANGED earlyClosed /\ NextOther) + \/ CleanupDeadline /\ UNCHANGED lateOk + \/ \E r \in Req : LaterReply(r) /\ UNCHANGED earlyClosed + \/ (UNCHANGED <> /\ NextOther) Spec == Init /\ [][Next]_vars -\* Fairness only for the clock and the deadline service. -FairSpec == Spec /\ WF_vars(Tick /\ UNCHANGED earlyClosed) - /\ \A r \in Req : WF_vars(Expire(r) /\ UNCHANGED earlyClosed) +\* Fairness for the clock, the deadline service and the steps of death +\* after D (framework code: no user code runs between them). +Quiet == UNCHANGED <> +FairSpec == Spec /\ WF_vars(Tick /\ Quiet) + /\ \A r \in Req : WF_vars(Expire(r) /\ Quiet) /\ WF_vars(Answer(r) /\ Quiet) + /\ WF_vars(Discharged /\ Quiet) ------------------------------------------------------------------------------ \* Properties @@ -365,7 +423,7 @@ FairSpec == Spec /\ WF_vars(Tick /\ UNCHANGED earlyClosed) TypeOK == /\ now \in 0..MaxTime /\ holder \in [Req -> Holders] /\ slot \in [Req -> {"none", "open"} \cup Terminal] - /\ jobs \in [Req -> 0..2] /\ phase \in {"run", "exiting", "stuck", "dead"} + /\ jobs \in [Req -> 0..2] /\ phase \in {"run", "exiting", "stuck", "sealed", "dead"} AtMostOneDischarge == \A r \in Req : discharges[r] <= 1 NoOrphan == @@ -383,7 +441,13 @@ PanicStopped == \A r \in panicked : slot[r] \in {"open", "stopped", "timeout", " \* Exit errors wait for death: a Stopped discharge happens only in the death \* step, where the reason is final (otp-exit-signals.md; the same property \* Lifecycle.tla checks as StoppedAtDeath). -StoppedAtDeath == \A r \in Req : slot[r] = "stopped" => phase = "dead" +StoppedAtDeath == \A r \in Req : slot[r] = "stopped" => phase \in {"sealed", "dead"} +\* No reply is admitted after D: an Ok always comes from a live instance. +NoReplyAfterDeath == lateOk = {} +\* A caller woken with Stopped finds the whole death answered: no answer's +\* wake-up (user code) runs while another request A held is still open. +WokenAfterDischarge == + (\E r \in Req : slot[r] = "stopped" /\ caller[r] = "err") => DeathHeld = {} \* The registry closes only at actual death: no token is answered Stopped \* by a close while the instance (its stopped()) is still alive. NoEarlyClose == earlyClosed = {} @@ -396,7 +460,7 @@ Safety == /\ TypeOK /\ AtMostOneDischarge /\ NoOrphan /\ NoSuccessAfterDeadline /\ TimelyExpiry /\ DeadlineWithinCaller /\ BoundedA /\ BoundedB /\ PanicStopped /\ StoppedAtDeath /\ NoEarlyClose /\ DownstreamOwned - /\ CallerConsistent + /\ CallerConsistent /\ NoReplyAfterDeath /\ WokenAfterDischarge EveryAcceptedDischarged == \A r \in Req : (slot[r] = "open") ~> (slot[r] \in Terminal) CallerNeverHangs == \A r \in Req : (caller[r] = "waiting") ~> (caller[r] # "waiting") diff --git a/verification/tla/TraceReplies.tla b/verification/tla/TraceReplies.tla index 898327b4..7f91c902 100644 --- a/verification/tla/TraceReplies.tla +++ b/verification/tla/TraceReplies.tla @@ -8,8 +8,9 @@ (* Discharge (the slot's one OPEN -> terminal transition, recorded in the *) (* trace section together with the CAS; a reply counts at WRITING), Exit *) (* (the registry enters exiting, after committing the turn's drops) and *) -(* Death (the registry closes; the closes it and the mailbox disposal *) -(* perform belong to this one model step and are not logged). *) +(* Death (the death point D: the reply gate is fenced and the registry *) +(* sealed). The answers death then gives and their wake-ups (Answer) are *) +(* not logged and are taken silently. *) (* *) (* Turn-internal steps leave no trace and are taken silently: Defer that *) (* takes a credit (or finds the slot closed), Store, DropInTurn, a turn *) @@ -55,10 +56,14 @@ Silent == \/ \E r \in Req : Defer(r) \/ Store(r) \/ DropInTurn(r) \/ LaterDrop(r) /\ UNCHANGED slot +\* The steps of death after D, which leave no trace. +DeathSilent == \/ \E r \in Req : Answer(r) + \/ Discharged + TraceNext == /\ i < Len(Trace) - /\ \/ Step(Trace[i + 1]) /\ UNCHANGED earlyClosed /\ i' = i + 1 - \/ Silent /\ UNCHANGED earlyClosed /\ UNCHANGED i + /\ \/ Step(Trace[i + 1]) /\ UNCHANGED <> /\ i' = i + 1 + \/ (Silent \/ DeathSilent) /\ UNCHANGED <> /\ UNCHANGED i TraceSpec == TraceInit /\ [][TraceNext]_traceVars From 681b4d6ea2405121d077548d7e4dbcf88686bd64 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 17:25:56 +0000 Subject: [PATCH 151/208] verification: split Lifecycle death into D and Discharge, model the exit-signal fence --- verification/tla/Lifecycle.tla | 117 +++++++++++++++-------- verification/tla/MCLifecycle.cfg | 2 + verification/tla/MCLifecycleDeep.cfg | 2 + verification/tla/MCLifecycleFence.cfg | 2 + verification/tla/MCLifecycleLive.cfg | 2 + verification/tla/MCLifecycleLiveDeep.cfg | 2 + verification/tla/MCLifecycleWideDeep.cfg | 2 + verification/tla/TraceLifecycle.tla | 6 +- 8 files changed, 94 insertions(+), 41 deletions(-) diff --git a/verification/tla/Lifecycle.tla b/verification/tla/Lifecycle.tla index 65225678..a9eb9d79 100644 --- a/verification/tla/Lifecycle.tla +++ b/verification/tla/Lifecycle.tla @@ -10,10 +10,11 @@ EXTENDS Naturals, Sequences, FiniteSets (* "run" dispatching; *) (* "stopping" exit begun: no dispatch; stopped() may run and send; an *) (* untrappable kill still replaces the requested reason; *) -(* "gone" past the death point D: the reason is final, the mailbox *) -(* refuses admission, every queued request was answered, the *) +(* "gone" past the death point D: the reason is final, outgoing exit *) +(* signals are fenced, the mailbox refuses admission, the *) (* link and monitor table is closed and one notice per edge *) -(* was taken; destruction (state drop) is in progress; *) +(* was taken; then (Discharge) every queued request is *) +(* answered, and destruction (state drop) follows; *) (* "dead" published: wait_exit/join return; the notices taken at D *) (* may now be delivered. *) (* Registration (link, monitor) and request admission are atomic with D: *) @@ -77,6 +78,9 @@ Past(a, s) == s \in {"gone", "dead"} (* DropKeepsNormal a runtime drop keeps a requested Normal *) (* AlwaysHook v0.7: stopped() runs after a kill or untrapped signal *) (* LateSignalIgnored v0.7: a late signal during exit keeps the reason *) +(* NoSignalFence an exit signal sent by an actor takes effect after D *) +(* PublishBeforeDischarge publication (join, notices) before every queued *) +(* request was answered *) KilledReason == IF Fault = "KillAsKill" THEN "Kill" ELSE "Killed" VARIABLES @@ -100,16 +104,18 @@ VARIABLES hook, \* [Actors -> BOOLEAN]: stopped() runs (ghost) began, \* [Actors -> causes]: what began the exit (ghost) lateSig, \* [Actors -> reason of the last late signal applied, or None] (ghost) + disch, \* [Actors -> BOOLEAN]: past D, the queued requests were answered + sigAfterD, \* an exit signal took effect after its sender's D (ghost) sends, signals, lates, psent, \* children that received the supervisor's Shutdown afterStop \* a message was dispatched after stop was observed (ghost) vars == <> -life == <> + lates, psent, afterStop, disch, sigAfterD>> +life == <> reg == <> -budget == <> +budget == <> \* Mailbox items. o: "death" (link or monitor notice taken at D), "late" \* (registration after D), "direct" (an exit signal sent by a live sender), @@ -146,6 +152,7 @@ Init == /\ hook = [a \in Actors |-> FALSE] /\ began = [a \in Actors |-> None] /\ lateSig = [a \in Actors |-> None] + /\ disch = [a \in Actors |-> FALSE] /\ sigAfterD = FALSE /\ sends = 0 /\ signals = 0 /\ lates = 0 /\ psent = {} /\ afterStop = FALSE @@ -162,7 +169,7 @@ Begin(a, r, c) == \* skips terminate). /\ hook' = [hook EXCEPT ![a] = c \in {"self", "admin", "parent"} \/ Fault = "AlwaysHook"] /\ began' = [began EXCEPT ![a] = c] - /\ UNCHANGED <> + /\ UNCHANGED <> ELSE UNCHANGED life (* An untrappable kill: begins exit, or replaces the requested reason of an *) @@ -176,14 +183,14 @@ Kill(a) == /\ killed' = [killed EXCEPT ![a] = TRUE] /\ hook' = [hook EXCEPT ![a] = Fault = "AlwaysHook"] /\ began' = [began EXCEPT ![a] = "kill"] - /\ UNCHANGED <> + /\ UNCHANGED <> ELSE IF st[a] = "stopping" THEN /\ why' = IF Fault = "KillNoSupersede" THEN why ELSE [why EXCEPT ![a] = KilledReason] /\ cause' = [cause EXCEPT ![a] = "kill"] /\ killed' = [killed EXCEPT ![a] = TRUE] /\ lateSig' = [lateSig EXCEPT ![a] = None] \* a stopped() that already began runs to its end (R) - /\ UNCHANGED <> + /\ UNCHANGED <> ELSE UNCHANGED life (* An exit signal from s with reason r and origin o arrives at a. c is the *) @@ -196,7 +203,7 @@ Signal(a, s, r, o, c) == THEN /\ why' = IF Fault = "LateSignalIgnored" THEN why ELSE [why EXCEPT ![a] = r] /\ cause' = IF Fault = "LateSignalIgnored" THEN cause ELSE [cause EXCEPT ![a] = c] /\ lateSig' = [lateSig EXCEPT ![a] = r] - /\ UNCHANGED <> + /\ UNCHANGED <> ELSE IF st[a] # "run" THEN UNCHANGED <> ELSE IF trap[a] THEN /\ Push(a, Item(IF IsParent(a, s) /\ Fault # "ParentAsExit" THEN "pexit" ELSE "exit", @@ -220,23 +227,33 @@ Fire(a, r) == ELSE mstate[m]] /\ fired' = fired \cup {a} -(* The death point D of a with final reason r: admission closes, queued *) -(* requests are answered, the table closes. Shared by Die and RuntimeDrop. *) -(* The code fences outgoing exit signals and replies first, then seals and *) -(* closes the table, then answers deferred and queued requests, and only *) -(* then runs the answers' wake-ups and logging, and destroys payloads and *) -(* state (actor_cleanup.rs). Between the seal and the last answer the code *) -(* runs only terminal transitions of slots (no waker, subscriber, callback *) -(* or destructor: wake-ups are collected and run after the batch), and no *) -(* other step depends on those answers, so they commute left to the seal: *) -(* this atomic step is a sound abstraction. *) +(* The death point D of a with final reason r, as in actor_cleanup.rs: *) +(* outgoing exit signals (and replies) are fenced, admission closes and *) +(* the table closes. Shared by Die and RuntimeDrop. The queued and exit- *) +(* pending requests are answered by the separate Discharge step, which *) +(* other actors' steps may precede; publication waits for it. (Deferred *) +(* replies and the wake-up batch are modelled step by step in *) +(* DeferredReplies.tla.) With no request to answer, Discharge only drops *) +(* the mailbox, which nothing reads past D, so it is folded into Death. *) +Pending(a) == \E q \in Reqs : req[q].s \in {"queued", "exiting"} /\ req[q].t = a Death(a, r) == /\ st' = [st EXCEPT ![a] = "gone"] + /\ IF Pending(a) THEN UNCHANGED <> + ELSE /\ mbox' = [mbox EXCEPT ![a] = <<>>] + /\ disch' = [disch EXCEPT ![a] = TRUE] + /\ IF a \in fired THEN UNCHANGED <> ELSE Fire(a, r) + +(* Past D: every queued or exit-pending request is answered ActorStopped *) +(* with the final reason; the mailbox is disposed of. *) +Discharge(a) == + /\ st[a] = "gone" /\ ~disch[a] + /\ disch' = [disch EXCEPT ![a] = TRUE] /\ mbox' = [mbox EXCEPT ![a] = <<>>] /\ req' = [q \in Reqs |-> IF req[q].s \in {"queued", "exiting"} /\ req[q].t = a /\ Fault # "DropQueued" - THEN [req[q] EXCEPT !.s = "stopped", !.why = r] ELSE req[q]] - /\ IF a \in fired THEN UNCHANGED <> ELSE Fire(a, r) + THEN [req[q] EXCEPT !.s = "stopped", !.why = why[a]] ELSE req[q]] + /\ UNCHANGED <> ---------------------------------------------------------------------------- (* Turns of a running actor. *) @@ -251,7 +268,7 @@ Send(a, b) == \* also from stopped(), when it runs /\ a # b /\ st[b] \in {"run", "stopping"} /\ sends < MaxSends /\ sends' = sends + 1 /\ Push(b, Item("msg", a, None, 0, 0, "none")) - /\ UNCHANGED <> + /\ UNCHANGED <> SelfExit(a, r) == \* ctx.stop, ctx.stop_with_reason(r), a panic in any turn /\ st[a] = "run" /\ r \in SelfReasons @@ -267,7 +284,7 @@ PanicWhileExiting(a) == /\ why' = [why EXCEPT ![a] = "Panic"] /\ cause' = [cause EXCEPT ![a] = "self"] /\ lateSig' = [lateSig EXCEPT ![a] = None] - /\ UNCHANGED <> + /\ UNCHANGED <> Link(a, b) == \* idempotent while a link or a pending link notice exists /\ st[a] = "run" /\ a # b /\ {a, b} \notin links @@ -285,7 +302,7 @@ Link(a, b) == \* idempotent while a link or a pending link notice exists ELSE /\ links' = links \cup {{a, b}} /\ UNCHANGED <> /\ UNCHANGED <> + req, sends, signals, psent, afterStop, sigAfterD>> (* Unlink: removes the link, and cancels a notice from b already taken at *) (* b's death but not yet delivered (OTP: once unlink returns, the link has *) @@ -295,7 +312,7 @@ Link(a, b) == \* idempotent while a link or a pending link notice exists TryLink(a, b) == /\ st[a] = "run" /\ a # b /\ IsGone(b) /\ lates < MaxLate /\ lates' = lates + 1 - /\ UNCHANGED <> + /\ UNCHANGED <> Unlink(a, b) == LET pending == {n \in inflight : n.to = a /\ n.from = b /\ n.k = "link"} @@ -318,7 +335,7 @@ Monitor(a, b, m) == ELSE /\ mstate' = [mstate EXCEPT ![m] = "active"] /\ UNCHANGED <> /\ UNCHANGED <> + req, sends, signals, psent, afterStop, sigAfterD>> (* Demonitor: no Down is enqueued after it returns; one already queued stays. *) Demonitor(a, m) == @@ -361,7 +378,7 @@ Die(a) == /\ st[a] = "stopping" /\ Death(a, why[a]) /\ UNCHANGED <> + budget, afterStop, req>> (* The runtime drops the actor (tasks: the future is dropped; threads: the *) (* closure never ran). No stopped(); reason Killed unless a non-Normal one *) @@ -375,7 +392,8 @@ RuntimeDrop(a) == /\ cause' = [cause EXCEPT ![a] = IF why[a] \in {None, "Normal"} THEN "drop" ELSE cause[a]] /\ trapAt' = [trapAt EXCEPT ![a] = IF st[a] = "run" THEN trap[a] ELSE trapAt[a]] /\ Death(a, r) - /\ UNCHANGED <> + /\ UNCHANGED <> EarlyFire(a) == \* Fault EarlyNotify only /\ Fault = "EarlyNotify" /\ st[a] = "stopping" /\ a \notin fired @@ -384,9 +402,10 @@ EarlyFire(a) == \* Fault EarlyNotify only (* Destruction finished: the exit is published. *) Publish(a) == - /\ st[a] = "gone" + /\ st[a] = "gone" /\ (disch[a] \/ Fault = "PublishBeforeDischarge") /\ st' = [st EXCEPT ![a] = "dead"] - /\ UNCHANGED <> + /\ UNCHANGED <> (* A notice taken at D is delivered after publication; late notices at once. *) Deliver(n) == @@ -426,7 +445,11 @@ Enqueue(r) == (* ChildHandle::exit(r) from outside (s = Ext). r = Kill is kill(). *) ExitSignal(s, a, r) == /\ r \in SignalReasons /\ signals < MaxSignals /\ signals' = signals + 1 - /\ IF s = Ext THEN TRUE ELSE st[s] \in {"run", "stopping"} \* inert after D + \* The signal gate: inert after D (D waits out a signal already past it). + /\ IF s = Ext THEN TRUE + ELSE st[s] \in {"run", "stopping"} + \/ (Fault = "NoSignalFence" /\ st[s] = "gone" /\ ~disch[s]) + /\ sigAfterD' = (sigAfterD \/ (s # Ext /\ Past(s, st[s]))) /\ IF r = "Kill" THEN /\ Kill(a) /\ UNCHANGED mbox ELSE IF Fault = "DirectStop" @@ -437,13 +460,13 @@ ExitSignal(s, a, r) == AdminStop(a, r) == \* ChildHandle::stop / stop_with_reason / shutdown /\ r \in AdminReasons /\ signals < MaxSignals /\ signals' = signals + 1 /\ Begin(a, r, "admin") - /\ UNCHANGED <> + /\ UNCHANGED <> SupShutdown(c) == \* exit(Child, shutdown) from the parent, also to a child /\ c \in Children /\ c \notin psent /\ st[c] \in {"run", "stopping"} \* already exiting /\ psent' = psent \cup {c} /\ Signal(c, Sup, "Shutdown", "direct", "psignal") - /\ UNCHANGED <> + /\ UNCHANGED <> SupKill(c) == \* shutdown_timeout passed: exit(Child, kill) /\ c \in psent /\ st[c] \in {"run", "stopping"} @@ -460,6 +483,7 @@ Next == \/ \E r \in Reasons : SelfExit(a, r) \/ AdminStop(a, r) \/ \E s \in Actors \cup {Ext} : ExitSignal(s, a, r) \/ Dispatch(a) \/ Die(a) \/ RuntimeDrop(a) \/ EarlyFire(a) \/ Publish(a) + \/ Discharge(a) \/ SupShutdown(a) \/ SupKill(a) \/ \E n \in inflight : Deliver(n) \/ \E r \in Reqs : Enqueue(r) \/ \E t \in Actors : Request(r, t) @@ -467,6 +491,7 @@ Next == Fairness == /\ \A a \in Actors : WF_vars(Dispatch(a)) /\ WF_vars(Die(a)) /\ WF_vars(EarlyFire(a)) /\ WF_vars(Publish(a)) + /\ WF_vars(Discharge(a)) /\ WF_vars(\E n \in inflight : Deliver(n)) /\ \A r \in Reqs : WF_vars(Enqueue(r)) @@ -485,19 +510,32 @@ TypeOK == /\ mstate \in [MonIds -> {"unused", "active", "fired", "cancelled", "late"}] /\ \A r \in Reqs : req[r].s \in {"idle", "sending", "queued", "exiting"} \cup Done /\ \A a \in Actors : (st[a] = "run") <=> (why[a] = None) + /\ disch \in [Actors -> BOOLEAN] /\ sigAfterD \in BOOLEAN + /\ \A a \in Actors : disch[a] => Past(a, st[a]) -\* A queued request is in exactly one live mailbox; past D it was answered -\* (PR #182; rank 11 death boundary). +\* A queued request is in exactly one mailbox, of an actor not yet past its +\* discharge: nothing is admitted after D, and every request queued before +\* it is answered before publication (PR #182; rank 11 death boundary). +Undischarged(a) == st[a] \in {"run", "stopping"} \/ (st[a] = "gone" /\ ~disch[a]) NoOrphanRequest == \A r \in Reqs : req[r].s = "queued" => - /\ st[req[r].t] \in {"run", "stopping"} + /\ Undischarged(req[r].t) /\ Len(SelectSeq(mbox[req[r].t], LAMBDA x : x.k = "req" /\ x.q = r)) = 1 +\* Publication (join, wait_exit, the notices taken at D) comes after every +\* queued and exit-pending request was answered. +AnsweredBeforePublish == + \A r \in Reqs : req[r].s \in {"queued", "exiting"} => st[req[r].t] # "dead" + +\* An actor's exit signals all take effect before its D (the signal gate). +SignalsBeforeDeath == ~sigAfterD + \* An exit error reaches a caller only at D, with the final reason; until \* then a request whose handler panicked is held (rank 11 slot) open. StoppedAtDeath == \A r \in Reqs : - /\ req[r].s = "exiting" => st[req[r].t] = "stopping" + /\ req[r].s = "exiting" => + (st[req[r].t] = "stopping" \/ (st[req[r].t] = "gone" /\ ~disch[req[r].t])) /\ req[r].s = "stopped" => Past(req[r].t, st[req[r].t]) /\ req[r].why = why[req[r].t] \* No message is dispatched once the actor observed its stop (PR #185). @@ -565,7 +603,8 @@ Safety == /\ TypeOK /\ NoOrphanRequest /\ NoDispatchAfterStop /\ DownAtMostOnce /\ NoProcIffLate /\ TrapRespected /\ NormalSignalIgnored /\ KilledFromKill /\ NoEdgeToDead /\ ExitAfterMessages /\ CancelFenced /\ StoppedAtDeath - /\ RuntimeDropReason /\ SkipHook /\ LateSignalWins + /\ RuntimeDropReason /\ SkipHook /\ LateSignalWins /\ AnsweredBeforePublish + /\ SignalsBeforeDeath ---------------------------------------------------------------------------- (* Liveness (under FairSpec: handlers, stopped() and destruction return) *) diff --git a/verification/tla/MCLifecycle.cfg b/verification/tla/MCLifecycle.cfg index 3fcfcd6f..04974cea 100644 --- a/verification/tla/MCLifecycle.cfg +++ b/verification/tla/MCLifecycle.cfg @@ -30,3 +30,5 @@ INVARIANT StoppedAtDeath INVARIANT RuntimeDropReason INVARIANT SkipHook INVARIANT LateSignalWins +INVARIANT AnsweredBeforePublish +INVARIANT SignalsBeforeDeath diff --git a/verification/tla/MCLifecycleDeep.cfg b/verification/tla/MCLifecycleDeep.cfg index 763ae745..c8eacadf 100644 --- a/verification/tla/MCLifecycleDeep.cfg +++ b/verification/tla/MCLifecycleDeep.cfg @@ -30,3 +30,5 @@ INVARIANT StoppedAtDeath INVARIANT RuntimeDropReason INVARIANT SkipHook INVARIANT LateSignalWins +INVARIANT AnsweredBeforePublish +INVARIANT SignalsBeforeDeath diff --git a/verification/tla/MCLifecycleFence.cfg b/verification/tla/MCLifecycleFence.cfg index 988fb4ae..edd014fe 100644 --- a/verification/tla/MCLifecycleFence.cfg +++ b/verification/tla/MCLifecycleFence.cfg @@ -30,3 +30,5 @@ INVARIANT StoppedAtDeath INVARIANT RuntimeDropReason INVARIANT SkipHook INVARIANT LateSignalWins +INVARIANT AnsweredBeforePublish +INVARIANT SignalsBeforeDeath diff --git a/verification/tla/MCLifecycleLive.cfg b/verification/tla/MCLifecycleLive.cfg index bccbcf9e..04878343 100644 --- a/verification/tla/MCLifecycleLive.cfg +++ b/verification/tla/MCLifecycleLive.cfg @@ -35,3 +35,5 @@ PROPERTY DownDelivered PROPERTY LinkedPeersStop INVARIANT SkipHook INVARIANT LateSignalWins +INVARIANT AnsweredBeforePublish +INVARIANT SignalsBeforeDeath diff --git a/verification/tla/MCLifecycleLiveDeep.cfg b/verification/tla/MCLifecycleLiveDeep.cfg index c4cfc3d0..769b6213 100644 --- a/verification/tla/MCLifecycleLiveDeep.cfg +++ b/verification/tla/MCLifecycleLiveDeep.cfg @@ -35,3 +35,5 @@ PROPERTY DownDelivered PROPERTY LinkedPeersStop INVARIANT SkipHook INVARIANT LateSignalWins +INVARIANT AnsweredBeforePublish +INVARIANT SignalsBeforeDeath diff --git a/verification/tla/MCLifecycleWideDeep.cfg b/verification/tla/MCLifecycleWideDeep.cfg index 59ec56ed..df72af04 100644 --- a/verification/tla/MCLifecycleWideDeep.cfg +++ b/verification/tla/MCLifecycleWideDeep.cfg @@ -30,3 +30,5 @@ INVARIANT StoppedAtDeath INVARIANT RuntimeDropReason INVARIANT SkipHook INVARIANT LateSignalWins +INVARIANT AnsweredBeforePublish +INVARIANT SignalsBeforeDeath diff --git a/verification/tla/TraceLifecycle.tla b/verification/tla/TraceLifecycle.tla index 801957da..17674b30 100644 --- a/verification/tla/TraceLifecycle.tla +++ b/verification/tla/TraceLifecycle.tla @@ -136,10 +136,12 @@ Step(e) == [] e.a = "Publish" -> Publish(e.t) [] OTHER -> FALSE +\* Answering the queued requests after D is not logged: it is taken +\* silently between Die (or Drop) and Publish, which waits for it. TraceNext == /\ i < Len(Trace) - /\ Step(Trace[i + 1]) - /\ i' = i + 1 + /\ \/ Step(Trace[i + 1]) /\ i' = i + 1 + \/ \E a \in Actors : Discharge(a) /\ UNCHANGED i TraceSpec == TraceInit /\ [][TraceNext]_traceVars From 98d1c387323cd50582b71856bb06d9f23a7351f8 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 17:28:13 +0000 Subject: [PATCH 152/208] verification: wire the new model controls, document them, refresh the coverage cache --- verification/tla/README.md | 13 +++++++++---- verification/tla/coverage-model.json.gz | Bin 34696 -> 34697 bytes verification/tla/negative_checks.py | 8 ++++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/verification/tla/README.md b/verification/tla/README.md index aec00049..39ec14d6 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -16,7 +16,7 @@ this fits with Loom, Shuttle, Miri, ThreadSanitizer and mutation testing. | `NestedSupervision.tla`, `MCNested.tla/.cfg` | Dynamic subtree ownership and shutdown delivery; parent scope completion waits for descendant completion. Bounded three-node/two-generation model with fair cleanup. | | `DeferredReplies.tla`, `MCDeferredReplies*.tla/.cfg` | Rank 11 design model (not implemented), part 1: reply slots, send-time deadlines with shortening, deferral credits shared by a token and its first linked job, the exiting phase with exit-pending drops (unwound or dropped while exiting, answered at death with the final reason), a stuck phase (cleanup deadline passed while `stopped()` runs: nothing closed early, death later or never), death, and downstream request cleanup. Fourteen named negative controls. | | `DeferredJobs.tla`, `MCDeferredJobs*.tla/.cfg` | Rank 11 design model, part 2: pipe jobs with separate execution, cancellation and notification states; keyed replacement including notified jobs; optional job deadlines; a FIFO mailbox with emit credits and the terminal message last; `ActorId` fencing. `MCDeferredJobsEmits` is the fast configuration with emits; `MCDeferredJobsHeld` the one with delayed emits (`emit_at`) and held results (completion, held notification, scheduled emits and credit ownership as separate steps). Eleven named negative controls. | -| `Lifecycle.tla`, `MCLifecycle{,Live,Fence,Deep,LiveDeep,WideDeep}.tla/.cfg` | Base actor lifecycle as proposed for v0.8 ([design](../../docs/design/otp-exit-signals.md)): links, monitors, `trap_exit`, exit signals (`Killed`, ignored `Normal`, `NoProc`, parent exits), administrative stops, requests around death, `Exit`/`Down` ordering. Seventeen negative-control faults (17 safety checks, 2 liveness checks): the v0.7 exit-signal behaviours (direct stops ignoring `trap_exit`, kills ending `Kill`, unfenced `unlink`/`demonitor`, real reasons for late registrations, `stopped()` after kills and untrapped signals, late signals ignored during exit), the PR #182/#185 bugs, early exit errors, kill and panic precedence, runtime-drop reasons, early notification, a split death point and a parent exit reaching `exit_received`. Per PR: `MCLifecycle` (7.5M states, fewer reasons, where the safety controls run), `MCLifecycleFence`, `MCLifecycleLive`; weekly: `MCLifecycleWideDeep` (the former 43M-state per-PR instance), `MCLifecycleDeep`, `MCLifecycleLiveDeep`. | +| `Lifecycle.tla`, `MCLifecycle{,Live,Fence,Deep,LiveDeep,WideDeep}.tla/.cfg` | Base actor lifecycle as proposed for v0.8 ([design](../../docs/design/otp-exit-signals.md)): links, monitors, `trap_exit`, exit signals (`Killed`, ignored `Normal`, `NoProc`, parent exits), administrative stops, requests around death, `Exit`/`Down` ordering. Nineteen negative-control faults (19 safety checks, 2 liveness checks): the v0.7 exit-signal behaviours (direct stops ignoring `trap_exit`, kills ending `Kill`, unfenced `unlink`/`demonitor`, real reasons for late registrations, `stopped()` after kills and untrapped signals, late signals ignored during exit), the PR #182/#185 bugs, early exit errors, kill and panic precedence, runtime-drop reasons, early notification, a split death point, a parent exit reaching `exit_received`, an exit signal past the sender's death point (`NoSignalFence`) and publication before the queued requests were answered (`PublishBeforeDischarge`). Death is two steps, as in `actor_cleanup.rs`: the death point (`Die`/`RuntimeDrop`: signal fence, seal, table closed) and `Discharge` (queued and exit-pending requests answered), with other actors' steps in between; `Publish` waits for `Discharge`. Per PR: `MCLifecycle` (10.0M states, fewer reasons, where the safety controls run), `MCLifecycleFence`, `MCLifecycleLive`; weekly: `MCLifecycleWideDeep` (the former 43M-state per-PR instance), `MCLifecycleDeep`, `MCLifecycleLiveDeep`. | | `TraceCohorts.tla`, `TraceNested.tla` | Policy and ownership projections of real cohort/nested Rust executions. | | `trace_check.py`, `negative_checks.py` | Replay recorded traces; deliberately wrong selections, premature resume/join and broken propagation must be rejected or produce counterexamples. | | `SupervisionInd.tla`, `CohortsInd.tla`, `NestedSupervisionInd.tla`, `CoordinatorInd.tla`, `MC*Ind.tla`, `apalache.sh` | Inductive invariants (`IndInv`) checked with Apalache; see [Inductive invariants](#inductive-invariants-apalache). | @@ -389,7 +389,7 @@ and `--negative-controls` removes one guard at a time and requires induction to fail: for `Supervision` the retry limit, ordered start, join wait, restart table, cleanup, unpublish-on-stop, admission gate on `Attempt` and the reverse-shutdown wait; for `Cohorts` and `NestedSupervision` the existing -`Fault` mutations (`Selection`, `Cleanup`, `Retirement`, `Closed`, `InPlaceRetry`, `FirstStartRetry`, `Join`); +`Fault` mutations (`Selection`, `Cleanup`, `Retirement`, `Closed`, `InPlaceRetry`, `FirstStartRetry`, `PerChildCharge`, `Join`); for `Coordinator` see below. The type annotations are `\* @type:` comments, ignored by TLC. Apalache also @@ -593,6 +593,7 @@ exactly the property it targets: | `Supervision` (`MCSupervision`, a rewrite of `Supervision.tla`: no `Fault` constant) | `FirstStartRetry` (v0.7 retried a failed first start) | `RestartFollowsPolicy` | | `Supervision` (`MCSupervision`, rewrite) | `NoDeferral` (v0.7 decided on a ready child's exit during the initial start) | `RestartFollowsPolicy` | | `Cohorts` (`MCCohortsAll`) | `FirstStartRetry` | `FirstStartOnce` (in `Safety`) | +| `Cohorts` (`MCCohortsAll`) | `PerChildCharge` (v0.7 charged the restart budget per rebuilt child, not per round) | `RoundCharged` | | `DeferredReplies` | `NoAutoReply` | `NoOrphan` | | `DeferredReplies` | `NoAtomicExpiry` (abstract double discharge; the real race needs Loom) | `AtMostOneDischarge` | | `DeferredReplies` | `LateSuccess` | `NoSuccessAfterDeadline` | @@ -607,6 +608,8 @@ exactly the property it targets: | `DeferredReplies` | `LeakDownstream` | `DownstreamOwned` | | `DeferredReplies` | `EarlyClose` (closes the registry at the cleanup deadline) | `NoEarlyClose` | | `DeferredReplies` | `EarlyStopped` (rank 11 before 5e: a token dropped while exiting or unwound answers `Stopped` at once) | `StoppedAtDeath` | +| `DeferredReplies` | `NoReplyFence` (a stored token's reply is admitted after the death point) | `NoReplyAfterDeath` | +| `DeferredReplies` | `CallbackBeforeDischarge` (an answer's wake-up runs before the last answer of the death batch) | `WokenAfterDischarge` | | `DeferredJobs` | `NoInstanceCheck` | `NoLateResult` | | `DeferredJobs` | `NoCancelFence` | `NoStaleAfterCancel` | | `DeferredJobs` | `DoubleTerminal` | `AtMostOneTerminal` | @@ -618,17 +621,19 @@ exactly the property it targets: | `DeferredJobs` (`MCDeferredJobsEmits`) | `NoEmitCredit` | `EmitCreditsBounded` | | `DeferredJobs` (`MCDeferredJobsHeld`) | `EarlyCredit` (a held result returns its credit when the work ends) | `HeldCounted` | | `DeferredJobs` (`MCDeferredJobsHeld`) | `HeldDeadlineLost` (the deadline cannot take a held result) | `DeadlineArmedWhileHeld` | +| `DeferredJobs` (`MCDeferredJobsHeld`) | `ExitKeepsHeld` (owner exit cancels running work only, not held results or scheduled emits) | `ExitCancelsJobs` | +| `DeferredJobs` | `NoAdmissionFence` (a job admitted after the exit cancelled the job set) | `ExitCancelsJobs` | Measured 2026-11-10 (TLC 2.19, fede; the two `MCDeferredReplies` rows on 2026-11-11, after the exit-pending change): | Configuration | Distinct states | Generated | Depth | Time | |---|---|---|---|---| -| `MCDeferredReplies` (fast: 2 requests, `BoundA` 2, `BoundB` 1, clock 2; safety and liveness) | 4,035,345 | 21,287,403 | 33 | 2 min 19 s, 4 workers | +| `MCDeferredReplies` (fast: 2 requests, `BoundA` 2, `BoundB` 1, clock 2; safety and liveness; death split into D and the answer batch, 2026-11-11) | 4,864,810 | 25,029,343 | 34 | 4 min 22 s, 4 workers (fede shared) | | `MCDeferredRepliesDeep` (clock 3; safety) | 11,016,668 | 58,096,711 | 34 | 55 s, 4 workers | | `MCDeferredJobs` (fast: 3 jobs, 2 keyed, 2 permits, 2 instances, no emits; safety and liveness) | 229,070 | 487,283 | 17 | 5 s, 8 workers | | `MCDeferredJobsEmits` (fast: 2 keyed jobs, 1 instance, 2 emits each, 1 credit; safety and liveness) | 24,842 | 50,944 | 18 | 1 s, 2 workers | -| `MCDeferredJobsHeld` (fast: 3 unkeyed jobs, 2 permits, 1 instance, 1 emit or scheduled emit each, 1 credit; safety and liveness) | 1,946,047 | 6,230,609 | 22 | 2 min, 2 workers | +| `MCDeferredJobsHeld` (fast: 3 unkeyed jobs, 2 permits, 1 instance, 1 emit or scheduled emit each, 1 credit; safety and liveness) | 1,946,047 | 6,230,609 | 22 | 1 min 30 s, 4 workers (with `ExitCancelsJobs`, 2026-11-11) | | `MCDeferredJobsDeep` (3 jobs, 2 instances, 1 emit or scheduled emit each; safety) | 7,980,942 | 19,676,218 | 23 | 82 s, 2 workers | All 24 named controls find their counterexample, each in a few seconds. diff --git a/verification/tla/coverage-model.json.gz b/verification/tla/coverage-model.json.gz index 3da62d02fecb7efe7662ccaa9efd0369d7d05cd4..0c244de0be59fab670bc3e2faac1b6caf613f903 100644 GIT binary patch delta 31182 zcmZ5oWmq1`(#0*fy95dD?(XjH7J>%XfdmQe?(XhRaCditdvFWzv3qa6&E8+U&rHuu zb#?XWQ>VN576}#{2^K{RtbI}m4(iS)0K_q(+i^YmmZ`}&#OBCxO&=nIkk&?yJKMR ztLM0}^L1sT^Lg#!e1kXcWhO29Rk2kKa=aC$* z?hjj+k4G;^z!OL3^Am8N+S$r*HFguZz|n{l;PbT7`Si50_PT3u&d1r&{Mz&U`trij z`S^PFdcD6r0f?}?zLXjO&m#+sjRTqM8$LIWYd1ljtAOWA$@ zTPI+sI9nIc^tcE*w>{T?eK=&eOW=RrCyK0O*l4>3E-w`980a_J?|a7D)v-7l8D4I; zZXcg#h}H@1Uak{I27uSXky6i0{^z^1$H(Wph4apqd$QRPftR0DKNA73C-~>nueck8 z=U?3}A1)Y(&IHD{4LW!_JfFIr=U$tbR*z$SbI04Q=Mf4RKerSF^gBGS@=D%Ed?@pr^b3a#_s$a=D;X%n`W}gb6$;=>$I>{$N0KeF)rQR2uf} z4pQndRH@Bx4?=7TIWS3%xlk$Q+IVfYV>4s()-S5a@0p&{(DuU=qBqi-?Pb|hJRJan z2KeiKffYg?S2BvJUZ{78{Um4Acz?*Tmy{M%@?C7R8ho3k6bXD~cMh0!KF%{#Y|KDD z_BqN1_jqPhO{7nYFY`_kDN)dVkM+2)_yC=3f&&EEZ%4aF`RcKr zo*#`_7@=Fvp>q|#lWcgkE5BdL7{3IFM1c;<6(%C^`ScXLl~_;2iU9tU$(`ZJOBf1e zHk_0pjq-yzqfj|tDUSa`mRK8P+3vA$G+3ZzFtaRjG59+oQ9XG~bI9ZGhzgj>o5DvR zA>^RWjJ_ToXf8DRXhp$M^U(STxCWWPb$=riNwTIOa`4@as>Pfl62JS$^ho(`<%=jmb(FT3I&cly=Ait&=fRfyvJ%lE zzbc%wON1VyoH)<`q};&o1?DoI+r`USvO}Dyo_z7K4iNf)_s!Pr-r3DSiDsVBOipNu zx=BGx|70`-SCVB-nGI-&y@OsDzBr_4lvak*m)I}K;4`LY?n(($`GdnkDJx@Di%MGt z2kPnP<QW$3*+MlslNt8!Q1g?=rAaEDP_BR>n3DSrTRi$h*!}bsqyI~PMpLJ zYIV_$Pg8z}4MIEHv)A(BBJn=;8!IiJ3KK9qaeUC3`-Pj2;gVcQP^?b44%ST#oX5T6 z@8$?IsY5-XM8R6<8#GcCga9AOwAG+= zY;dS{*P|JpxE?k9nj~U~`&weF_ATz~M?sN7lz~7E%njX*cZ%xS`}tpRS!wk{8imV^ z^?r0u<}R~9`NLCsVY9)rv$fVVwh0?)QWtShrUH?QrHV%Eo&A3ViN@{l!$6&>BJlGU zsd$Aj)>R~KDGV%Hf+!^k1o2*?OX**Cl_w5|z;2I3O; zE%exUhmo52OYyFy`u9C_D{+#2w9o89`>0M7LVGCInBVu1x8UEW36Nwlz)D^yj5JFOwXqr_NRDWTuVBC9oUx0%%oSVYZDOk5XGpOCK90dXy(Em%fV@39U`o?;{w6_lQz~ z!x4!ie3ctZxb0VSQ7kY`BM5V&4!AsksQU?HK23^!5yqwZq~zjV`^CAZ|G^1P7lMKL z=+8*fP%lNXS7$8JVQtI+-M&0tFC}y=BtJHE$0Z~Mu!xq)ez2*dpiuX^BOp8xiD0QDob|W`N)(?m6)}2cg+v@s^PyG!UA|Xj<8Rt&o!)u3WvY8SRdBRNT z4A^l>U`#a}BPS{2XYqC=~T)A5RkUCfj1a(XqdDyi7t<(;!A*O|;lXYZ9B7@Q3BG+f z(K>r&Tx-ilsRfM1g+Dqfmi zZwfCv1Jg@PxypXQT7!f$N)G*dF_7Sj$#eAs6UDIO6+oHP4bF>RaE!+lncIB-bJ6c2 ze$L8Y;T#O|oAQ{wzri}BhkhC1)^@8PY~Ce*rYO$IuMy~2DhROOGmvPh95@4$3!%5d zZGXq=Cl14|YLQ4M4|1a@Q6ZGaxN8hW-KzCDoG^e23&(Q$hJxcRzfS~1R0001>gkL2 z>U{$}Yb5yUeZSz>41~3N%x+3K%S_R6Fms1Mahgxt1vO0a_fgU!WxA8>8UO*e8 zIP%eU-O$$W6Tm|h``SX*?|}qTNUd?OfUb+}D#w|3sdJn}f8C|V=~41_-d@ov_8?QR z^8)|Y?Z^_|&Cvl`lnu_Sia-w*Nq|kHt+#rwj-fM0W510TfIuqF(a|R;kKxFOV0pm$ zW6|EmVLq77){7>Dj?$<@%+@QnX;Nkf|C-94e6YP6Zv!R|Ufay@ZO!5MqUP@pEAu`M z84USKrqx*U^9xge}-*Yduf1LBYQ?YFxdZm0GJNFp2^ zN{htiZreKWJE~03GrocHq!9=Um;>o6MT&CVBTMRuqZs*Lr15l_p_NJ{b7l6F@hPNn z6z>@UD{N~k7c)C|LZ%i|dOCeL=nu#zG0o^f!OBqR{9(9IsVRC(G=;nuZm}K~$gt~`z)lOs zLFAw_$}QTrn8&hZmOO~c<$ZiF^_2=&e$w6};e&;{RRV=MCta-+j_lA6|FA+6Zk$hf z#^Iu)c}5hlxm4H+U2cz%F_n}zDeIPv)XbF%p*1Y0<0K<7BW699KucSmrEDi{(U~TO({qg>*RXivul&Afn&WOYJHj>6But44(}02I^WA|et;0F*Z(_yG z`#b!i-*B(fPWXHO^cmf0_)6aqf3I^f7txX0l=nB}gl0)z6T~^6xLi>WqW6NT7a@n( z9Y`pEAOgW=wv|1K+ez2`uoYWO;T;K>Xt6nG9>ci^FM~SL$%Xef2xWIWbBAC2qJMfG zytVcT8H;@HhPoZe!A}!IS3atw_4-Q;)m@1$*9ketbU=PE(MXM&YszXEW&(wesM)~U z{?So=&MY_Mx)~H!yf@(Ja06^zZiQ6s&KVplTlfUM_4&${@CW~$Cu+idDnz!vpi=#DsO4@` zr5d|gk?Z?sJwc=xLp;QyNl&IyL$$aj<8P0X#ZW(vUHQ3TW%@+Gw8W~s42<-w)+4j- zFtDwNX}t_cdVeqXz;9A`=^bG97Az^U)o+bx9i!d)@k^d53$ZSq;J9)^G-Mg>-+s(a zf)bWMVfhUphE5YXW{J$L&A-WIzy#0>^5buI znJpnsOpDr`2f!eahj^mJA$+k=L;l81C6>*a|IDl*8rdtTN#T~lJs;T{HUcF@=1CU# zBj=l{{1804bAVYQNz7;N@-k6%99dRFO}^V&TGqxulgXm7wRmJmer6de79*1}JC^WI^x`7mIN z+Cb**8fbi&kxPTu$$p(`dURjp^UfP?#{w4vKA-=M+kCew=x?}vxrhY#BaoU$KIi=} zaF-K4X<_Loc6+_!y9Ia1PW;U#^CO=@BysA7yMCGo{28=~F2`xK-Y3v+IJiA({jfOU z-b}_5O7ujM#0-|0`(^>mqSAZ%iqPnuB3~G&goVgw^u9gka6aG`+~{l4p@{%hlu94I zEI2NAI48TV4!*1H@qo{KA7)&x4$s0Ha;L7Y4=I*P)(-dx{};J`12?JI?V^Ha8#=h+7T4Y*0{M8OrPOMKQKBy3V&cQJ?f^Qw_Zk-*Y2j0VKfRZpqVly zm*P_D{x~g{Xx*ZUmgh$aD@tXUQnlqYM)8SX?J77$az-=;N`*BoYwMa;Vf=EIRt^}o z*EWx3PKxP+#Zee=VxE`m!wpW7>XAhiS7FTXy|{iyW}tM$2Ao(Znd)c0bp|uaY-qQ} z=y_2=k+r}8Q+E&k#za79D#=Hy3g7I_J>5SBqo!=TOm3@o2 zyPPr+$dq9qdeTZv@r)pRGAa@RCSZx4U>%Yu&HkW1dbDp%65gnfemvi8kKzAQAL-t| zLW)|SiD3KfhoT6yTM@1<1R)P&M~!y)&V`8pSB@$R{`A}6reTyq-~Os50x53esO7v^ zMCzgz?)E1~k%j^N_TabnHr`jLR~{Gww?Hf@)Ye=Q@u*t67CIvBK=v0HA;aM2{B#Gv z>GE7AK?{@3v+AR^q27!lvpOgR=y$R*H_Q@hEY9jZ`q+wt;^MMvYW8SLv9~O(#3pu! z+KrQj3eTha2YmQ=nVOX9b+3lHN3muS=UC>whfY z_s`cF!AF*`tW8)I)C0XPV`rMm#%fb%V@Oz#j7qQeTIJ-?3xBCeO+e>$G*PO{B0AYm zk0ThR+%r?CAqLd#@#mWk7|Vn>E?2(_czM*FMJ{LzSdUOar`-~QJ=74mfoG~&C%GwW zBppn?(=ScI8u>&v$ecjjw{SmtNI4ftC%I5RFgPW36v{JxoZzDUU!1r)PTvGK4E|LH`W&$~nNUL_lTLxtamQKx$-qRwO9Wx*mb#P)C=Oi`|;tvW&+WOJkYf1Dd=F^#>agSes z>y&CipfW?VW3oc*p|m>ZF>N<&2sqZ4UKLcB#=58p${l|P55bt>M>nfz48&t&_T<4X z7?r!I+GKG}ZZzU_9Z4r8lFKECzE8%(MW!)Fl1h|glP?~Hw3I+|Nu;%%EO8KOo+j}? z%S$|Q*Ke2{JS)q@PY7MyPbOQY+U@G*Nt2?$t95YS)ZJ5wlGEu74_e)nDP1CY;BOJ>o_ADlU{>7xvFui}q&2CbN`}ELb-k-3|*Jmr)vFR9+3CY0L!|^qT?WEJI#c)%?%M3F8oo$$6Xy zh8M@U!Yops3rHaTHUL1EcC>nJU1?na#v2+F#Xhhx_rN=gPpaTxCs{I=2ax8rn|&LbQ+o`ML!vwKBXI|=l2l$nOkqU4`NIasgoBbO^PROdkiHMa%eLpFtrc9x zoMhP@Wu46sT5(*kBq}i(jA)C;2hJni+vm*m85;vx$lQJ1kK*aDGzXGzoNq3HyBjRB zI=8!XvYhfR5AT?X7FBpJ!^4iR|DPz6`zSEPcSzw0KLP>WY*9Diqs>0TcOmZ?5&FKS zxp2dJ0zh!H8a=G5LDe#2oMH)ldIdV99uBKpjovFJci+#^a#qa6`mE`e?j~wuG@BZ;} zjLH6`ZFBZiXezlUQcOhSZ&`Ci=f{crR6wij*KWrkqIVMX1|9BLP_B-y&hc|OLPH-d zb8Bh`Gls>_*7ZMxU+2xvj?H{#!6F)_TlH)ZHgNScfYV4Hui253J+5Kc=4ME3#%s7> zh&{u_lU^AqRah9CrH>4e*CWKtX#Q1fe*$JtoKyMO%s^i1W2^P~$ZBmOJu!T>+#`+Q zm|3i(k=7k|q2@$N*TRg{wN%yat%Ys+kvCPW3Z4|GZfrUpHXbd9(|IIOs`4$Es_Ly1 zMwp*I{W0ZoM9Ay7Fob{X28G$jhbGSSYkkQTh?`qc&Jf zuPTf#6_Bg^E+Jf~Cvf2EX3{@^soQQ(bmsn0_V z-FSr5(6W-J7V7u$^Hb}3;l+4---eVyw}0oZ!d)%jepw67l?4-Dp#LLDFaq6_>m54S zKxEIj9D_BzN~Kq(0JJJIMX^SEz5y03sc&)OQajeHjw4W$EbmUYkeD1(37o&r7FoEtY{{U?<6s0kk|B*Ao!NKbJ|WKdQzRPM4aJ zQuExHr2Fv3`=naWng=NGfPC}NIwiQ54QsR|#rhLI8~7!IbKe*xys~LPdHgr!J(&M6 zN;&1DCa+&N-RL^opS*v>rRrElIjU17%}b5+0L5St4#R|rZMl-!_R- zKrZSl9_@a8iFEtWt=zRPX^Dc3)VM{pa3FFg_)LoRN}5 zx}BXvmJ_Rgz&7~e3##PtF>d3k&k&qOb@nMXbjD4YlPA=_tR=jDodU++fd??B9+Iy{ z2TykE?V6qFat`zk>xpMqTjIT_=L%;)@g+>duWWRYMO*~lRPczU_Exjh`o#~o8u8NVrjBM=tG!Vt;3iLH7< zQ{7f^fFTa;CWaGb zrkZ_WVIui5dTAV{MO2uCzH0)X)(x#GgjLhTBra=5_16XxPElCowCe3Hw#lCGqHke; z8oMSQDFwILy?xum#P%hGj-I`>qcd(f{rwWK=Q`kOM`R28a0KXfcxt54g$`#_ATv&m ze5~m9c`-)}_vf+$M-E){=afbxhs=es5vZIM8k!6$gM{D#gVKJ)CP#x!da}}$@kekMs*?z@8S1OL`J!vqE{(}k_ubQ(sRAG;vA3~RwvxF##rCV z>FWJ>R_efXLp!W~wVjNeSWHr@<^T!LzFEvvI5RA58vk9CO(dp#Pr{msewM(rwC`Nf zBB0h9BdrhE?V_blEl+v+*c1T8WfA}7V|pELnpJ-_uO>k}-4PYHdU+KK(t7G<52Pd1h*MW;|u!w1HvRy(O{=|J6~ zMQe4eiIIaXnwo>b6v!lNi};(pA3&|^@EJ$7TLO&{vf9K0c8>1XiJaBzTl5Lbdxk-|K(jzI?F&7a*o@E{TW; zma!{?9}La_ZEKf9A{az?ArEzf%SCVjfezHQ%-Ae~T++1|f+HsV7EnJb@cQ@FR8C6u z%CsS9qoIk<71F;2`BEYOclS(gksn=n-hh?aU$4se8Is`u*Ej9z(LXIN;hz8wdv)wO z5L()LS7(9WZM>v$K=y-t9jZJSHunez%kW3t2~Or_n?)Ll!)AC!HS-j5mPn+VLj#wH z@s?O2Z@}$PpT)V;unHa>^!vTzQlv%>*Gd5&o@Ew=>Idi|Sy8Y!ac$vi-LBONV1u4^ z6Wbc;Wi{}iU&H=`c%g{GiIdnzo^HO)^k0rmhee5QcX~Q{E~ej(j=0b31G<7Z1{M2+g|%m2+6Ou*KGliL|$Lhs(ui_kPuMzfw*O0uS?q-6}XT>OmC!zb6JB??NL>C1T z8G`OV%k`V9rcknXK};~)uONnZHTbUKTS40&ezp5}nCI;acR$}=9$ofCwp9{t8($>; zBs;1HC1@%XPdh2=Meff}(EIiGP=1R>qE?|yY-K8nW9NQfhi^FQrkE^@+WgB{G1-J8 zizBm>rD!z#T6Zv3gq|Nb8R0np>G2=|fBcnS*iGgIqNo0XRCYH1ZJ(5;`i{rH@5U}b zzT*XzXx5iR|7~R1-qXE}EEfgecXzdnzedHV`1hz-Fusin9zh6EI`uozyYYM0`l-*v zMjh?_17s7#)vGL~sJrN&zNq?Bq2aYEph(It$xUPdM!s#HgxQK&;kNFOiJ&^mNl30e zNR(4w5seOdoIJ+K`QxN-l1w(^E|$j3cLpLM%c36e)30ehp%=*Nc!clF8sQa1Cvn?K zg0fObbZSu=@=e!32Hj`zeczx}73?ZX33wb6hJvoR5~l)ck@1MXZ)Z2LPi0HAHOa(u zYH@M^#4Th`sR65CQ|Xace&Gq=C@e2y!e7P&e0j?Ss)@NH`l%k0G1kAS+wc2-luwV{ zH5iRznf;_1idtvr9O!}2Sm&!ykLei5Exhk@#eOudSm=iAW@0*EQS^z@u#Yc>N5nf2 zt38TWga8jD#8uqMCy={Kyln^GIRHGTlx4OU_|FK|FtA$`iSS;mUk{$aJKKW7P97y* z;2c4OC5fG%j4LA177=P^WJ6IzJ+}L#WM)T9f^zVP=xrHa?BX2F`Xa8Xkjgg){Mf=df|JC4$TFT#+{aYh2^SJe%a-r@hJ!?X3xCt~1zYtR$I1|_cmqdu6u>!x zA#iLZ$!E)A4T{|}G!8=Li7GP6WBGO+7_|~9n#Oz-&c1q>JEAw{iXkC zJPpfDP8M5@RMBFbh0{L0QP_Qt7ho6sVi&gkYt-GBysha&jK%l_!nc>f46X?}}kxzP1@CCDe0sU1=-YY5dE#qC%%FW8-5~|@>@%Jqk<4848 z7_^dw8mY2N6wFk z6fTt2;K8`!++QW_U=9(8TUgFrcAq2KHo`f$FT}Abcz)X6o${Q!EVK&^v)V#YAWAqM`lUK23Kx<8N)=LP^LZ{_SYvHXDdGPj1h&xR#8ZbB{5g~2fVOWgB4^C(c?Z7e-0M*V{b9L(HsSEZ9kguuOg&o0v#Zs zXN`AP3U|}8-;kWu*K97PM|&iiJYN-v2p*0MD(!q)-0n`2Q;am;5!+?_Iy6-1Q0nQ$ zYUdUYbIE+ebH!Et&IoIQdUO(#79XHZIK~Ttae!Y)xG@9CzT&8oHZOv~=9{Dw2tsFe zYwfjfwxI$?04eaDnx@zzpMZ+|>}C%t51NiEm72Y;wvK6o?2`$l_Ic)J)@%Bn2Qrji z`nFO&^;q6UGdsG^eRFmP&2msifgV*WoYfDu?b?OsX;Ck*;3J;^gpR3R2;pT}OKVV2 zDSAc;p=zGyu2nejuhoVTISvz>bPEJc_z@k3llFDKLiK=l&s4YUmzU3{(yIvJBwC}8 z^PZ!D>$8I`?-;`V)u$33Uld<1%ra;%zXK)4nkeqG1@M0M|8$2EIw{9cVPP25_^mVT zH@;o+D=5c8GUEjyau>8VqY|IHs6ytSnb7;`rg~dCw=83k1?O+(2{C3RYtzvWpsH5`TOGY^ze>+NU%65 z#Eo5^KELesvx~lihz*!j*9RAXLEQD1`l8inwx=g~2<#k=<4*rNDfY=R9g0$0xE`OW z(GuU??KSfoLxQhOsL0k7p%e*n8@@$sxK#f(`T^h>cfP%df+Y;RS?3w^A}4KI{T@0c zS~_pq(e91)Z@bx}mWK3NvBNu%Cr$27DStTu+njXx!qKhxJ<% z>w@FJopZ`1w5q~XvV3bPEKnDUcZw&vCJ?9AO3F0U8RFP<=hZWekx{7$y&!QFt<+QN z?PLk;_Whny=ndr-I^qfloLc%+9PGiMk4vihQh*C5G-Zm$vbVe`sQ9`*{-K@Qdlj9g znV-3(wg$5u2?&?4$8$IBC8p3$keZDFFwKwBy->X)B|NNolW) z4E58R>QY$JKz@3GZn7ly@fsNkwN&tlM~Gj20sH#Mz((ZD&OElmOI$zg)LqRXYW;B` zW)x5g)RoFGFw}gmKs!bgO;FV!-pP|GD2_g9T_)pZpK%tsbd5WKU7V4NsaPCD`~inN zdoez~H4M_#{43xbGo$M=di4yoU^8$A0{8C3MS~HNBaMC1*f6`*7X#+wEVky(_{Im? z#Ty;a7)`uZ@7}HwytC6wu{U1)WuauR?-Ia;W$spHv#a{r2qKC&6kS&PN#c#C=N&Jb z>`-EN?w;wwK`}d<|2QRRp8cP_i-(j|8=3DrGn-!FCspahk{#x*^Tw>->gtgvH{?Rc zoYtcLKhDq^Rz@--7F-)GXUe+DK6$|DdlOoknip%Ia5btrXeJk#zIl+ zha0#em>!q;wWf`?OUX{ph5n>>V`cB4cN%KkjUlDG;l_I8p!l79O|XVxjmzZHFpV|9 zSR|i#pRvw`O8(Bp0%_B8U4#{f?1Y}E>#i;Fz#}h*#|Jb*dLFNMXMV$ibMcvm(P%ZC zPtY>=VBC|(uQGy?TABp=u!nXZZQ#4qfnB|3j`Sw{ZAh~rt41oFXW z5nqcps6$`I3+__fz&I6An5l@bg!&qfzmdUQVOzjLIMv3{LODBaB@|vP(JF4zM@>o7pQX4T8|$dITO= zf=H~;BJa25VW=Z+27@YubET`Rzb}QSNMX!-(gCSOMLsu&{m*;dKnh#&0sMEo{Sk1l z!i|d{;~a$OEg>q1o0hJiU?hKSYs7|pfwJ#-I}l|7&Z+ef$hl8mUgZK}QM!yS_4~b# z-S&4csiCC{1Fjnckqql#E$!_@FCPkZVD`!ysDBPw94y}FsEvf6+ADq~(EUB_loE6W zblYC8ZfI5-i+qrnK0fPztyMr1gH&>v zFk$gThnzyX?>(t_YukF|RN%_-LUeg{tZxVuyvwRWZbESbFXIe(xV^Iab>QLSQdkud|C7Q}zsu z41JQsZPb@Hrx=Et(UeF0lTI0Rp)-8Y$ZFF+-ZizNCYvm)PQ(A#!YIRDm8R=A-LkGJ zUmz|$r#&)Mom_zz6T8nr2@ueKs08`wtxsHA>0nFTvwOVwX4Y(_i3pxq`azjVp?Zou z=WXCOx>M~KuHhIOodNwgh2#3SKbwhF44syv3w#Sn@5hl>?YV6_Cx`RC`2WuXug#Kv zQpb(2%`kOJaPtq%l2-YL{w8+c7N;x7lXGc-<}Utm+}Qc^etyK-w+71tu;qVlQ?p^M z4#4_4$l0B3!-&*>Tgwb3{`cw=!?v-ZBRqY(d3P5TtXw_*qW`8ueED(v=xlmiln(=5 z;008I?A|P&p9_)tnWsOu&&n3n8m=$^6QO%)fw{ zB(>-;-J?T@ZX++kPPxbPrd6ZWd+luYM7~|U9(VrG39ti|Jg!~a(7_(GHw5g$rATc~ z&Y9kvXL))a@ISs--AGM*bKdI4$VI;3tt)kK02J{58tD<_KugiQtS!Dj{)Tg1CVrc^@9c~L0XA9v5*sw^Du;=9H6Hyo#8rmg z2OpQ?Yf7^yDzG!zXAJR@y1UsGw{y=iQj4!g5VZKV*^xD?_^(+{rI!FTMYP1Q3Dv3ySGm-J&LhD8jlO>ykzEiRpQ-$X4g3y$O4VH3kI#OqzNX8El>DoGX_>CqhMq~=-M1*n*x&edYzt9QG^ca=Pf3`vWA*rO09)`=%*m41vE_}jr-r>N zk5`9K?Di2B)xFh9{U<60WcL?P#c#NM>5kdIGju2zQV2J?NctyVK(hvhL|VB3A3oP1 zlHPQ*t7a~T$U3PDHOEV@u2Wmu6k^D-1TNl;Pp4I@K+o!=$k4KhSV^Elbzv5v>T>2> zV-|&_OOKp^Vbs+5v8;y{RN3&YFnb(@&C~~)>ns+KThAdjSV5LAFO+$GaGqfqP9n zRqeeK*=|w6JL;Q3AieHzCe};9=(C@l0Je9we&P&Q3OL_md`fY^_Nt z$vH2bia2gY9r?Q85(32OBy`)MGaN{ABU)?VauxdEp@wpJxg7qH+jN2(xETk4Tqi>G zvp|C$#B^Q$B*&#knYhqknZ1GCp=eu5=LbUEn&2d&cehN1S(g2zG4~UKo?~!tyB@TS zJDZ(@J#L*Plm$_(_d4&Ez`>kRC_jBu4)t=7iOO3p~h_ABjE(RhUpK ztDetLnXEL;UgGChUSVF^W?PN;`Eso{lTEgb9XyXj6bsJu(-*`h2`~cD>n9*l?#ESl zRR}VEDL#1fi_5*)5g%-VITT&Mg?O$(?P{n9axQO;8Vd5`1^F9=4`660kE)A(EaUH8fRq|IgcCE0hH>Fd1? zkJ|#;bc&Wg6C3t>f~LQ7jSIilv8{S*F~;-1EzIZX3O4lFKhfbe#3SeoW1gCZs!!XH zK4VVX5rqTu*~{YBPnGBijqh$~s%B+F3Z95}T(S{l>w1vAMF1lI|LFlJ!xD7#ourc6 z<5q8pK&K)c@yPwkf;1cLSm_j0q705sD(CD)9^D1hur0A=5dUU=rxIzRv5n#T=Wov& zMK%sAiPiPNA_NM{sCHQ=P+tI-191JeOc-0Ll`hdo6plrO)y+pXlTT_Hyz#%3e8#~#T}ik5$S8-v-ct0S&V9bgF`W}D`T)vx3eZZmjDjRvXN-3 zTUBhGxA8gjt%>dO-%b8#!BlI@^n6>R^=YZ9+VPllflVu&^9Q$HN zU0VwJ_RM*~ZT|5HD<88)Ry#xccYCh#=-CeZ(XIUeTi$SoU#)`G-p$KD`I*MCYszaD%1RkZ81 zTG{@l5NU%FHdY8e3mXT>9wOn=-qu8^I?<09#t?ZsvL zyd11(M&VW=XJ(n(m6SZOoj7X|QusI<$PPD}bXZ&)&AviCGrPNc(65SPTufn+%W-7{ zwK0tFU}L_Q6d#1aSv&%h^6|5e2*&5^K)YS2N7i~Sb1`y2jV;ZpR!Uhd^{cv*GRATj zwOBtpd`vPkmqBm=u1)k+{Cp8g>=mS2#TX`+fnj8_1DAns;iL82rQ5LN3~s!SwdKrm zdT!$yhQ$=0Nbw+gWQ)-20#(%5rG!2?j>5?cEhv@@O^XY}40bO?MkN!H^Ujhr*2<}l zhKvNsaLxhdsa{R2(B=1DoV9*x%k)I!*32r@L=}^&xpJ3V!?ze7brkTGS1W5!XxB<{ZPVq~^+ z+FLntXKOPqLE-(y+3na3pck%f2ansJ6G2$VVYBnFRS=PEVUZE`7dP$9HC5=!u#KrR z^lX8Go4diGEIlUYeh@B>7Or0mQ(-DUW?R4bo%mx~XF}5}FIZ%iRU{tlyvr8{;O2~3 z7TgB@pXyb&YnqB+_yoXz@-}_{XxOWqPgXM*#bFDZ93A)%GDDuv#QE1n!BTv} z!{B@{%?=lCm<>T?v;imoGdEITu~6^%P*fM8@$&>cKwWxEnMZlJts?Qi0wwU5LJ7H% z7>=xbURYGNzVLHLq)#30@bg=%x^?J3WK5LI;+t~rPOCl~m_;tr@(VHnkBQ z=hZoT?M?w3?U-dgQB&uqt98(=R6mcnBEEd_5C-&AEud!lUsSfrdv%BW#tUNb=pH~# z_KNQn%vOGrVG`yK+5S07Y>M+sv9zL-%~CB0)Z<0WiWC;p+I%*PcCX(Yii5d~ynD1| zuW1ba%RgBa2Hxr@OtewCovpcl0xI}}v2*Z{@m=WLqPUVHcj#R6pBaA5&M;OfxT-cV z@N8UPr3|;ARYu{m9HEJ%{h3|OI>h4RJN_C?f~*>k4XrZFH-C79&h`E&%GvZD-mez_XVzze$WbyXgat6zD;dF525geXI({?w5Fx}4t)U=m3T>oR!FNZolndGb*Ox_g;u zk|W1ibO0;RNB_!LaPXfh`+Y12`jNP>OzE+QFVLTHEVq=dS+8)#|1JfS4a{yaT&wCa z3IfOaKF|a2i_6{hPbKAr&7$-7EL$@Uy@jrlUn0QOA`tTq#hLAg|_SnIhYBr*VA$zrRZiSgRqn;MklpE_Fc;<>7~ z8g%*cjySMlH7IR2f8|o9u>VvXDVMr`Z!zkbrG}>eGEl7c=K(mH(&Jex{kJ+=+WHjm z(n~d~dP*$`Ar%^5T`!>wZE{U9Kr*R{Oc<5~gbA>{8xgutvtpxXD>o3k2#&%z;$20? z3hXFS!1$-&V0xXg9ygu}=R@(_zA{;Th~T?OO)EBkl!+vfdZM?7=`=a4Q?&cu{@(y^ zeMxp8;!j1kCKz59Loc&>$?^iouPu-&rBqea1_kO@G0y0m^}uj+C7Ru;0xQp% zi5hn{2MT~Y;O>auCK^x9K}Slb;6YD$CjT=%34_GMBl!wMauAiAodB;p5{l0>JT|x@ zy+Xi7q6}J&B;`V9IgWvdY9(-)@J4Tn->L7wDl3?#zmVuxL;vik0PW`ikfj@JFkhEG&s5^4{{=;D4rouL+3%pL5*f?fyi_SxCvuVUNz~ywA z`8$L2II8v%9{K-P;OQiFpcMc*&RS5nKvJ*rY)*Dlz9Vkq`<0&B7NHU|YrI754r`Ke z%#=EO!O>@>oaD;l*tI671YQe;ry0@4~ylJPka{02T&SQ51uJF!jnGMVsHg8$2B=?=Ehb2ia4^~^2m{0oD7xXo!pJFVAr_a*^F8 z1(s)ax5`QASf(%k5@gOGHiETKmp@TbejmOY{=dq;Dk_g=X%`Rf?(XjH7D#|V@Zc`N-3E6jc<>;>-QC^Y-CY7N zddRnb+55jcH?w;6tm&z$dFrWNUDMTgxmVY7p3vm4rwjfv-zZi8V@tPfDZ6CZl#HH z?ljcWWfAg3<C}c zDaeNd{z4?c{6zh8(?3CkMx4dvBYt(=hl%xyG~a({y#RIWf$96$0WcR>cI)+bpNcj_ z?Y7I0SO!9w89go}*0!pzZ^ge^{{@lR`HHmAf0jDRadnV5?MEBCa95sGG5)pGP*3`u zNYJ@&38acqKYjDQ#s(PK!cB&+d=W1uN+b~GnFi51GYQ$h`Z5$Q#bILf9 zT5orWd;*JSkd%9nCE7_(M`*RdfD|pnD-6+uIYN!?%X~9*%Ohy)gtCEVeB@;kgJ&X8MI!&QRtP?tJ$@M-ApPCcJP>+mS$o)S+kutJ;*1F~U#ZmCMkh{Q*e;ZpT zQ1C}L0_LAd&VPYL&-2swcI?lV2K9doaW zVZH*Mq(@KHd1-!RrS;d8t0fN6X!pgU{~ovuMrVgg`R|JZrT-_E8H@t6VQKOQlMiy8iT>C%XU*8e_i}r3=^g_6`ka72=H>2E;bh>AV z@9|mm3W5JM#unsE#p}J$Y5ECCg{_Uq^WU0V8^GO31psO&*%8s%@On94V|l+mPUy9@ zw=^|1TjRoME0WUjub0Cs8r@Q@j?lU3V-m(0vU*@52F0;w4B#xFU$<^;H9@!&8mtU7alEa^lCa zf$*C}1Krj(p>^Q-`WQ#cCj|UM_=6tZX2{OU$x{v)$~K4o0NMw{9Q$vr-kxuo7jGNe zF8FShgKaRD&W4+JV|(7NABqReI-@C&am9B!=XNj~W;z3;&bgFIR3JqZ9CuzXq>hLf zef0MvHg0?_Os=1mViyq7L}6Cb%?18eC8b1u{M{e!$G zC|MNKaR1zm<_V4#O*S|-0K$8cR7`1upu0ys?0FHq6pB28t5GN2(}jhv&M>ey6~3^| z1pha(5swYz|omW(Ws8v z)vUnEpway+0a3bn94EG|3wHxI*g1nHyG?=*1uy)+TR264&rNg!#a9CeLCA#QuiwBC z6L!AJpre?%ZB`$^R&%5TfKgrbj$}+E2&yW9MLnfI90N z>!}e|!vKd`xY(GfQB_9-C%j+a!Ql}JVOdh$o5(Lht&<-q%%9JPbe-pWOHT0Cc!hqi zT^2I(cZUrmaCX&mCUi;sdWyY0x5f@uxf2Q*yld~}(6l@z>4)U2-agmbXK)DRH?fJ!>Sts~WX!4oFv9m0@3yXy zYXL-FC8$X7TZocmI^J_qKhKDo?ozwGC@D;=cq*bXV0${Z@;NyQ1#Hm~>K>llpI=u` zC(vN}BaEkTgy_F4x4$^`W(VkU)J>q>iI5H9BsWB?L^(N9(eJ1$Z5u|~xS z8UNLnKxynur2-F@*9ns5_JRp-^Ct^%Z+!R zulzT|h%Ui=#)<#EvF%d^)Ou+NrVTSyERu(6{IYvgU#kuh6$gj+TY}Xj)7wTyZ*1UY zKE4qK6RNA#*o&C32N}5`xiX_qQ)ExZmo+6TQg5aIzhqPH-cOET8&&W_ zhQaFlF>xExff^ZMMi$HWdK%)#=N}w7T$p!sNm~j*glJg-h|0l14EK*O`DyxZ>Y_>V z>y85=L#!XbmkHiS_~9vS>;G|}Z%WKGfDd+ZT|W{Qk2wyRlDJ~Qs#s`&QU*Y33rsN5|%=hC)X((EhHaVN)l zwceF>WFKm)sCos_rgMk)gk}I*$HECAQ;e9?=_0NLrgp3HjU6g=RJg|w43L~cq_32iQbPR!IjRRQphHO637ZBzVR)UIBpIdOfAd{tr)i7YG z*f64{fb83CLeSzazt{|54JU7d{3fk%zSYy&)_2CK)MD8o6)nLh*RI7MK*(##LQ4Kh z*nN#&*{`8%F!(Nd{5LrS#BT4XA&JX)6^Fenuyes_&k~U#{KnRM6#e+VgX8@U7~poZ z>ItSc2O|W=UC!N+bU;&tk68nP$p+>2zg+CSL>xI=*D2hxdGxLacYhYCL}BKT1P zj&Cu)ncD;3^+CcV2Jqz1M|o1`r-vOeI&8}2vO9?TfVAfMD;bu; zz--_K+wHul{eUxM3wdIf;lWzop{W?m(s|tPe~MW-dbC+W;~stygQ%Ft;V+M%A;xN2 zO1|EWZ-RaI_%5^L(X!!~1*7Cpix8q5Twv)JWC>ZMX>;dKU);Ic6**Bltm3SXZIn2p zlK(^i=13+>$s-~LdCAGpMH4!-Xg*Eu_GjfdENXMFUFJwEdULNswFsyrkI*!X?H zp*~9Z`Y!NJtVOg1ttQC_LfCsVYDDGDP_)~+^O*sOp?u`uN2a!d_^7`REcr_TNW)%` zV?lEdzX}dQ<{3Vzo=oALA@(2nhf6EoBFEZK#ARV?8#!Qo;gI^!$iA4yy!KGR`Y~qM z^}>kJmG{BdSiPkinuu&YT_Go~Asaf5DHb(ZN@hT{vMkn@77MiR1ZIqIsF(ThF*zQ- zbFcwIyiIHNH)4qQw^F=9Q6ghx;OB;!4mm5_8_eCcXXNcV%%I_%gQnnb4OQ1>S`O)Zmr~BCnzh_7f_>{?aCy#F*pkKP`iB@t4(S<)47F{oyAqLA=xM2Dv1# z$pB`cQduI<2ou8d3%gxUgB&|)Z%$?{FK)c=U*7bbxuy}o`@|1tP$|Qtm}$&;uvWPGZB9#n0958 zlVnupoddftoF&Zx3+jxE8CBfmz`{e_o}|T|P#omI0*CkClvP_djP3dldgRUZ1L1h` zkas6mo~GzUEzwPOW#^_`YpOf90CUZ5*bu3<&^%gLu=wosS<&2XC#ZtpalT)bpngQj zciGCwL*0jyPFK-3w=i@PLYdpL3P3BLfOmZv3 z2HdhjR3OBGn0^TjHcG|`2J`itPqa*Q(+nND;4o-v?G?`YLMdqsuKSCLUjO+Y6#IF_ zXoN=|j|L!8(EMN?A15x~kn<7Q_~Vd$momzmfjjIwp`iRA?Ewei*OxOS?uo1i%>4S0sC26elFdTi zX)Fr5H2PY*)W{kNI)TF}NFsRT5zWKbS67;doQ~C&(C!tQIZxqTMtrEXR?}p;5MBcP zXd@$-wZ31;SOc;bOw!%`oda-gM}fVUAIy^hx#^@O-cR(&|=uRb+Y$Ul-9h6w0xV`;K*}3*fEfhD2AFv)yo3-k2n7B;>ld9{+?R4w>7( zAX0$kM%Xm!m?VN1Cp&Ut*e_(|U0N1&0rKTCkG$Cf`NK5#TyPB@cVDVz)Q%u@A^kCV zpu@i|T+@ZMsk}M-GQrcJr1k<^oNBTfg|FBwqMv;Py#Rji+f{O)i=c-iN)x*A9nf3Z zOhEc$tZXTID{j*s*A8Qt6RTM-DRMw+i|@3V`oV3vtGt>MW_q5+j2QHyLA&#UnLcx}3ja)5S7lv>`)J*-G0r_Q!+JzU@^-?P1huHi?CpX;Pi zQ$?1E0;`CYbw0fURfPmhur_?It~9$gs@-D_bgNEKeWJ;=luC znp7?^?VK+>%EXKMN+n$9(z!L{f1g{E<#RVwV&Hl2upsa^YO*>l3t@Yt@u+^6Q@nmtS3ZsOdw**xj+?_Pp0%^W_q>yC0U&navm;Zn@DP zLiInpS59<#kl2R7j`|2hY*cbzTKl?NfX^?HHBy0(1Jp45wg&p|mMZ%j(Bi#Gqhp1~Zf@lxh?0(NpkmkBTnD^Sbd;>B9&6ii;@ z@_@;QNqM4zBx@fD$@doG&PX+P1}6_8^W`TWBZYz?w-)K0jQv$-+^`nkczt(to?A&i z;W!vog0Ab;5f%W4n=EptN@cG5wi>E0V5US=nwZcq`h_^m5z^*b;EVp`Y|~0pQA9&& zhKPyii^!CEYKVw~_`mp3tC^G2CzqqF#tWS@8gMRzLlZU;y^4h-vWiJ^WmGdo^JjYC zIr@D-XYrGdh(8CO_6{I|T_KPqgr#^|nG^Tr$5Fozm1F}~=;u(3^o#IFtJtcHqJ;)C za%zSHz3&?vn@;rDPB6%h`)(u(+eK<_=KPgVGoFV4Fzm)=c1&W~Dl- zD+K$lE%!WzIcUo@nzIUkzEZpVAK9%(e151|pIUFB(@r5eII?_%}|I)UM-|}#q|>4@pLOVpA&KT;E_#MZ(k&=- z^mc)JIAjYjjwn<0jHGOCkBtJ3@#I31Utyy$r|en*3gUydHC)(55T@%*;vCSYt&S#Q zbiOb%d}@+0AEx*+rEPA;h9OjP5^M|95d>Nz^RA>`cy*UHoqM9xtq^i>rkuxMh@?b6i{=R-5GdRts3g z&Oy}2(+D23wYPf+My?m`WF%NGC+(CDu%pCrr6PrF#x^Eu<=Zu9)Q8q+X1c(dl&&OY z-TBjTL*rt7V+g7i{16dUOZk1yCGGNfqXvPa`~xaq#iMgQ=hSW9@36y#Q5QYfnA}PN zpB%VS|KUHYnOPpvDyeq#l?scePhr4-U=TQA8}Zi5dX|sckCZBsuQA0wZ?=Y#F{T+r zopq_v)E>{Q>Aqxw2;w%7!PHvxyGu!{a84H!`~(_sJU+kIMDI|`*{77dWDz<}Jzc8y0}j$R4b(BdjZM0r zX=_w=my}bt&Wp7@GG!}xEX+xkNJcGm1c8&a4MLVkQN6(P1{pO)SX6r>2+E|7o7{2O zy^-_w3siX*D!&A)wIW9QNuNubRcET5;B>S79BT8)?aOi8n%Ns#*9Nv`VXP0fqDaSZ zdVZrVdUW?K&9`h#2hNH`Mhz8L0wSxM0#z%Pr@^HU>*%2PMhNYDfy4#Zy^{k+Li}js zBX*w)9eYd2h27on_7?ehL2{wtIDV9^bitOSS3cpM7OVPYh|Lib+zz`DJMgRV*)z1y zga)AAIEV(xL(X7EM_}NBK=d+~_-X--(Blev69WO3o}z+2(<`QxGw#e}XhYgXmkI~l z^eq2sWVnB2cMzQl>=2Lvp;)qKM>msMQ( z+}NQH+%0>wt)@(7jn@&>4*BY`4T#cvC*_T7933x=vV9vV1GK`2<+`06@5=djNJZ50 zygAt~y{=lXtaj%K4yK`I(HOJSlgF9bbAFX59xd0rf;;bwIhtf+b(BLF6F{DHw2^#2 zxu2l=>GInet^rhH|Ax%pe*-UuBkU>SebFA0scqkep!ID1eBOS?%+Hx~^5fKSaoFHs z+IJn{XhU$e2AJyJ0f0L%EPC1(J}GjOtaS%OR+i?qqUV~;k?Z>xk(8SEWuz z6Ghf@!kGr-iMl^%rlikyq%8JPaGhXCH}NdLW*Hdd@zYIEf<+y3P}_VpgWcC&EgDC? zag%o6%vJpy9wSr?kFqic#i?!yFG@C%K3~i0^xOKI6{dZ;jjqIMu{RT#Qb=(SlGq7# zeW2+5JP>6K7iK8u(DLzWI$y9Q-Iu(q`q>6HR7*2$D~inCQ;csN6{kK~a75cb*2>Xwhn|_)v+!(aT?X+ zkz?LOL=UV2fskaJhOQGPDx}HlY@naxgaD$>Fumjy& zJw$Xmp26aoZ4=Cd?vvL&Ml40VZDQ+`hC)WA*`f~wzd-KxxMJ%#V+Am%kkANExm(}- z!GLV*Y!?13Twn9;c5}9uL4A8Y&eSSa*@9J2Bo&jc3>)*eTg<|lpFhYhBBUL2e?W() z-ANs&6t(h3`WmZ1Vun#&bYahwBPRdR&vVBI7p?k;;=+9dF|{kS=w+GQbxmw7NMIZr z1ov;YQut6yk=HoxIZaS!Z3KK_)DSVIWR=@BGkqq%skS;9^F{9Vl2QFuT|Yo| zGw=~a8t(xgon|X(XrBZ;+M~3(_cOo( z=6^c-$oT$1G1tl=-c@&&N)BuoWp=3d!1$}S{=>1$w6F*zQa;dNbdn5BpHw7)C z5Q3!4KAFibLiC1*=GKGxEJ!_{h%{dkL8Jd_ms( zNxN(;N$;C$VrHZaoya602u`x8I-)S%2U&@zQ+w*PHHEAHTy0_|b9Hb@OEjGGGeL^L zSQ2`&);bHL@8tV1#7$@2=_aEv*vRDP5oDb8Nbr7JZ5OBKH>hZ{126`S0N*XbHu3&3 zf{7VEz;g?N zW$F7vaem7z(VHw!*GUqw$;I3A5$0IKKm?CS&(k&amNFM)|u@$l!kAWEKp^ z(tTFLOO{$7k2gqgcRXnK)2ue3D*!kqE z0ezM>e6x7Z2f9|$H^DfQn_&(O_L)bBZO2$L?oxaPYevOPN+3&Lbe&hU8TBSj1foEp z*{8aD06AUPFsi&L3)@CtTeDj(Msezkytc1{_>`Q?BEO<$cVvrZaE*VzD)=qJd+E}7 zMterdX`Bp-tB~6|W<`|)(LZnrnYO;2St!Q8^kUr#RK>VB!WPC{VBEAoz1C(hYFERy zrP4DHBs>sYX#n$-gyr+5XEXuJ(v8_Vn;j#D*Q$`z6)jFW!Zp`D7XN4b%Y<3fLP6a- ztYfqzOoVpJM7v9#Ji5t`EypNp6cbd9pIkQ)CSajei-YN}=pBQ9soJK|PRXR;jSe0a ze{1r{T^HP6Ts_-PHH47T3!`+~w$Kl+ZIx!$&A&OF>;Q5`kYeH6z5XPLx~Ee!4F5^h zfN$89=3Tl7*ba7_3*rb@2Ul0ZBqbZ7(5gow^t593_!9r3 zmz{~233Z@x`N%7bR}x%q#&U?>K3s)3Bn(oQmbpqUhjeYzs?U-2IO}?D8xdNBW)Nb_ z-`)?i`4X6rWtVO>N36^w=wmFzvfJX7*3!kPd6~DVcbm z1D{)AaN4L?W5E+7Tl(cHMt<4BBUb;ww`wbIK2H{HbV2DFfj3p-2n#|MFx4rqk$mXu zbq_x{>-jVt8iW~^9y+uh8iaPe|9Bpr=~6hDOabUc3k&=V&q)GV*A6K4w~pH*HuuCh zqbR^|U}(IN?%=1(?(<_>xc3z8fFz-L>Y|MoMj@h!eH@lVhyL7uVTUcX#ddr zK^@Tj$Uo=b)bT;jEk@Q0XECl4=A2~?Cs=a+1Bo`NDvp&4FH`t0!+wPPFqN)`R*6P3!A*zl z=tYkt%06=DF6>(@U->vTbTEsl$UroH)eRE-;JGH20L=iu2$s0M>jnxB?&5&t%KMv*`4}jFGGT*0}!uW zxEz)+^Y#Z3{itNE5U@;uCg{Xs`Q|+G$AVu2w8dTIXj0>v{&f}N)y>^7=g-g&a9WBn zrqL;)(1S!g6B?f$mdbc)oIkEI)9~e(j2)~8Rix`*;nQ+c`j@52Q6#?``VICTImV$M<2S1sEmKI6X*|uxYDm?6N@to?n z)Y-2g@hj3Eg3;@xi>!01u+g-;NL0-bEcwU>T=(p}p>fY_ZO@wr+{iJkgbJvS$v8r| zf6Hu;!o;MTB8F@*^CCf3YBltk8b!|!Gu_icJ`dT#8Lrj}e9&M5<#r5CX#oxkGsYI+ z{&^2KKS(*d2#9zA^ySSu)^!pCY&%SQ1wZRf(P=MoPO_Hihf|oENq8xjFZNnJjFLH_ zT%m&y@&1gEyOY$;m*9sGqKF|`(MY4_+bES<7p?s}bU<`Se2HJ`#C#k$tsenqUk7o&Q3z1z$^|eq^Nbp#c+mA$b@kn4$W_L%)Os1UYCoMRpT<7?x!%7~-HlNB z+%s}b_h{WE)-{1HXS#PsKH6J&3hhw(xkqmFst|+8X3I%h zlyS3J5Nt(JpP;)l_zgC}NT6HtfP*s)^&A#P{|TDsQ0kkjFe@+#@=y5fY?AAh43*lM zv2|>M5mbnVKF2`UFTO)?J#Gh9pVjFS3`y`xF*t6gpL>Ic{~6Xbc%;FWu=NI7 z)l5UUC-2iF3vcE^tL664fUK$7I80)V8>0%rVG-HNALFBA)df)+%sIJS8~~jo@8fL= zHL)xH(6{xa%`?CUWLn5QnIst%j6tFEMY7~$CdZtMM{YJ6x#w|>^hsh!P8hPcASct* zQu4P{S>bK?EN5>9gj$$QIu^Gktklk|jnTT!k*a>Er)QV)#18!t#!lOV>JfOw_M8c3 z27+6OMocK-bgMBXF)#S|w!o9g-?$hQTXuS#RTw^@gsDKMcrlqnkWH_mbNG)jLa{K_&{dijR0J55s2Pnf&?JQF=p$C;LyuU7-UnNX$(i9ti1Gz!G zj<6}%RXEEs$b-+p!rwk%;FU*W?!o)v|efU1!vNRg4&IEebzr zTT4fVT)^KcsbrIk!orwp`1n3kj0$dZ#AFJrpVo2WxeMj3l`!q=zp-fBviHle4xAi4E zjrh+hau}^rAvHhIN3=Op4tgKdi0qi~6`Yj`8Ug&+A!JrxVYrNQFbWM7r&sdo&Jy0A z9B2gMF<#w}h?mqdg@T=-=A4_G5TK>c8{&Ez3t4xj3Z{hs?7a$d`vQr7fN6>AlLWz3 zL+#T`4%|14hhJj0Aa!mxE+zNk^*faBotD`*Ut6y9(z3IZsCis2^xl-d;CE$7An%vG z0Kf}@oCRf>{x+;AVhl7}2$+MQYaW}|wj&rBuZ8s=N>Ou)lH_YfQF91&D>kp|9WXjA zpqI&#ec1di&UY5(H8$!SOcp$B*P$b~1D-Q0DW`8o{)EZlk}vwm$}erMn*m5=il_}mL-LchRIiE{C?Z`fZhJh^14|wQ=ic4Js>HPHm~K58@A|{=8yR?^q#oGMT>$P~ z?FAFc8%MwSe!ngD#;$OKuXNU%eLO5!8}2>6Q;ZIZYH#n4+Ob&C(S2p^WP}G;!{6D$P<`wMw`}(yqUyJ$c;U?Yc;lY7m{X_aBdlGs)uW$auOjX zu=p&{wB;RT=6`c#5EaBV7KLKv^%XF~zA$K=9%Ou&6Tq(!#Efqd{{9#m)R*|33b+gC z3edvtc<7oe^{E#A{<;_hlg2Vz8t?88mifWY?oVPbD!K?m_mm`$P+%!b6fg(Aawt_d zrBZsQI#+?Ov1qxA8lEXihHVF3S%cuS#lyjhljC)6uM79UkpR{btJ|TZnF4@WNZtx(FQuojMbD43V#L{6C_`RTBvXWY%GPui9))@Ys&vM= zF;cLx*pYLV)syC3Rk$W`SO4hkc&4-NloPE+zd5B5zme;##(z?au0kD>P5yi^@L=!I ziLF4Pzw|siZ4rplo~$VtwFz9x*N|maWcBM`Htm`lB`fV5Wav6~){YOY&ok!*U%+=< zPbz^m#~15yFn=2KL1rr0R{uda+mCphuA!vLLOWphlxj}2qk`8^Y`LyeVE$63^ysIq zHWx7l5#4(efS{-293w;^810J^7 zTO8>``{UjU`%pbiQKWo$T1nVT7X?$8fUEqQQ*av7P%mHskeBK#yN&HvyjtDV zmi|Z4PuU+U5cNN$SJ%omk8NLFG30k}5$66;4_F z_{rXqLSmYp;W5@^p?F-Al<^I@JU2-*wfW|U(Ttmzp=mh}mC{#$Pu7M~Ayj*R@|SV^P1L7yzFyJFeuw7Vg5sfyS}2Hx0AA^O3TKN zkba6AxCmX(jEpovCLlVv#%{k`0I=M+Y|9y#eL(`+v`Bb z8*{zSR3NWjVT=A6N zsaV)!f8*FbbPc+3Zv)+2bZm@osN=Pq-L?u&fExM6jvnk<;J2*$a9zg4poGVjRoR4S zzRaSi;6$(2F)T>Nu%p>u& z;J$0;344~jP&A0(vVL|Uv@qWX+f#GbC@du7er18xj&Hq>2_{Ng#mcxImML!0z%nQY zi8)=-O8b7C;9Mr0xpJU+WadmVuTjIK7mu;xL-==~E>l^;q|h#daL-Kfd)e*CU6A;q z2Z#{Sm4FpeZ)L1I9|<)UR`+vcEap$tc*F%`t)FR?l$yy6U)t^cex&QLdTkL4ldVtE z3T87eZ#tDaMen)kDL0asPw9Jr`7=;!>Yrb}^N)U61b)gokriraVqcXJW(a5LZwK+z zEleH2tT9kKLc-fW@H@tg#}VCthi^D%_UE~L^J-UE>e7JLjJc2U96LWIWMJ1Sn@~>9WEqp_x)vezTYPE!;=R~Uw zO$Yune>x&n_Ha?tzApbAgnAC`G*l-AV*zl7vlTU@`^9cfg*vUMqBu=#TWcN3<^`0L zvCz1lU1)eU^eJLe8749gTPovBN<$4VH(naHxh3Re>UFgZCADC1B(o|Tv^7#B+MjleD{dvAUukRN-S?HI8zPzGbN+IBr z+#x0#FFG1;FA;sREJKV2em-L(Wh-GLW+|dOTx%Z4r}cbtMPJhTV_(IWTD{bwRdxdS zDg~wfPZKnE1hm}=81f;R9SaWRO6hG!%be9*I4MozUM=fa(g$agKZ(*SP{!u2+#fp$ ziPEwjR|!*+jT{~Hg~J`$1l$^``j3EMIL?hNElQ-Q?qIS7J*H@8qF8Ip5VkRV(AVX- zVIs=2QXTMbae}j^G!5lp_EZM${C+Wbl75^7%%UDyIHJ?+4hPX~f$36?^Vnf|! zA^Hq?g-2?d7dafmMi zzlh-e@n<8!z4VQHP4?6~@vJ)7^I7oFIdZHTNPO*YLwZ_1Ju0kFeA@@!z`?-Y-XOvB MMwLW-5#Yi8ANG{p)c^nh delta 31159 zcmbSyWmsLw5-sjda0o8J-7P>!a1HM6PH;Bv?(Xgy2=4Cg1Se>4hsR{@%uVilzuvEN zzJ2=iuCA_Hy-GSb5-cbZEQ$zNw#pN;%XeFsA-=qlR+O0L);c~hgN>pff^4E@#6#Kj z=?zn~M4*!*!A@Ccrb5W`Mh;|b~69+ z6RZ7s*~xnSX_o^FxLc=xdA5Ezuzoq`5O^H52App=fSXh&;OQnY(HeMpKGtq+y*<`` zx;|gErhhqaX#dvuIJ*9FUHNhn=+xfI+xC#^lxQ7k4g7iu;VJz|1R-JBQ>OOQee(ly z|ME-Ay|bhJ(@$jJDiL_%0A3Q?J*X~@MezYR&Jt`*d&3K^QTg`Lc6^l?XJs z+?*hD%=askYd=4njSFb403-qr)5m8g*L#Hkzt-BL!{gT3=J9wXUF+=)@{K@C`~B_K zMP{1^(9+oU^3&wUtbn_d6L43!2e?#S-U+t3crb?t=rAP zI{(_LyYq9PQ~xsXcriP@{_^ACzO!)rrM3Cn-Os1dKx=w__^W$D;1>V(5OBW%d+Ptl z_U!Bqv$^;x%H8cx+l=R(=B)>($d&cSo6?(8ndcwc)>c3h=TrYv@73(iV7tf7*3;^8 z>WeFIA29xM69^r?uFYqDx0(xfL-_pTed^Zoe1Z1UPX3cJz)9?K)o_@~G`cz~)1Ug# zyeF{ge%nCE-}`*o`nKdp1J-Y!Mii4z*ZdFx5~c5h3du)O<`!Ya%7kt~$wU*zV?V}0JQ zx{8)P6!#kG6CT?Ga4Hg#A|ZYB=jVQ$yGHM3Fc&bvg}N4S zp8Zrjodre`Zd=7mS(1aCscw9UvDOe;81_%E9`Ssx@R@e1%5?IBR4J2!rhY6f@>}4h zhH7|U-e8Al^w>z{sOTqyL+RL!QhxVaKVwe_&0;;f4w%Z|ceycQC8Qs+S@WNml{Mm} z4-KP7T50hN9ELZuB}f{Ce;)7@(=xDpFJ>y79uhZ*?-T(KXviEZ$9ng@XrhImUSd2> zO3>GtR0Z>@B%^%T_*!h6RtMAAq^<^FKRF)@RdlFzI4(vJp8*?LAxKI?l?bu8Z;4H*MQru9;nP>@JfdOMEEG;p{4Z9ny0zjN6gvm^B8iI>jdiy*u_Y`us-4Wz z8UFblcYAEf%CKw>UV$Z@AHG+EZwZ&4Ml>`a1EtI*a@C(()eHmtCo$F3Lgb?cnw_!13*n?%y<`ZEMBGyf??XQx4L2 zmt|K4R|*4*o*+U+3_-M;_~kIoLY}$Un3o}y$X;-=oP2Sy%mk%+a6*3nAhwyd9D+x@ zr`M6m4jTidv*Ubk8Dl8TQo}+Em=8H;;bU1K+C@p~YcEm9AlcEuotQA~sose1ZQwu# zi~GE9Seylg7@D|QE?cUS*vNQUbR7mt{Et&r9MM(EVkf{IAP()4&QL2RjaJ$wQ5gzu zbQkGX6)(oO+C4MT;mi>7z63>=PV_$J_WkAv`Ky50>(9wEEZi?iBocyrl}T8LBpVkY z_)Yh5Mp~#|1mv5WP1syva6vj|Tq&QcxcpP(2JZaQq$a*Wq>DwlK%{=C;u7KXh1o+E z!wj+$j0J6a95)1rScF}0hK8o_;f!V3f%C=K@pO@0vU7?gwv9AcV@a2`IAaOZ3O^R| zRDiOR_T5;tb3oojuIQJUfpAG9gL9G;jI*>LV-X_4keck$hSO5}fV~g9S3eXMcl4H3 zgS!^&j`(QguxSG0bc=^fW-AdS@()`I&2P1hCy0B8V+=r(W=Lf7a)Z+G`Xpb%i1O+q&3eXZu9xYo zR@w5WgJE@shBf)&a(4*Nekb|4;C&W9YX*^qfpCC1T5jCY_}Fw)(!XLl$gPqQ&_B={ z$R6w*{@KqDR)QoB1q|f^yxECk7LwPPeD}&s%Yi~G7~4----l88%1Dm03O^8z1Oo6T z-%a?^_cS@Uo=96udr;kD=jy1Yqd6_htOQB{+DUfwCdM_Er+Sw@wn8queu#5XTrgJf zLrlG&T*?H0a2n*uCPMgF*=MntB3U`*Ij&LtXzom$1htEEUJ4Vhz^DFn zBS;>6_fZLqN|q0Tsai^q0aOPo8TBoX|rIcy#33dW){j-K1Xij z#Ts1;JS&tR0+l;98v;9<`VwA4^sHvqp}al+%u7zQATh0dv~wSR$rIJbm`i|O)~jeU ze2)?=g(jNy+_gepzX(?Y;w=wjTa8-y&mWpuAaW%*5kH5U1;;~YN!{JCsJs=|_F}1g z(}6o3W`9Aie{@aS{Q45HS8U65@}`)8=?pkHs_BKzdV4Uy7WAe-V?@x9L!x3bmvM&# zT&ihpd$+C5xyWZ92<{~KqAeGSw?5~;YwLx#j%_kgn~k?Fgq;t296O-eK|3~KVraRd zh{@4L2QM`!TPbDbu3t1D@E!BKz##|Ux(jdJP7t*uNqO@1%bx&+yVw2KkL3ECC6!k0 zJT#vpw||6NgUC5s70BK8ae({lC~>qQ4v{aax^Cfh#i2IhBd*^DbVQ&5wzisK_@a;^j8!cyXXVuFmFD|>9B_C& z2-fbQ-f7gk|De&*W}`hGMv0g)*YZKkw;V-TQ(uW)A~OtAhe3^y-wyd#JQ^nzO&rPZ zmDum{NuVhTJ0>LE1x#jWq=k%jV8VqA4(CDz^cI_41#}yE ze}6srz{1rluUaf4z61F@ucT1YIYUaGK+vIfWojFKS>W3?qMF%Cbj*#iHLTdFmY&;mvb%U7R z9EdOb$?2r2^;rdjCT9z1eZ*#`X>CHTreeEETM2}b^Ua7|4NUr7X|G__tg;%|NYAz> z+)CYqjM~MF6>L0`y%cE|_vus^B8XaxiJ<$27_2B8oL3Q%9xRdoTU)hU^!$=v#EWVc zA@Fj?BWSPpz?-iPIa-kVbd)w?J@C`tiUjL+L$+uQQTg7VuAz+Ss817t0D6cW+Zu6-pamx z^LvD?XXWodQDCBoYbM;;tN-!qS8=v(smQAb zIOE;}NghBO^wnCV3h369K~z3`N~$6gf+YrF5~xvct3-LDIm7zc=HnGXHqa>&YiJ2C z>lgYT*Bm*|Lr;4%f}ccTe~E(>QHG!QP4o8gBLdO0zt!yBTUP^!E)rtN`@o;FIgJ3v z`$ZyWB}nh0Sot^N3QG+3A1*&3_!;^X7VAHK8fmnM5~=y9M@1jY}>A<8*}j5LUM%${tx=x$^VhvC{&ls%R6ym}YujkU{z_SfMGM{q$DgI+pa+8Hqif7RFVpU3JC@A3&pq+*m@M?!pnJ*=A?26+%!@$&bF^|bZ;CRlV;-UumH!9&@ ze0WY=_wcOTN-YG|39W&q#^=kZepUWBOi#^F#)JG6%ttcc02W6;+8n!|i9a_Az3&I8 znAxH6oxBQrMX(_zd38P7dn=3_|K_3Lg>SlVp@#ak8HUDiZ`Cpxjae{px6?ku6*x|v z+mv|e$HDuY+j2P-fgEj0qwIpigB4*neSS6{o2wC4c~w0<+#%$fRx93FC$07tkjAhvP#vyw*V5LUW9|*C zVl__&--NN8i@__R@yeK`py0mrnJ?MxJ;#NuVM&m|GcGKh)OMPV@iDfd4Z*zheQJrV z@x|$FITFL{fs5$I#Ucn`+ZWYx{rNB+Irp@&d!J;*3nAxAK`6oDoBEt&irUi!q+ox2E=v9TOWSZ6GSU&cGvi*xbRUB3F z!So35|Gw<6cWuV-H-0^&`;MJ=xTL>r0%Tn7n71Jz7#xs!=snG=nsRR};E{Z-GZ)*X zVxf<^^k!2GhSDyKzf8VT{Np_YH3Tn@TfI;H&7HWzhadWxj9O*y(Cpn>nxAlN%7!90 zZ#6RRYqj{Ep3cs93%3Xf%@)TuLQD`i58*k_^RCUnwRg@a8ub9*uMAab|0p<&Tr7f) z-=13{LIHlt@tDcZApsv9;edW>yHWpD6EYF;@dfA+t;Bi~%?Sn(%-gtx8G_h~?S+e+ zU{_R9sbGuF5RPf-$K!|~X~Uv1`nM7){PcVM zX9SHK0IPT3lY=Mkx;LIQI2ATUhpk8O$*rdTJP$h)QE>e6f(dx3G{)(I%^yl62?TA_ z&;3VCiOILG*0v!ii z_u#D^`!tM%F<72Gd@wz=2!Cppvy>!GyrewAJGgwDCGFEVNOI+?=;LtMZCRf;@Y^c$ z?Pw7HP!__)^sCrzJJbi>SP`Cld==+FmO%r3c6#JAVAvgwdCex5X z1hk%-!X^}AW2}CH{}Dj?82FXVws3P(b&w}z8Id)H;EVV64yKq^!_I-hn(rOZRTP9s z-g#g6jVg(YQ4*k)3EfR5f3JiTvtsAp8v*ws)G&8v_?i3Yy-I-CWIQ!%f7n?{5;Jvj z9``STroEf;`A3~rjzz5yi|m9WCake4uOetc8n)JF#59LpC|QK;^E{u;Kp!N->Kp4clui0<1N|ADaNZ|5V&{>`JUvO@Ec7 zO_ZDH!%?AO?F#y7-CiRVNvhlZ==F+MVPBdw4NdfCDD+kx=i8UW)VNH-W;_Zo!$B;o z8ABRm3aJCf4MP5PQG!MuDlSJS#V^3_$&PU^I5HG@;oGnwrMDtJnrj+BOLJ@cqax!^ z7kT%STLi|dXL2IAjpMWPgJUL}=a%m?$L}YTyJbEIE3Xq+TN5s+Zyvc!`ZxP_*#~CA zeWcfJ^&o_DasxclXLH^UnwsR+)DC0}i=M2ll7(O9&CHBW+p%B~4%4l;RSRKjJGp7Y zE2oduY>P=A)>LkBGo*gQ`+8OxdyI=G-duV(K0i7`9~mO8`4%(d+v7KzBQP7HoXY!e zmE@(KR;qRfW~<}riQy~du4$Er%pXb`XkBpUYmUS;j15VhN)@eNM_8pFc+kblEChN>pgWh7j_X6avUftOULa<|G**-Tyea- zu+u*wIr&eaMpF|s6Fy<~RHD;y=b>=CP;Son?VP{#!%o%ZVVl2qn+byMxKALXIQkN0 zbgh?aS~f8Gtv=E8*4l^JPztty7ltL0F=d-p?pI2>$ACm3fo2xDkRF>Ok|(2V`_ub` zT0vr)N6w>_%dTDLDY=BwLk0FxD%%fdi#<8?Dxy)+g-Sg_LJaygM6uTIXFlX` zjs7j~(p3tz?iRiAW&Okz7;GhpKElYzf&XlfARl+Br zcFYVPt-t+dOwc&Gi|!gl6!;^8%GVmkSTf9h zw@7lOcAA~r)!XpC7;AY+JI70XAayF3Q#IF)0R% z3WZANNIqyqX3=8hZ|PcCw4_LV&fqdd7XFN8IZFO z%D`m!N-`c&$w8JGDzNTRy@V7wAwX{HnM7spLS#4J#0fiKPl zM(J1wxJuth9_HU$tEAC^Tk>^UUkGARZ+5fd)dW?NLWNZ+&T ztDdSD`-gH5f2m^m#U?GLW4-Q zo!kAJtcsu?6Y6_-gz=MxB5M-t`l+CMP?yR+9Feg}#Iix1^BwoC2Xe2B<~7Jsw%xNaY4fn#(F}phe|JYEx$C=yP3kF)7uJ3aiU^?^);5kp z98MH<$y2r1Hzpfd6&~gb1VY^Q6EoDDjuYwc$RE?$jTECaJ75L-Ij%eXH#EW2*T&YLs-y zg()^=ku4t#=da=joD~*=u`j`}0uu7Vq=Ir;<>VPdCaznIHP>Ha;>vkD7O7jhKX?aE zuPATn9jL~Y@Ak7W(JKfWm2SdeTl{uHnFWxkk47%&X%z7cD21*^kHHx)Qgem^!``Zj z?CNYc(>Cykc4O3#)7+*moU>v2E>+yr>N5%sWoo*{dL5Rsf4)D*SktPW;`NYm{yKDR zijl-Q>cK`|kYTuJ=01YE#mr{$b%-Oo`Y+X|D=CqbB2bjzf{)?;q!}JGtOdU;EHD%!i5$6DYEXd7lkx=dwFyem- zgw;>Fna$LAZp?#*R{(cK}HP=f<$QJHuk@B+Aev6 z7IKgo&7>oZas?J{o+?@UVGQ*Inaq95@)XlMD@IT=BOG zOhre=1-1tiu`*FPq~CA<9^Ltoj|AO)3&N+v0U#4l0}DyVLi%y4If+5DOdY~s<&_3o zFyYcks&_!_#TCM7vd{RdYrwEpfGiLc^jmE|c?c-*Ed%alL$wJ@{lLiy+4OLVHlw6nh_le;R0XS7wjE*`g8D;$gB{3)6V zs3n`gOk`8GRiP4(X)lzxtsM82bOn5Eu-z{; z!Zp}wy?)m4V?a?w#f6qSjJvbjYrBVAp2rOg1`q_v8SDa?2&%5uQFy<@h3QQDut~{! z>Q5<%>yW!gM|=AnZ`%*!rV5=XUgL6<#UM2WIFXj&Wtq$#cmBo0;DD`XCqM8UhVQB) zG~nDeGU1$t+Poi0+Z};NKMuZgh#_C=t0DjJ2pBqO?ArZwN?H2$gKXmk#Qo8|)2nF~ zQa&uS6DQ?)WkO)v;|d!qwB3(Vt`c#-gO6>~z?eLMk-q+DGtn8ZjzTodT{c}M8EMvZX(v5(+6tF$ne;zIYFYMp7;9d2w4SiFp0IS{wwyP7Ws;He!veZ0gqN^g z>gUGT72%mWIrOV*+S;M5>%_Db0FA@>RpMSG187nMJaMb!ga&RYZOY7pUN!4B}X-LKXBp63OI+ z1&)e9JZFiVGFnnr|$sXq5cJWr9}w+=AEgay;~^k>wI`9CG8K=Xn_~AfV6C_T(!6?$WoRW^TvEg_mcV~ zmKjv;o74t5rvUC=-wr%d9)4j>VNjZr=oOp;mX>}0ZJdJz5ZM-%oC}s2Q1(r7<02Y- z6wUN(>A$x=&ZC-3m|!#brr_fj<05{nOJ}mfo1`$Ni*b#CPSO`CMtN(C9pI3*I7#r0MVT zIQ%Z5P*#(jn$JByuATPbr90=s&l3R74G-7pPR)mFSzBva%O?&?6~klWvXzY03ukAC zkbco;C;BkHd5xfmx?zf}8BFvAAB@M~RU+T8Dmm@8beQXP72$ew+h=roAGTl&vFufY zqPgqi?-`GQ({A+gMf2KwS_xi$ydH6}^4?#=8ShQLfWT+$Hy*=^QaQO4xyA>)i<~(~ z#zZr{Y{7A}QSwbVcZ<$K`$s1_m)Xibsx?d@2v}2^w@^@o$@!}0R1H{MMt&#y-nf|X z37?bjNgo{%l>0J>{XDxG64}W9VhkAO@JR+8>>0a>lqB}& z!mwN5 zGG1UdT_XV48=VW}@0oPOqH02y8XYSY>vQ7?Ab6~3li^ODQT}Cs&U^0GCbzF-Y+iid z)7|$d&GcysnysHi!jn6-^JgE1`(^aty^;3=-6RwriC2(sNdEFFF#g%*QojGO86K(|8YNseX0lYA(z&GZbFN1Mt%(U#XqAd!MjJR7la zC%oOvdfq+gT>;r?}kd^L|7S{9(N69>AZLYNXW5~Mx^MMV zd$eJA554m9bryQe^=BIf)-3RwnV?vLMOlHB&S1$2a7DL(3=PTddAIKEUEp8h+P+^G&vA zqmxe`;-@E@Q#Li$w>RCb3^v!@t*-_FhgRCn7i|sviS6nbT?PXyw zfWe3;^<{D&I$GilT-_vHX}{i_jRfAshSkF3;{}F_Lwjo`(3EH-7-%+UI2~WDdAV5Ezvd;T5?jPVsQhzq3cp9tE{@Fc_c%?bXf600* zzW7-dy%I;emi1q))W}jBWc{#;@7GR`Y?Ykq`phrgYK9_$Zi^e`^&hgkcU5c+M&7#O)JYlG)q@SGo zj#gc9hvRVN9pxXaG!hV1Ua~{1@-N8?GG4o37$U7y8hm*Nw)fm4xL-f;s7z6Ivu3($ z9(ScoG*?`lmWd1NABzK6{3i(x{K|q$EyhT;gLO7(Q7#rn(e~5JjmRYG>eFo8)ZaN; z6&?p&Wsw^z^cFZLd>Q=s^*$jeug!`5J^>R(mV(PY@$?OJfNOI47J z>Y2K3DO2FdaXb};qVY&Ho|}wLRb_*S$Qk>Wt0;fNo0aA1;nx$2aRg9n3eZE$Mj+UH zx6HFJq)yD7dBi!t*e^`sj6LUe%6btW_R|iUK@`1QE%xsETJs;d|0A}b5v9|y9_zgj zZ!M|%mqL`d#YMzseCeFya*9i*LpvVJk7dwwT&dLTy|s1dZF|~aIq6Msbrn}ku2=Cz z(ZLoAK$t>x$r;z$nN!>m?83BEjQrQTs;U6l8>MM4P?&Lutnd{NCPJz*)-Ds5$n=O^ zC{U!EV6yGLJLl<|j9;9ZAHSD4G z=FuO~#dIsCx09b67|gVXpLhk0D!0uiRoQhL(1X2=K;A@% zA~)B6LSO~2`l*AGyBr3KayWXN+CCh$uJ(gid66@&Ze2^ibT_Y%l^PnjY|}}Jh}%t!%2Ef;=?8^{ zuRw++4jAsn>NxPt&o1FW$ZUNhp;u>ET&)%ltd%pY;=$Xz_Wb+4MG%Vbbg zED45s`=5X*5#`D0>qPSanbkV;){vj{ERTR|RPa)(#)e_6 z#wrM^z4W6h%p;?7bsiI%?cZS!%p?alIBm=SQD_>7 z-!r#spo5H+@x5TH>HgwFbeFQK8ahcdH76tpaL}<&I|`rwV`kC|Cl#LDYb(Hdwp1-x zZN-mw&Q~s3;lX+{A!t=4J4_nY93mE`^D!fGLKQBj?IMEfN@7o3PwC-b82 z2n6%R{c;7`-6l7M=N?uR39?2{9kS~f$HFqx6>*x7Y_?-bBT#18R^X2Eo=8LI=O7bx#$vjoLj9XaLd44EnRLQJ?9{WmiOS&dt;i)s?IEK%}qDgy-#tp3+a4b zr!L5o22oYb?jY+IS~-W&At$$xoYbn;c`@pM_II6|$__tUj%$$Y@Eo4gV~7H!6lu=u zfFQbKwHPk(0gOu+omiW2F}@3Z1LFo1T&B)jZ0-t$NYwIam$@8s%#^^OJ$05E+F3W^^X~h*#GEu%vh|XATLkUlVtMbFS`uSNYELMe!D^sTazo?JR5JbDAB< z$7XmGn7_(L>56Yyy)Jfch`I4P28TF$xEM_R>%lqIXA3?J{U=j2sT}Ztyso98{dOI2 z;%#$<-#S-BL|nDDhIZE)$B#{&$bT59G*X&ZQehF$I@{S1FQcj0K^64it|JEv9*3h z@ho1DrPH$-qJcL9bh;n@#l6A%YiF0M>mapWn3%7D@St82dDwsBBXkG9tKwgQ&##04K<)MX8FA6b**57f)c zS<`Kf5K+~a>dEZm$mdrP)w_6UGUpDt7$5M-+ujs?_39luJOc~)Z)UB9GX>8&6*I}P zRjRX8Ms8ss>yBO6TWsrU8DXL?&rX)+v%lOj!hc*C_{)dqG%QkyL0?tVFdBvZnnbia z$BpSLPmE0tuIXY;h!uVfrQbBnSCXoKr2Bx)*gHIkK~*{L%9V@@%W6(b*>BWj_31ex zh7Z)ONo*`1g`=&Gyh-HP+6w z>+$QyE&0(Q1MY?6I9ovw3j{}+^fE&ob1_l&|NH@s_!NLHj1j%=7t!;-6k1M&lzL96qt|b5t@@YJ zDu<2Z8CD@m>gwVZolV|^j4Iym1e9gi;=$0W5W1#($v>hl3a07X8)aR9DyXVLoVne1 z>ouYVp6rB`#v2hKFw6c0Sn{pmbe1KQgoQdb1JRoCK#TYkW^=mi3rw^WQXX7Jt_3@zqN>rjpZnkiO4OkGJn}=0mNQU!|FSB!^=T`DiaQn!jT;H@jXm+> z8MMBHx}q$^8HzeADcxnj^{e6K48~p}0pG&g-Nq_D2&kO#gymuph}TiJZl5Gg7*e{v zURez97qdvuM_BZ zzXG%T*HoCM?_T@TTak2zo1}h*o}woOPm~j&=^8zPhG%c)rmP0zmv?_^wPu(t{7QnJ z{#n*4AFZJQPHou~aypFoOKUBw@zD(ERl+Ws#Awt~v>#{{WC;XT1XU{}wSM7LO|zgs zt>yQySWP}6;Lc9UAJ$wDG=dCoIVh{#5Kp(@-hI?CwL_L6!wNh5<_t^}6qY7itAk3i zEXr6J#qeyd%5miNz~wKy>zZghnF(+`bLfU?wI+bUG%9gUf%X9xZgp$hNYlDp z0hY&6aMZD+L!Kq?3z;s6O*(alKdhd8y7)qIokgVr>OTzO7eN@~)HFBw>TjvmrF5)Q zsx(pJG1^pU{#K=sno$?gq)Y*Woy5dCJtVc5U)w3YgncHo5Kna9qfNR#z%PEH5>Q}5 zkGBZ-W%oxV6z;!D?qAn_a%lP1j;>4&vVF9ShpvBzn=rP0v|u*CBQqQq9YARr%pv;) z{5{VlsI_Z4nBeYlPenF6Eu`IjyrRB2dLVa39r>EzA|k}`1a!-fczBTH2K45l)v7lG zKN7!&mrLOvI8P-wgBx<7)Cp()i~-DXsPOBO#yo}RHArl&=ilS4jMaaYSvF~p-&TLO z`i1N#QA-ms^zN~^6|9S}cZRF}r=+E>^Z-Il$A_HjdW)!!WgAyW5aLULE7^xhA#Y|M z9EKLy>FeEp9Jctks245S5$QxY(ZjM@v$($0=u=PBRfiUbS#HS>C$^2L4>SY*C3Mhw zMyq=F$A-0YHM;7HRrwZ^(qB!;SJ|hmiJz_~O$bUX^Eh}w4A!`hpe(xOvkKM&*uJvc z7Hx^CZz9I63Qiz-bfj8{2Ntz>B$b<*l1GhO$}0R3-YD>U3PT7gU9|%;5LI!wS#ILn zT7ig?b=^S^fG-_=8RP#<*JXYk_w$%geKEM*RNpDD?M?{Xo-f4>6mP{~it+P2cRQA# zg|k4@!S#n@^F^Xal?^SBCcqFL?~cRb6w<=3A&mKJxCEW{J-0RSDQ3x6e3+y&Ey2Am z0z=ZL1iYiy5f>5vjEK$Trvgx)o@xAdj)k7!XxU!5_qd;=xHI-=buEB#@%0_nnVLdaD zuY#BKNkH#~!%ThU+R(ysQ?@_$YY!e(vv(H3y%&IWO6j9M#LgJ)*@JkE&e7;Oy0Fxf z9r0gNgRNYW$>Zp=G^BhkSbh|>tW7pa#lf2a$;Ru^lFpR)W2hP>bA;5n+jE_DC- z;bH1&7~cQSz^+I9CV$W)RDEHfT^G=H4Q(ciueQ`MyPKK^T;N?AWb1bXJ4SBu@vtnY z@qRR{=)Dd}W&dZ$KO;uU)FsvTpYJ~mHt~-U#}60}zyjahA@wnan=V;1 zPzp54ma%#=Jq}QQyQ>HPVEB-dSawKirER~BqvK=?c-(D$Z(56V!fMF8Hn---uT=Ib zET@ruHr?PCbq^t-f8zZ}145ZD`}w{;uXbvk@A+YC8hu@3ePxy3JwFo~>kt6V{EPuf zIFp&5UJD?PtR!_d*y(FE#B-VK?7WAhnHYk4?G3VJQl{7sPm9tv(+kJtIu}MRq!wq#)C8(xtv-2lxi7odT{RQpkIPetr%N7nR>_QYrPZj$j)Lv zGPDJkHe(1e%F5NA4fjp9yTK0JBV`G{HYcV8R~o2aBr7f%eNV$(3}>kmPoj?pH7J>! zRw<$o{&nC^+t)1Co(wZ_Q$s7b&QXdUv|X)68n6 z_yLU+UnO_`w8ai;%S2>j_T+c*$jT-qlN5J0hT*t!cQ)aBOBON6^Vdl@$;k80CF7!Z zHkRW_KZN8s`#|Q{SWRD;%2YV(%rLX zK4q&}^c5$~UE!YI@`ztU9Q~jWga66^j#YffBC9kn;c)IRPwH=04d4{&sm+_WZ2i;u zS-zk}SCb9bE)-DD&y4E9s8=kHwrDnh*WfquyZiXuxQGj>4M4=!hr^}0*Gz~KtDAj* zP5SDMR`GA=r$ohGGE3KF~K`St$7LXDc^)NDU8%jRba{I^iUM0 z_(nld>D1i$JX?>(-{ITqD791Y|BhM;8lSmWN?*_EvsPGdC_Wx-S}Xez?z&d8Q_g*2 z##z3e$w4P}{FbDq>B;ca>*uH-pZH9MJZUdh$Y_~>>K`4abPN2J=P|DQU$r6#-5`7~B=yk{@a!yRvY7VzEWUU+ zbFXYj(|*MWs*~s6_#*_-%E)h^YL0`0MMxsp+r7G}$DsP5*UTcfh%3GKVCDBrj;h^G z=qxC8`Icujpr-Q9|BzM*{f|Qnl2Ek_NmjM>lvWA;RUfUrW_rJ;8`(Em>f1i=H58pK z&)is=xz$#1n7ig?G?=-RMl|C}A38OuXcE;lwFXcS8k!J;`Z%8B#(h{;q1+P<8Jbk1 zg#ImX0>q|#=rky`!E3nb=LhGZFdHuj@1boIvJA_(nzo$gGQK72N)3)WtVTT&kh_6h zJH+Ne;@yHuA`Tt6Zb37yyxQ3zG-Jn~wiwNqi}q@WV;QLI@$k85L^9!f~7*0 z+$rw01%+)T@eMV%kPhi()Wi1@V+A2nXS9}`f!A-$U+qcVEo4QUSkqnxb-|QJ@4qSY zTgh_k**iM3NA@egBXw!5>~J(?t!S@YZLMUhB4GNS;PhPus_N>TTZlP4Y}C=age=;n zu?!M+dBupe3_Yvg8>Q(MrucfviAuM=$e`j-DNePw!b&;pLsots4c%cwGtgE+n$O%N zVK_~5bH9Ab>%z>JHaAs!tRFzRA~y?x7Zw02U6_ar{<-HnwKg2enfRZZ^_}3~5bAzX zxu&=vl4dXknh8*J%W76xEo@YEdxw7C(uETI`|{7iAfe49OEhVZvcAv0!LuQD;#553A{YliWsOx*P; zSg)6CCCYa@O5Ees=)~vbCT8G>u(YEJm0*=n(I{wvU|As;!I8}6Cs*0m5#bV2hstv9 zBnAQ8$Q(WX*$f8?kTAbpi;^rCebtVb*;W1NQ_q;$F6_9g%yGDF?*+huBoFE9PGSHgJr z{Q^K^6CaXR&QMHhLUaPY5zB#NnZ%mswZAQ&w*0&(M`@7tQbL6Lds`OL4R zbG`*AqqlUHLy3`bmco{i_Z23DO?nP&6FnGP7s#IK`esIXJ7VLIy{oWo=(QvUl9QZC z+&+G^=}c@j#E&Ku!gD7{A(hfqznDmh5bIV$-8*eJ$x+05zF&opkjYs=^w)?N3%)Kk zM~ob7QZIH;i~yj=(Owgo!c(!A@W^Dh>2sUJ%q#aT#7tWR;q69ko0G-It9k24!1Q>$ zp}I;m`#1gkied~erG`Yb`^-32X1TR2Dt*w9)W-w685g> z_i$&#*V1+05h%&le%N#{?1?|6aoXxx(wYwCt7`*Mk9?@E)e=)t<$P^@Uz(ol2&&(* z`kVAwo0_*3F0Z6)85!j7N&J<{M$5uz$o13M;N~t%mpJzf!82Oij2Zn@QE_^PqpIY>cyE;Na`P4>cT3 znoM&8`*4jh13QYyWgUlg9gCaXQbKhO+*5Yk<0gGgr@X^(`&K#W2wOYAZ`sU!+04N_ z((*XEoU4NEBO6b_8S3%f`Omqi1RPVBDf{=fPClnSWiU7jb>xhHZ8#|FKO0UIlBxSY z8_xK@8y<;_Th}+eauDUCjG8r{3@nQq+-&;0&$APIA89T%M>r8t?0)IQne+%}q}zzS?lL{{70eP9;C(&6#1Rwg{an`4vB zA&3%Jd|I0=otPWE`1$knl;2+_^1kk>v|#1e>42V%)l7!Cxa4bmsx0+4UMmC{JtlnJ zy=XVhsshtzcTsCVHHsSr?a;T ztE)-EMq#sYcefDST@xG%{N$TCXJ-D3b8+t0 z)B9Pys%pRWR(Dr-bqb#u#4!BmsR;3P4DmN>qZ3_+meIIz_*jIfvv)V?*jZT*w&*#c zqtxKqKx3yzf4Rcpo;?4XsXPX+hG=2-%B2q%ogMs|{_y|$V|a4ooyoX6TROw zut*mdfGKHzXjEdK`^JX=;#zm^SjVE)T% zV=C{oYpSMiEMd%>@xO{L3b+DaI+9V_cWZYM%eZlF^uy3l{zZ1bU$nWn|Cx>d7umIT zSWV0?{tw&n9;+Gg2zdR?yGFDVpAiXkV@E8UwJ;O}kMbRzsA(mi{t2%ZB)d6Q3;Aat z^B=O~IZ+!pssa9s?s{7ore($w|DG?R7GboJM*Q9RjjL5-i)4ye_Nj%zii#AT&K8h6 zS}q2m{KwWNy{NtO+bs;;lW?Q1tc$)iPN&nj<_G$?U;dXjCxVfW)%g2@J=C)PDQ<2hPw2qeN!$I^iPv5I!KUys-B>! zo&eA+^g^@xZhBzs<+k1R=Cm9gAXaUd5^lW>E`N4Z-+}|4h)#xt(HNHRG7b74&!hal zxIFKK46!n-3<1yD7y8c!ZZm>U2bRkocQ+Smu%bVqYWqZHpTQqf zT+gQyb0@pgo$~n8Epmg|PK}pmRn#^QWq|jlZz+?djQnAwAoOC|NWZyBaOL@Y8(-Ti z2pT^0_T^JMdY=m~y0|!5H-3906&iuqW`pa^@p|?9$F`m$u2aJw9;nm%-R-GM#X-kt zs!^=oq|zj8{=GiMee^a3KJZ}>t1R^bIIooL-lx^#NkPrW_Q9amou`$94MFW^C;)o+ zrCAJJSrAc>52D|hOGC+YvZ;^1I<@WEh2bts;G!26?uB za=mp_qr}S!XV?{z&iio|HYEHcHgv)c)zl5;RI2ZNjkYBuQK(pd^Vo^$0gV|+*?&}U zNBAHymec~Gzronu>=HVK3Ev~E1~e$P^bwG%(v7~Ey+5(lhq@-|Xz@ju-SK8JM~qeA zAlP%!7rllv@iZZry-EA0-_LZjFaLT%QGT1}&eCk2mrQH4XXwnvG&;Paw}afCLDisv z(b1^D#;Dnu8;c^Oa(q?W)(5+8w2WbhPwtfl~7 z?m6157E#BB2J@1?AFtNXK!GMf=dH)D7Y<@wP}>^IE5xXk9V*D1&4Y27<$sxr1vH;w zk9w~9imQFK(qMqTSFFOX!94abcNaIALK*g9!lMjq-u_LnAA~NZ5(R9Sk`4*)L<=>5deRgw8k$nr|Nm%H3~n z%Z0!$3xNP;7gn8B#SS70y1_FK&{`iP4x?juD=Ux!^yEnz-IWuQLsje`9OFuWotj%4 z5xQPYcWsa7L*e1a-pW3FkdBM6C>VDUovl9>N?~!LG}?s|{|Bk-+ykYZB@b#^BlB^d zf(0%8V;Jqn{(YHz94INDiwJ8lW~4F#3F3$IACO^xAB-R=zlj(uF#Qb$LL|awx-Ckq zf*1ing4-)=N5{>w1lRj4T}Z&liQp0okWyBb%ncpd#pNsP)5>zCz?rpP}2ARJINW|dfLH|bkD z+i&Uw!Q%WMg+4N$+!)g#____;3h@P!lGv~s35HdB4Mss3u?Z8l)kXYUwp{-aR(zAB zbPy>DLnwXb9y<fIlWi;qbVC4&wkuC5%^;o?2JdgcTr|)(3pzu)uK<>D4gR;Et9z z!H&&oFJ3W5XrLP2*B(5lXIk_l^n8WF*uN z;*EN82Jv)6)KwQ?OdA{QYhxK53hIo$yS=;rXB^FWhdGYa%tZSs)I8C3>)Dx(D0=@mF z6AJ4SrGyd0PG!@2LWpP(# z*4dV+FHXQ)6wZe~??qAKiV%`^<3&>N^&8EBiU&ByxejMf8_z5RqVl5hlNIr3IgE-< zef_pRB`LVj0SEjVYu(ijcq1H1cld_zG)PyFk4M;&WtQ`dR~SiyZxp0MRu{S178AqLVe*iVzD(DphQM($^jMCWtDe`9~k7Rsbi7 z*Q|j|Mb8uQRUaZYA%G(Rk{t4?P<7`YzPPFAo*(T0kJDo$^o=`6TCI&zp%XRUd|<_t z>Ku*SVLpW@e;>0%>mtn<&^TR4_+zgIJ-wd>( z!m;ijS&~AO{ze8}0-fpLNkR35;4I9_p-*o}9IC86@ABR-%8lRl>k%-jj^B=#;vq6! zVQ|wfl5IF+JY{jUAL8s9Dk;mk4Y7=QFbx)z3g|i^Ds{J%yn|=NbdXp~_}PW*Ah#G< z4I2g!#ofWBAQzu~lpjMc)krFu{mHpV7(NAy89nw$zg0{tWaU)xxtgdx^5>i*UZ!Cnh$bWhP%mFuU< zqGLy8vfvEoJj7Q&=V-uzEApuAs0Ay{NJ=WYBy+4G1toN1tA2P*JKNEFDto!gHGvNK zeLY|?MdQU!NO!p@xVF_oJ5$md#VGtpbNvhwrjgKQB6OsV8a6zOM|3>i+cFl$A zov29gUx~mi0&b3P#Yh31SCDSX)N>|Vu z=wDd~bi|3itC0z>aY;1SMA^|r!8#bx6?%AulLaQE!9zf}k_Xn*K=`RU#eOV{O5_T2Mz|_oYC+IelnizAF1yFpm zz6Fm4cM2|pruBWCUk5L$Ra6*6QM)K~kIN3;+CJ@LZ;|9CkbgEkVb8B*Go@nt{~T-$ zUp^w=D9I$jzqGxyc_DBcA8(O9CK?aydLF{~99QvUXpahKl3H$kZR?UyspySij+W0K z9|HyQSFYJD^EY%+vFNQ9qR4s*neB9;muPlA z$odMI&^LJXQzA!2Pk8*1s8xoh{*W57S8DzjzY#|gs-BgQtkh(HT*&rXW->pch68qb zhB4HfXMP7be`%=0-h$0O=3r7h?J9QDH;?SO>m1{iP{`>-17QH1=32p^W5M~zF;Ez> zOtY~(>loTAjS(Shl5W*&_RS+8(U|Wd1)(HfQC&how}cBwd@@|&TYE&-l}dk=kqe>G zZGgKlI^q5iK2g|qF4%hNy6tv;Y@0AEqQ$mo&(E**bzSZX zY*|iT&u&AbpT3qLZ1Z?|?z!sf$}F)uoU)c)%d}y3`J-q!_Utry)Diyu_JTg*3Zi)f zJzs6IyhH07;xl(phGprNzQLNEu<+}h<_N<2RKzx3>kf=MV4SrB`;2w)mFSDa%uCOH zMb%oHzb$fg-YlBXSyn@ro(Ul@xr47@8Y0i?7gNA^6CY&)i}Y<&;=~2W6-Kq^i05d zv4CYFzAy3N(5VX7AH}8?42a>*#Lo;{o9vQ%OjXXI24e0EN1$C~3b_pC7}`%`tOc)? z#ajWRZ3$?t>!2k*9LSya-VK0wQhi757Ne}T4)GclUE$ho=R`lP@zXQ4M9D2x92Gi$ zs1$1G82wpq!j|q8ow;J)5T1Hd{{^WN*f)%@BNZ^INDdsIDjby%4{}Psb#*y==F?>( zdGAgUJ!X6FzI)MIR&m;VbCezUxb+)1oYTEUI}WHrBQBMCIKlL?vux~cvB5@IaJZcc zZ0?Q;S|Pl3bAw@&P*O|H=E=1+AiNQ8vBod$KIG7-fBcx^^s*A}2I7C|M|;&!s2v==!PM+xsM;Es4Q z5|!0gh8wO8nErs@dRSA?rh)A`2j#dSTw`c*qWXR6jMWQmhk1T2j6qn4Bj$()cHFWQ zu|j3S;ZXG|KEzaz)rVojiCXpt)8gkb>Ga^we=WfZ}`4^niy4d51pa{W(vN{ zm7C&kN0o61-Rlmh2MVwcWqmkviv;D0WXu7bw%ib0^eDww%ZRb&Oc+Q zc=-m^*L3S9oW$~{*YINNYdKySu(d5LxoepB<=DeqHV0~0K&FZg38h_uRTnQ}yarSO zD`}jYokJ`b!`(x`KwUxlp|ym7Ae#Gw6UApe7T)Yy;<>c31A*5WKY&BqL3pd`5%^wB zSR$%O$Va9i2-ht9>3)ei`-+e+>gHe!7Ucv?66hIfmXzd60R_hcN9$ zzcdZuF#QViHZ^by-I^J^iZfTzt#lOD$|2T|ihF)KTYqV7E@}}pU3HF|CzeCNN z`ZqbNq-|W3*CJ4m2~Nzzjn(eJUkKY`-dN>oB2Mc|eR(L~hpI)}bcNV&TVKi(4Uz(? zH~W!}xe2aA2PLEfs%u?gZy2nV;)7(r)rQO%q5GK;sY>&N2SkE^9Bv$8BsDmJH~q5} zLNYmP`mhT`gH)8+A?pb=eWAAnR8)Nr{a=ZbNFDI($UO_Lqg+g3=cQJE-CP_cEs8kB zb!i(4yoDENz*SEDX@FqQnPUbkd_5)Q$XDf2g7%#cxrI(3{^yhdF1%>2Ph?8tCslZX z64aHTcwwF?sm9iR-~^jw63UEdq)dQW^y_7-ym=a{qVNaoWby^m8n|@-=v6Z-CMDS6 zW`_VXqE{u={V#RUHR`#O1Si zVyrm_DX0RW6s;!Gy+%nALaxm{?0A?mf9E>ZGGGyO_`gp`V z(7}3vCZ_PoLV+v!@27jNnMfCs0BE9?QB@>lxghoY7=wT^`OBOitLilRvdt*D6GB@u zN=8}OLC+)teoWSrC&EDq*?^p$R4e+l zfCP-X_kZ?kfd*zV!gh`dl+@LYwNq45>+jxLJk5Hd-Vt!?A6SlMMBp5m`vqJb`X@Bo zkt!T@Rv5D)3`GrlkANM;!}-_BRHkldc*K?;7mJR#)@Rzx@y&G6DeU#H4y-M6_M}y? zC(4tLS~18Zjd&l$ur*=hkAA-sz6#IhSXVVnuUNRj2BO@MVEEkMbw0ERKb6q2n+O+@ zsXaZfb_bJ<2nc#xagxek-Wu^hi@W0aL*P*4YgGHInSW#Nu4Fp#RAAXWerodBr_?pg z$@Be%?p!Noi7ojJ6-Ec&EW#b)x}%C@%i z>v0cU2~d)5W~RYN)Z~E8_`c2Ra7^-5fpMME3LVGFkbH7^`6928mdTL{+PN#b(}88@ z;ogBfN~A*)0$mreXocTB;?&)~Dqp`*8krhBhr&o_T$e=5HU0Lv=-oFs-FBTF#<%51 zD2!d6nc2Pe>9tzv^iCw>sWW6*ahuW6te9jy2a~s96W{497>aGF z2ReVD&X0Bxi>9y!s!&y2`G-}Px^(3s{_-Orj#V>y@oIQYjMdM~qiUVCP@hwOgRMJu zVJJTtoEzKW<02WDG<{rY+L(qb8yHyHn3R!5j>o`d1$}f?L75TRc*41vZ0OJ?bi~j6 z`llbi54V9@v_ScmzwIJU$%=-q&-m5%DUKda(GscDLcSIoDsxJDP`BNTn&l%bKp^DgIF-bF$X_31+`Hb!Sk7 z(Mu0ml^1Rn@DN^{J;{{iJNmSMWpkmi`Aq56Ey}iu~mB0H0MC|7Z#X@lv zu_iSrr&A``e)}mPKDxjkfG+AwY$GpRE3zwPOR4 zxG3T@#vqt*7z3qw%mC~kav>DO3So)La!BP5iPfq?rokQ+Mk^UrBFaC>tfRMgdW^ts z23)B`A}Baahhc#Zh#uz_OaQI*~%G{fu2HF4-J`bBJ8f}wh7-pw!MB1^>H z7+(C+gB$XwIJ1rx#TzWG2W_0k^6gLlWHGA^z;M7>)~wycxKIMV;=fyPj{?ONC{L#^ z5Cv|~H&)v@P=Q$lYrY7BDGRqvGwUF$6qvXz+I3M_8=+pME1o_{a%~nleDns=p#0F{ zuc=8?Ckvem#lnwQ`lyS#p!8Ia?bfp8ZvP}Q!ET=JTx9k7(E*UO@*%eNniB> z8k)mG&7&3q=ZJvBVRG6nzYphxf4&qjZA0m)@`RK)&x%RRzb;z#yR)+NME_yTl!4Q6 z#gUka^r*8INIrZJ#R-GIwBC#3SYko>mSrw$9=|}d$p{hFZ+qH)t@nq166Ioj5&>mk zZ6(Eaw0p<%1u@kHgre;*dq)cW>3x=^lY61Rk^I4h1g9auPk~9XMU{d_DXh=A@j zpxv7m=>#%SrTOC=ax7voe?f&Nq(^+jRjoTb1$%mHFbneIqqlQ7PE<_=NSk<{4JD+L z_GEVbbfgQiSHSxkjfpEVRrOr~r@WpXc^GD{%CGtsib%zp@3pQUy zI!uWnDk#w55{Z!4R!EVxve4(kz=H-(1Usk=L=9LFdqDTt-L`^v~ z=*LlLb+d`%7fT_?Cx4Z7(GpmBz;_+6>EdP9%iq6+uR>g@bBr(DoRRg$EnzcGs0>q+ z^Uoh{%quP-JqbW={#?IaoN)Zf<&kO`T(09D*Ya|)qrwFU={G+St|apM(A-E3XLTs< z8UUi}d2F0a&VU)6;C{l zwl+%Vjz`f^ELzbTE)3_fH^UbT=~7&Wy-N&XO4Z2dh)JH+z{vDNsWh6=G%|5Je+%4! zVa|J@Cj*k^9x>URCa+>~-#gD=Vp%y#J~i9HEDd6*yL*O4YA=G~utU`aI%nMmvCJSN z9BH^*Wk^{D1jhR{W+5h&F|E5KA;P)kPHrgz@KuUQm!2@-XHN>Mr_gTBQlz5fkIgo9 zob6MCv}S^w6`>klCfq4_w+)zu6M$m~*{4nTRR@UcK!)%|q>oAR+U7BRsPyl7-RSb) z<>Qu<-kG_9d|PKl3N%mEiv9iC2<3eg7S(`VGB=hE(?%ErS1G!}Pp{QFHIg~s zmby<#guL$@Hk4A(*czO{zS?H*+qS_U!@;JzmwDTPtq7x%yTxsP#*l6v^bDZt*?6m- zZdKQV2bZ|Cl<+j&aF&;`cj z(<3Gm`wZl%2hF+CCRm(K8C4OoB{z$aV#@shk&wHVVhWG0=e?P`tPzxKC3`*0a=Easic&jl z{_|ZR$F@_X$vCN3iH99WQgZt}JmK8D1jk(VhaqAq(lJ_whF?x^o6PX+E>Batjzm#r zhKJfm@-;)ogqtB{J~R!W=cOq))&zPep$OetH}-V((*$^3pWUUGlLz=+-oKMnp^ZE0 zj*PrYkS9m28IQs!2%u+kUaNU-o9ItmB{gY!y^)Am?5AbCPPr16P?_UUT8dxGu_9i{ zY3wu>@#*B|cHQ9t|GezW=MEXxhlLFcg|CKOYt5F~1Mmdv$-V>L1J-M^2BXJuKKd99 zr14sQ9OHKhRg&TAeXBL&4JrhL*)r3zkHdk5Ggnc9dGV2WubZF<$&THo{CIn!<#UB{ zxj18(lkBbOy5=QrsS^wjq2$Y}`I&p2mleJ!yf4Ui)!ws<#2?+^pFcoLVbWA?hbz3K z=^N7ELdl@tb!awL+muH?R#U?L^@FSw9sO0VZyr^9n1aw9*1<^YoW7kh(qUOa?GIN7 z&BL5~tS2PycK~iqOCT?qzybE~NtSzp}>w1w+o#Ir-AbP!MI%8_fQtk%c zY2Oewa;;)MiU^^ljZx&Wh*0@3o!}X)^9sP_ID?11zc1}L zDL|dBDlBx0$cz$6vP4t@FB<1FAw>|+FF$kAUAOHEO)9ez4I{bA_x6qkbC~?1~pdwXnTICm5K^RG*H*T9pwdCBmHV`#3BNSfmk+P2Ix?Ph=J% z=Q`c=kUDvu9H$!>#fbR@Wrr{)8$Wbw8{b^sj}DHrRnsIP%lMv1yFlygVwzIW$~>_`*w;M+_Xu&DytT9iy&~k!zdgpp(qjYYI5@r}BxZ69O^% zqg`K7L^2jguf<-&M6vEyw93~#5hpCYwN_9Ho26ye`D;n_4W+8SRKGgdu*K+^HQbRB zrfD`x6#S$)*xZk}UHLn5zRmxDI1@P{;Q{IWM5Im3jX!QTu0cbBSjqNf6>rQYa!kJ@ z8V7Z|Xw(~&y)j7pLZqLdIAM;AhUAr?#KCATy^KR5coq~yxPQ*z^%(>NI~)#$M2hfk zkgFR<3a49qLoVP18AH!)LC<@EmN4LW9;K5=mN01qGP5WA?WEbDmRK7^SvMLld;p(O z{4}{zM(nMoxBcK_6b;r?IIhLQfI_V$cI9GW}HiPd@<0`-J*~ z(UHpQ6EXg6g(VBqc;j$TKPAqbNG(Hipv!ymew9Qx0rKd)-*;9s{a~8_29*JW7M@r8 zOef1c`tnqd-#Rzo7H#!&H(%VTGENWukj1`49`tATT3FOJ)ULzo-M?K(E%S*aD*948 zds^hjW}D&)nS4Eq?5>ikg9+f~R_nr|6Mm#(R90`hL1Fj^Wxx^!ce_P9a5lus!39H$ z8;@FM^1k%LIw|U)IuViPl~y*QZuX|1FmZEV~))N_J$%BPeyW`VS=IL(!u4k6m!`h9P!3A>`$#dcuETh7@V`tswh zBRoMzO3nrT4_N)4^|X+A4EI0SWQN|n`~q9hQW8I*;qm*&Z0_;yI~$)%cpW$dpTrw7 z?N3{tF-W*{T9(KcgIn{KPtfJpQrc-z-G|z$I*NS;Rj$ea3mA2n^n%5Rmr3xGKgaJ% zkC*$P&`E@65_>k=qZi;{RY``*5*Aw;%Gi}Dm>(Ry`I|Um9Xk^Sq2r@TH7&Flu~ti* zIkXsdgLc?@EIIs#iF+y=K{Xee$d&_kIdg$#K+)X zRi#-jN){%ICbIODTB3O>N)~(rg_5N>`H_jZ0Q`ot+4r8id=?-RZaCVIpM$bxOXvDs zIi|>R<~mh3p4f9m{kdm<>WFeCR=Vp87G>W-5Lf_@y^8HQ8jLj1rrX*eXa0o`$J1$q zPGZn?=ZLo#?V*ZbWdC|&)*6B9Q&Xs(kQ8i(`q7Ny3#h>wt%8!Z+&fR{A%0uW1+C8D zqtopHEjvFCehbjr7Pgt}*kig4vi_-YRPEZM;$8ZXGb_z|q)ww=4%jnUUtV$BkSH!4 zpP#!x+kB*uD%^ErH3rrD{_!#~>I#E$KdaW;ZlFxFmY(?I1XW+NUQsDRV_=0=E<8<= z>Zsws_0UO|s%?O>*s;AjfO=vjgq;?JZpBn$Y2&H6`4bSRc17QekTUBiUzu>-%#Ma! zSx!kaF_l>}-W8LhkWE+J_MiuvUBzBmUi7hWjeBOB#SdeHPlXWttIOX8M&D{-M$$Cy zQGWhz`Iw3}FU49Ekzrg~UgR~^-l;%9csB$#Oxa}b34MFUbUg}fR)kzp=jFvUcV=LX^-)fu#lY>2FV~L{4f0HOf z#8Y73%S%sFrGX;5O$bBWe6X`EYhei^fR^f-A9-(|w-k+F)Il-vN#??sDH@=JnfZGA z@PH3ejn)Ffy-~3+b#_G1L)!%j3RIq_EyEj@U?9jK1@+AHi3`qej9Qww$)8C0 zO>YT*6hRqZQxt&)FAFjT$-A3EnM}$Fo*@J;ZEfsQ_!wKNhNY3ShesE3N2*}?W7&r~ z^NoE+;7F*xPo@YAtfUb}2HQ<4<@EJI0U6UpYxzcz+Zj^|^(m<$g^rrvKhM0wm?UV>|D4ZB^OA7RCL@fS4rC>N-y_4MoTZAS52(uj-lB7m?e*? zqxg7?WP?KKt@4a{s0uM*jJnoD5L1)7(}ZicYi}xw)swzcWqHTx zN-Gg*A4<7c$b{DX=RS&Vmf)w-K7QRPe4hgt*#ZldnM1;0&Wb|o2;?sy&!L;_{kHo| z^jqM2`@6>*;m4k~m;Jf6hb{EJ8$jd5t@%6GW0OJa%kuJa&sLP;%f-Vc%x&sx>?J?Z ze;VrIJLMA+dRnT-e-WnNnOu^<0Iz#I%lJ23VDnLS#QO~hU<_0-!wmL3x;XQC4gVxJ zkng5m`L$Sflu?#Us&klG*K`oH^`jiftVoYoqn zG;LFW$jyHhC1u3Y?IV0hDExpa?qr9^=8orT+T~85Dd>uh($&@f)&k2m#;z&Vl>BAc84V5JRyvg#`o^y_}ZVAT*O2QmW4^XGA_$#|gr9mO` zyt<%H{r>1)jMBQr6D_a_HOZ&^$ZUNQ4&ej1oJQ&ck^;iVKr0Sr1$oVwlA$HK)drEl z5f*}0JM>HpTMHaACH${;0_p;N_RXyH5cC=gws%k+xUo!ZF0KtA&5Bt`RJoH#ZiMc^ zjd}J^aiUpO{_J79P@s%cx3jb-`@LvqU5I7NWt_=75lN!){QXZ{3xZ(mS<#iz{oK6d zZ-YBj{`b(Nfv+}(aRL`xfY7+-t!%1WxYRtU^z{}HHyC3#g z^2Z%i{yz&a$VDbbO-*`!6!-2*l%G`Iu;z|9aA8j}6&$&?(h7yp3AdqaJ#mnZ#OEX1$!qdglN7F@84{QdnOn3_GyX<~tdsCOY zF-NypE~jdfj2fUxOJ|b56MK(}vpx`#CqA}n#gADyvf}uaXdwl8Sl7H&@5J@zg|o$I z2O^>XRPnMlZ4x~J ziUm3}lN9drUo}+mf*%&OY@IzJS^givUFu)m#M<64M28CAL?GzY!#>DQQ;aAyKFw>` zQ2A(WhfeLQ_B7Xcs0t*eUCJwvB?=Wfi|j^A>5*w9WRSCe%4ZS~6?46QyD$FX9(M1; zRgy4@t7um+YjYR2)RW;na<8XhJB-x5!SVOflb(#k2fYKQys;iKjeFu7ot<8 diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 71716ae5..28dcd5c4 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -251,6 +251,8 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCLifecycle", "DropQueued", "NoOrphanRequest"), ("MCLifecycle", "EarlyNotify", "ExitAfterMessages"), ("MCLifecycle", "SplitDeathPoint", "NoEdgeToDead"), + ("MCLifecycle", "PublishBeforeDischarge", "AnsweredBeforePublish"), + ("MCLifecycle", "NoSignalFence", "SignalsBeforeDeath"), ] # Liveness controls run on a smaller instance of MCLifecycleLive. @@ -275,6 +277,8 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCDeferredReplies", "LeakDownstream", "DownstreamOwned"), ("MCDeferredReplies", "EarlyClose", "NoEarlyClose"), ("MCDeferredReplies", "EarlyStopped", "StoppedAtDeath"), + ("MCDeferredReplies", "NoReplyFence", "NoReplyAfterDeath"), + ("MCDeferredReplies", "CallbackBeforeDischarge", "WokenAfterDischarge"), ("MCDeferredJobs", "NoInstanceCheck", "NoLateResult"), ("MCDeferredJobs", "NoCancelFence", "NoStaleAfterCancel"), ("MCDeferredJobs", "DoubleTerminal", "AtMostOneTerminal"), @@ -286,6 +290,8 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCDeferredJobsEmits", "NoEmitCredit", "EmitCreditsBounded"), ("MCDeferredJobsHeld", "EarlyCredit", "HeldCounted"), ("MCDeferredJobsHeld", "HeldDeadlineLost", "DeadlineArmedWhileHeld"), + ("MCDeferredJobsHeld", "ExitKeepsHeld", "ExitCancelsJobs"), + ("MCDeferredJobs", "NoAdmissionFence", "ExitCancelsJobs"), ] @@ -500,6 +506,8 @@ def fail_instead_of_retry(events): # Needs a second generation within one round (MaxGen 3). model_fault("MCCohortsAllSupersession", "InPlaceRetry", args.tlc) model_fault("MCCohortsAll", "FirstStartRetry", args.tlc, invariant="Safety") + # Rank 5f: v0.7 charged every rebuilt child of a round. + named_fault("MCCohortsAll", "PerChildCharge", "RoundCharged", args.tlc) for label, pattern, replacement in SUPERVISION_S1_CONTROLS: spec_mutation("MCSupervision", "Supervision.tla", label, pattern, replacement, "RestartFollowsPolicy", args.tlc) From c768cccc32028601864a24475ec22af7fb293e4d Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:42:26 -0300 Subject: [PATCH 153/208] feat(monitor): demonitor flushes a queued Down, as OTP demonitor(Ref, [flush]) (#247) 0.8 already fenced enqueueing (the waiter enqueues the Down in the same critical section that consumes the table entry), but a Down already in the mailbox was still handled after demonitor returned. Now, in both modes, once demonitor returns no Down of that monitor is enqueued or handled. A monitor's table entry outlives delivery: delivery (and a late NoProc) marks it Queued, and the Down travels in a private DownEnvelope whose dispatch takes the entry under the table lock and runs Handler only if it was still queued. demonitor removes the entry in either state, so a queued Down is then skipped. A waiter's registration drop removes only an Active entry. Ordinary messages pay nothing; a framework Down pays one table lock at dispatch; a Down that user code sends is an ordinary message. (A blanket Handler impl was rejected: it conflicts with generic handlers such as spawned_test's Mock.) Tests (each fails with a non-flushing demonitor): - src/{tasks,threads}/actor.rs::demonitor_flushes_a_queued_down and src/monitor.rs::tests::demonitor_flushes_a_queued_down (a late NoProc queued and demonitored in one turn is never handled; FIFO makes it exact); - tests/failpoints.rs::{demonitor_flushes_a_down_in_dispatch, native::demonitor_flushes_a_down_in_dispatch}: new site monitor.down_before_dispatch pauses a death-path Down in dispatch while demonitor returns on another thread (a cross-thread window the simulator cannot reach); - link_fence_loom: demonitor_fences_the_down now also checks the flush, and demonitor_flushes_a_down_in_dispatch explores a dispatch racing demonitor. Model: Lifecycle.tla's Demonitor is also enabled while the monitor's Down is in the observer's mailbox, and dispatching a cancelled Down has no effect; the NoDemonitorFlush control (the handler runs anyway) violates CancelFenced. MCLifecycle 12,683,151 / MCLifecycleFence 1,174,961 / MCLifecycleLive 457,444 distinct states, no error; 75/75 traces and all negative controls pass. semantics.md X2 states the guarantee; X5, GUIDE, otp-exit-signals E6, TESTING (failpoint row), MIGRATION and CHANGELOG updated. --- CHANGELOG.md | 6 + concurrency/src/failpoints.rs | 8 ++ concurrency/src/monitor.rs | 42 +++++-- concurrency/src/monitor/fence.rs | 126 ++++++++++++++++----- concurrency/src/tasks/actor.rs | 160 +++++++++++++++++++++++---- concurrency/src/threads/actor.rs | 135 +++++++++++++++++++--- concurrency/tests/failpoints.rs | 143 ++++++++++++++++++++++++ concurrency/tests/link_fence_loom.rs | 37 ++++++- docs/GUIDE.md | 9 +- docs/MIGRATION.md | 5 + docs/TESTING.md | 5 +- docs/design/otp-exit-signals.md | 9 +- docs/design/semantics.md | 22 +++- verification/tla/Lifecycle.tla | 14 ++- verification/tla/negative_checks.py | 1 + 15 files changed, 636 insertions(+), 86 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89d98fcb..bdd3ac88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +- **Behaviour (0.8)**: `demonitor` flushes, as OTP's `demonitor(Ref, [flush])`: + once it returns, in both modes, no `Down` of that monitor is enqueued or + handled, including one already in the mailbox. Framework `Down`s travel in + a private envelope that checks the monitor table at dispatch, so ordinary + messages pay nothing and a `Down` user code sends is unaffected. New + failpoint site `monitor.down_before_dispatch` (#247). - Fix: `PendingReplies::drain` yields live tokens in insertion order. It walked a `HashMap` with a per-process `RandomState`, so deferred replies were answered in a different order in each process, and a same-seed diff --git a/concurrency/src/failpoints.rs b/concurrency/src/failpoints.rs index 65532f54..59009320 100644 --- a/concurrency/src/failpoints.rs +++ b/concurrency/src/failpoints.rs @@ -71,6 +71,14 @@ pub const SITES: &[Site] = &[ Delay models slow cleanup (shutdown deadlines); panic models a \ failing `stopped()`.", }, + Site { + name: "monitor.down_before_dispatch", + has_error: false, + description: "A monitor's `Down` dequeued by its observer (tasks and threads), \ + before the flush check. Delay lets a `demonitor` on another thread \ + return while the `Down` is in dispatch: it must not reach \ + `Handler` (OTP `demonitor(Ref, [flush])`).", + }, Site { name: "child.factory", has_error: true, diff --git a/concurrency/src/monitor.rs b/concurrency/src/monitor.rs index 1e0a8662..a022e17d 100644 --- a/concurrency/src/monitor.rs +++ b/concurrency/src/monitor.rs @@ -68,14 +68,11 @@ pub(crate) type MonitorTable = pub(crate) struct MonitorCleanup(pub MonitorTable); impl Drop for MonitorCleanup { fn drop(&mut self) { - let monitors = fence::close(&self.0); - if let Some(monitors) = monitors { - // HashMap order is per-process random; under simulation, wake - // waiters in monitor-creation order so the schedule is seeded. - #[cfg(spawned_sim)] - let monitors: std::collections::BTreeMap = - monitors.into_iter().map(|(r, t)| (r.0, t)).collect(); - for token in monitors.into_values() { + if let Some(mut monitors) = fence::close(&self.0) { + // HashMap order is per-process random: wake waiters in + // monitor-creation order, so a simulated schedule is seeded. + monitors.sort_unstable_by_key(|(r, _)| r.0); + for (_, token) in monitors { token.cancel(); } } @@ -108,8 +105,9 @@ mod tests { let table: MonitorTable = Default::default(); let (mine, other) = (MonitorRef::next(), MonitorRef::next()); let mut entries = std::collections::HashMap::new(); - entries.insert(mine, spawned_rt::sync::CancellationToken::new()); - entries.insert(other, spawned_rt::sync::CancellationToken::new()); + let token = spawned_rt::sync::CancellationToken::new; + entries.insert(mine, fence::Entry::Active(token())); + entries.insert(other, fence::Entry::Active(token())); *table.lock().unwrap() = Some(entries); drop(MonitorRegistration(table.clone(), mine)); let entries = table.lock().unwrap(); @@ -118,6 +116,30 @@ mod tests { assert!(entries.contains_key(&other)); } + /// A finished waiter's registration drop leaves its queued `Down` for + /// dispatch; `demonitor` flushes it, and dispatch then skips it. + #[test] + fn demonitor_flushes_a_queued_down() { + let table: MonitorTable = std::sync::Arc::new(fence::open()); + for flush in [false, true] { + let monitor = MonitorRef::next(); + let token = spawned_rt::sync::CancellationToken::new(); + assert_eq!( + fence::register(&table, monitor, token, || false, || false, || {}, || {}), + fence::Registration::Active + ); + let mut enqueued = false; + assert!(fence::deliver(&table, &monitor, || enqueued = true)); + assert!(enqueued); + drop(MonitorRegistration(table.clone(), monitor)); + if flush { + assert!(fence::demonitor(&table, &monitor, || {}).is_none()); + } + assert_eq!(fence::take_queued(&table, &monitor), !flush); + assert!(!fence::take_queued(&table, &monitor), "taken once"); + } + } + #[test] fn registration_dropped_after_cleanup_is_harmless() { let table: MonitorTable = Default::default(); diff --git a/concurrency/src/monitor/fence.rs b/concurrency/src/monitor/fence.rs index 9b837033..f7bbaa27 100644 --- a/concurrency/src/monitor/fence.rs +++ b/concurrency/src/monitor/fence.rs @@ -1,7 +1,8 @@ //! The observer-side monitor table protocol (docs/design/otp-exit-signals.md //! E4, E6): registration linearised against the target's death point, the -//! `Down` enqueued in the same critical section that removes the entry, the -//! `demonitor` fence and the observer's own close. +//! `Down` enqueued in the same critical section that marks the entry +//! queued, the `demonitor` fence and flush (OTP `demonitor(Ref, [flush])`) +//! and the observer's own close. //! //! Shared with the Loom model (`tests/link_fence_loom.rs`), which supplies //! the synchronization types through `super::sync`. The callbacks run under @@ -13,19 +14,36 @@ use super::sync::{Mutex, MutexGuard}; use std::collections::HashMap; use std::hash::Hash; +/// One monitor in its observer's table. +#[derive(Debug)] +pub(crate) enum Entry { + /// Registered: its waiter delivers the `Down` at the target's + /// publication. + Active(V), + /// Its `Down` is in the observer's mailbox, not yet handled: dispatch + /// takes it ([`take_queued`]); `demonitor` flushes it. + Queued, +} + /// An observer's monitors: `None` once the observer's exit closed it. -pub(crate) type Entries = Mutex>>; +pub(crate) type Entries = Mutex>>>; -fn lock(table: &Entries) -> MutexGuard<'_, Option>> { +fn lock(table: &Entries) -> MutexGuard<'_, Option>>> { table.lock().unwrap_or_else(|p| p.into_inner()) } +/// An open, empty table. +pub(crate) fn open() -> Entries { + Mutex::new(Some(HashMap::new())) +} + /// The outcome of a `monitor` call. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum Registration { /// The observer is exiting or closed: no entry, no notice. Inert, - /// The target was past D: the `NoProc` notice was enqueued, no entry. + /// The target was past D: the `NoProc` notice was enqueued (its entry is + /// `Queued`). Late, /// The entry is installed; its waiter delivers the `Down` at publication. Active, @@ -33,8 +51,8 @@ pub(crate) enum Registration { /// `monitor`: under the observer's lock, inert once `observer_exiting` or the /// table is closed; past the target's D (`target_dead`, which reads the -/// target's exit table) enqueue the late notice (`late`); otherwise install -/// the entry and run `active`. +/// target's exit table) enqueue the late notice (`late`) and mark it queued; +/// otherwise install the entry and run `active`. pub(crate) fn register( table: &Entries, key: K, @@ -53,49 +71,105 @@ pub(crate) fn register( }; if target_dead() { late(); + entries.insert(key, Entry::Queued); return Registration::Late; } - entries.insert(key, value); + entries.insert(key, Entry::Active(value)); active(); Registration::Active } -/// The waiter, after the target's exit is published: remove the entry and -/// enqueue the `Down` (`deliver`) in one critical section, so once -/// [`remove`] (a `demonitor`) returns no `Down` of this monitor is enqueued. -/// Returns whether it was delivered. +/// The waiter, after the target's exit is published: mark the entry queued +/// and enqueue the `Down` (`deliver`) in one critical section, so once +/// [`demonitor`] returns no `Down` of this monitor is enqueued, and one +/// already enqueued is flushed. Returns whether it was delivered. pub(crate) fn deliver( table: &Entries, key: &K, deliver: impl FnOnce(), ) -> bool { let mut guard = lock(table); - let removed = guard.as_mut().and_then(|entries| entries.remove(key)); - if removed.is_some() { + let entry = guard.as_mut().and_then(|entries| entries.get_mut(key)); + let active = match entry { + Some(entry @ Entry::Active(_)) => { + *entry = Entry::Queued; + true + } + _ => false, + }; + if active { deliver(); } drop(guard); - removed.is_some() + active +} + +/// Dispatch of a framework `Down`: whether it is still owed (its entry is +/// `Queued`; taken now). `false` once `demonitor` flushed it. +pub(crate) fn take_queued(table: &Entries, key: &K) -> bool { + let mut guard = lock(table); + let Some(entries) = guard.as_mut() else { + return false; + }; + if matches!(entries.get(key), Some(Entry::Queued)) { + entries.remove(key); + true + } else { + false + } } -/// `demonitor` (and a waiter that is dropped unpolled): remove the entry -/// under the lock, running `removed` there when it was present. The caller -/// wakes the waiter with the returned value after the lock is released. +/// A waiter that is dropped (unpolled, rejected or finished): remove an +/// `Active` entry under the lock, running `removed` there when it was +/// present. A `Queued` entry stays for its dispatch. The caller wakes the +/// waiter with the returned value after the lock is released. pub(crate) fn remove( table: &Entries, key: &K, removed: impl FnOnce(), ) -> Option { let mut guard = lock(table); - let value = guard.as_mut().and_then(|entries| entries.remove(key)); - if value.is_some() { - removed(); + let entries = guard.as_mut()?; + if !matches!(entries.get(key), Some(Entry::Active(_))) { + return None; + } + removed(); + match entries.remove(key) { + Some(Entry::Active(value)) => Some(value), + _ => None, + } +} + +/// `demonitor` with OTP's `flush`: under the lock, remove the entry +/// whatever its state, running `removed` there when it was present. An +/// `Active` entry's waiter (returned) is woken by the caller after the lock +/// is released; a `Queued` one's `Down`, already in the mailbox, will be +/// skipped at dispatch. Once this returns, no `Down` of the monitor is +/// enqueued or handled. +pub(crate) fn demonitor( + table: &Entries, + key: &K, + removed: impl FnOnce(), +) -> Option { + let mut guard = lock(table); + let entry = guard.as_mut().and_then(|entries| entries.remove(key))?; + removed(); + match entry { + Entry::Active(value) => Some(value), + Entry::Queued => None, } - value } -/// The observer's exit: close the table and take every entry, whose waiters -/// the caller wakes after the lock is released. -pub(crate) fn close(table: &Entries) -> Option> { - lock(table).take() +/// The observer's exit: close the table and take every active entry, whose +/// waiters the caller wakes after the lock is released. +pub(crate) fn close(table: &Entries) -> Option> { + lock(table).take().map(|entries| { + entries + .into_iter() + .filter_map(|(key, entry)| match entry { + Entry::Active(value) => Some((key, value)), + Entry::Queued => None, + }) + .collect() + }) } diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index eff91c43..cf36b71f 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -18,14 +18,10 @@ use spawned_rt::{ threads, }; use std::sync::atomic::Ordering; +#[cfg(test)] +use std::{collections::HashMap, sync::Mutex}; use std::{ - collections::HashMap, - fmt::Debug, - future::Future, - panic::AssertUnwindSafe, - pin::Pin, - sync::{Arc, Mutex}, - time::Duration, + fmt::Debug, future::Future, panic::AssertUnwindSafe, pin::Pin, sync::Arc, time::Duration, }; pub use crate::response::DEFAULT_REQUEST_TIMEOUT; @@ -260,6 +256,41 @@ where } } +impl> Envelope for DownEnvelope { + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "down" + } + #[cfg(any(feature = "diagnostics", spawned_sim))] + fn message_type(&self) -> &'static str { + std::any::type_name::() + } + #[cfg(feature = "metrics")] + fn observation(&mut self) -> Option { + self.observation.take() + } + #[cfg(feature = "metrics")] + fn reject(mut self: Box) { + if let Some(observation) = self.observation.take() { + observation.reject(); + } + } + fn handle<'a>( + self: Box, + actor: &'a mut A, + ctx: &'a Context, + ) -> Pin + Send + 'a>> { + let down = self.down; + Box::pin(async move { + fail_point!("monitor.down_before_dispatch", async); + if crate::monitor::fence::take_queued(&ctx.shared.monitors, &down.monitor_ref) { + let reply = PendingReply::new(None, &ctx.shared.replies); + actor.handle(down, reply, ctx).await; + } + }) + } +} + /// Admit a message. Past the death point (the registry is sealed, or the /// mailbox is gone) it reports `NoProc`; before it, including while the /// actor exits, it is queued, and a request is answered at the death point @@ -281,6 +312,41 @@ where #[cfg(feature = "metrics")] observation: Some(_mailbox.enqueued()), }); + admit::(sender, replies, envelope) +} + +/// A monitor's `Down`, enqueued by the framework (the waiter at the target's +/// publication, or `monitor` past the target's death point). Its dispatch +/// runs `Handler` only while the monitor's table entry is still +/// queued: a `demonitor` that returned first flushed it (OTP +/// `demonitor(Ref, [flush])`). A `Down` user code sends is an ordinary +/// message. +struct DownEnvelope { + down: Down, + #[cfg(feature = "metrics")] + observation: Option, +} + +fn enqueue_down>( + sender: &mpsc::Sender + Send>>, + _mailbox: &crate::instrumentation::Mailbox, + replies: &Registry, + down: Down, +) -> Result<(), ActorError> { + let envelope: Box + Send> = Box::new(DownEnvelope { + down, + #[cfg(feature = "metrics")] + observation: Some(_mailbox.enqueued()), + }); + admit::(sender, replies, envelope) +} + +/// Admission of a built envelope (`M` names its kind for the trace). +fn admit( + sender: &mpsc::Sender + Send>>, + replies: &Registry, + envelope: Box + Send>, +) -> Result<(), ActorError> { // Admission and its trace event are one step of the lifecycle trace. let owner = replies.owner().raw(); crate::lifecycle_trace::life_section(owner, || { @@ -774,10 +840,15 @@ impl Context { .k("down") .o("late") }); - let _ = self.send(Down { - monitor_ref, - reason: ExitReason::NoProc, - }); + let _ = enqueue_down( + &self.shared.sender, + &self.shared.mailbox, + &self.shared.replies, + Down { + monitor_ref, + reason: ExitReason::NoProc, + }, + ); }, || crate::lifecycle_trace::life_emit(observer, || event("active")), ) == crate::monitor::fence::Registration::Active @@ -815,10 +886,15 @@ impl Context { .k("down") .o("death") }); - let _ = actor_ref.send(Down { - monitor_ref, - reason, - }); + let _ = enqueue_down( + &actor_ref.shared.sender, + &actor_ref.shared.mailbox, + &actor_ref.shared.replies, + Down { + monitor_ref, + reason, + }, + ); }); }); }); @@ -828,12 +904,14 @@ impl Context { /// Cancel a previously-set monitor. /// - /// A fence (OTP `demonitor/1`): once it returns, no `Down` of this - /// monitor is enqueued; one already queued stays. + /// A fence with OTP's `flush` (`demonitor(Ref, [flush])`): once it + /// returns, no `Down` of this monitor is enqueued or handled; one already + /// in the mailbox is skipped. Demonitoring an unknown, delivered or + /// already cancelled monitor does nothing. pub fn demonitor(&self, monitor_ref: MonitorRef) { let observer = self.shared.id.raw(); let active = crate::lifecycle_trace::life_section(observer, || { - crate::monitor::fence::remove(&self.shared.monitors, &monitor_ref, || { + crate::monitor::fence::demonitor(&self.shared.monitors, &monitor_ref, || { crate::lifecycle_trace::life_emit(observer, || { crate::lifecycle_trace::LifeEvent::new("Demonitor", observer) .m(monitor_ref.raw()) @@ -1389,7 +1467,7 @@ impl ActorRef { .map(|s| s.token.clone()) .unwrap_or_default(); let (completion_tx, completion_rx) = watch::channel(None); - let monitors: MonitorTable = Arc::new(Mutex::new(Some(HashMap::new()))); + let monitors: MonitorTable = Arc::new(crate::monitor::fence::open()); let trap_exit = new_trap_exit_flag(); let links = new_link_table(); let linked_reason = managed @@ -2048,7 +2126,7 @@ fn test_context( let trap = new_trap_exit_flag(); let links = new_link_table(); let request = new_linked_exit_reason(); - let monitors: MonitorTable = Arc::new(Mutex::new(Some(HashMap::new()))); + let monitors: MonitorTable = Arc::new(crate::monitor::fence::open()); let signal = signal_target::(id, None, tx, trap.clone(), request.clone(), token.clone()); let ctx = Context { shared: Arc::new(ActorShared { @@ -3041,6 +3119,11 @@ mod tests { impl Message for CallDemonitor { type Result = (); } + /// Monitor, then demonitor in the same turn. + struct MonitorThenDemonitor(crate::ChildHandle); + impl Message for MonitorThenDemonitor { + type Result = (); + } impl Actor for Watcher {} @@ -3055,6 +3138,19 @@ mod tests { } } + impl Handler for Watcher { + async fn handle( + &mut self, + msg: MonitorThenDemonitor, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + let monitor = ctx.monitor(&msg.0); + ctx.demonitor(monitor); + reply.reply(()); + } + } + impl Handler for Watcher { async fn handle( &mut self, @@ -3120,6 +3216,30 @@ mod tests { }); } + /// OTP `demonitor(Ref, [flush])`: a `Down` already in the mailbox + /// (here the `NoProc` a monitor of a dead target queues at once) is + /// never handled once `demonitor` returned. + #[test] + pub fn demonitor_flushes_a_queued_down() { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + target.request(StopCounter).await.unwrap(); + target.join().await; + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); + watcher + .request(MonitorThenDemonitor(target.child_handle())) + .await + .unwrap(); + // FIFO: the queued Down is dispatched before this request. + let downs = watcher.request(GetDowns).await.unwrap(); + assert!(downs.is_empty(), "a flushed Down was handled: {downs:?}"); + }); + } + #[test] pub fn monitor_already_dead_actor_delivers_down_immediately() { let runtime = rt::Runtime::new().unwrap(); diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 718ef2fc..b979ad1d 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -1,10 +1,11 @@ use crate::mailbox::threads as mpsc; use spawned_rt::threads::{self as rt, CancellationToken}; +#[cfg(test)] +use std::{collections::HashMap, sync::Mutex}; use std::{ - collections::HashMap, fmt::Debug, panic::{catch_unwind, AssertUnwindSafe}, - sync::{Arc, Mutex}, + sync::Arc, time::Duration, }; @@ -178,6 +179,30 @@ where } } +impl> Envelope for DownEnvelope { + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "down" + } + #[cfg(feature = "metrics")] + fn observation(&mut self) -> Option { + self.observation.take() + } + #[cfg(feature = "metrics")] + fn reject(mut self: Box) { + if let Some(observation) = self.observation.take() { + observation.reject(); + } + } + fn handle(self: Box, actor: &mut A, ctx: &Context) { + fail_point!("monitor.down_before_dispatch"); + if crate::monitor::fence::take_queued(&ctx.monitors, &self.down.monitor_ref) { + let reply = PendingReply::new(None, &ctx.replies); + actor.handle(self.down, reply, ctx); + } + } +} + /// Admit a message. Past the death point (the registry is sealed, or the /// mailbox is gone) it reports `NoProc`; before it, including while the /// actor exits, it is queued, and a request is answered at the death point @@ -199,6 +224,41 @@ where #[cfg(feature = "metrics")] observation: Some(_mailbox.enqueued()), }); + admit::(sender, replies, envelope) +} + +/// A monitor's `Down`, enqueued by the framework (the waiter at the target's +/// publication, or `monitor` past the target's death point). Its dispatch +/// runs `Handler` only while the monitor's table entry is still +/// queued: a `demonitor` that returned first flushed it (OTP +/// `demonitor(Ref, [flush])`). A `Down` user code sends is an ordinary +/// message. +struct DownEnvelope { + down: Down, + #[cfg(feature = "metrics")] + observation: Option, +} + +fn enqueue_down>( + sender: &mpsc::Sender>, + _mailbox: &crate::instrumentation::Mailbox, + replies: &Registry, + down: Down, +) -> Result<(), ActorError> { + let envelope: Box + Send> = Box::new(DownEnvelope { + down, + #[cfg(feature = "metrics")] + observation: Some(_mailbox.enqueued()), + }); + admit::(sender, replies, envelope) +} + +/// Admission of a built envelope (`M` names its kind for the trace). +fn admit( + sender: &mpsc::Sender>, + replies: &Registry, + envelope: Box + Send>, +) -> Result<(), ActorError> { // Admission and its trace event are one step of the lifecycle trace. let owner = replies.owner().raw(); crate::lifecycle_trace::life_section(owner, || { @@ -655,10 +715,15 @@ impl Context { .k("down") .o("late") }); - let _ = self.send(Down { - monitor_ref, - reason: ExitReason::NoProc, - }); + let _ = enqueue_down( + &self.sender, + &self.mailbox, + &self.replies, + Down { + monitor_ref, + reason: ExitReason::NoProc, + }, + ); }, || crate::lifecycle_trace::life_emit(observer, || event("active")), ) == crate::monitor::fence::Registration::Active @@ -699,10 +764,15 @@ impl Context { .k("down") .o("death") }); - let _ = actor_ref.send(Down { - monitor_ref, - reason, - }); + let _ = enqueue_down( + &actor_ref.sender, + &actor_ref.mailbox, + &actor_ref.replies, + Down { + monitor_ref, + reason, + }, + ); }); }); }); @@ -712,12 +782,14 @@ impl Context { /// Cancel a previously-set monitor. /// - /// A fence (OTP `demonitor/1`): once it returns, no `Down` of this - /// monitor is enqueued; one already queued stays. + /// A fence with OTP's `flush` (`demonitor(Ref, [flush])`): once it + /// returns, no `Down` of this monitor is enqueued or handled; one already + /// in the mailbox is skipped. Demonitoring an unknown, delivered or + /// already cancelled monitor does nothing. pub fn demonitor(&self, monitor_ref: MonitorRef) { let observer = self.id.raw(); let active = crate::lifecycle_trace::life_section(observer, || { - crate::monitor::fence::remove(&self.monitors, &monitor_ref, || { + crate::monitor::fence::demonitor(&self.monitors, &monitor_ref, || { crate::lifecycle_trace::life_emit(observer, || { crate::lifecycle_trace::LifeEvent::new("Demonitor", observer) .m(monitor_ref.raw()) @@ -1130,7 +1202,7 @@ impl ActorRef { register_stop_wakeup(&cancellation_token, tx.clone()); let (completion_tx, completion) = spawned_rt::sync::watch::channel(None); let id = ActorId::next(); - let monitors: MonitorTable = Arc::new(Mutex::new(Some(HashMap::new()))); + let monitors: MonitorTable = Arc::new(crate::monitor::fence::open()); let trap_exit = new_trap_exit_flag(); let links = new_link_table(); let linked_reason = new_linked_exit_reason(); @@ -1837,6 +1909,11 @@ mod tests { impl Message for CallDemonitor { type Result = (); } + /// Monitor, then demonitor in the same turn. + struct MonitorThenDemonitor(crate::ChildHandle); + impl Message for MonitorThenDemonitor { + type Result = (); + } impl Actor for Watcher {} @@ -1851,6 +1928,19 @@ mod tests { } } + impl Handler for Watcher { + fn handle( + &mut self, + msg: MonitorThenDemonitor, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + let monitor = ctx.monitor(&msg.0); + ctx.demonitor(monitor); + reply.reply(()); + } + } + impl Handler for Watcher { fn handle( &mut self, @@ -1913,6 +2003,23 @@ mod tests { assert!(matches!(downs[0].reason, ExitReason::Normal)); } + /// OTP `demonitor(Ref, [flush])`: a `Down` already in the mailbox + /// (here the `NoProc` a monitor of a dead target queues at once) is + /// never handled once `demonitor` returned. + #[test] + fn demonitor_flushes_a_queued_down() { + let target = Counter { count: 0 }.start(); + target.request(StopCounter).unwrap(); + target.join(); + let watcher = make_watcher(); + watcher + .request(MonitorThenDemonitor(target.child_handle())) + .unwrap(); + // FIFO: the queued Down is dispatched before this request. + let downs = watcher.request(GetDowns).unwrap(); + assert!(downs.is_empty(), "a flushed Down was handled: {downs:?}"); + } + #[test] fn monitor_already_dead_actor_delivers_down_immediately() { let target = Counter { count: 0 }.start(); diff --git a/concurrency/tests/failpoints.rs b/concurrency/tests/failpoints.rs index dbf17be8..ab2063be 100644 --- a/concurrency/tests/failpoints.rs +++ b/concurrency/tests/failpoints.rs @@ -1090,6 +1090,82 @@ async fn an_exit_signal_is_not_overtaken_by_its_senders_death_notice() { recorder.join().await; } +// ---------------------------------- monitor.down_before_dispatch (flush) + +struct WatchDown(ChildHandle); +impl Message for WatchDown { + type Result = spawned_concurrency::MonitorRef; +} + +/// Monitors on request and counts the `Down`s its handler runs. +struct Observer(Arc); +impl Actor for Observer {} +impl Handler for Observer { + async fn handle( + &mut self, + msg: WatchDown, + reply: spawned_concurrency::PendingReply<'_, spawned_concurrency::MonitorRef>, + ctx: &Context, + ) { + reply.reply(ctx.monitor(&msg.0)); + } +} +impl Handler for Observer { + async fn handle( + &mut self, + _: Down, + _: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} +impl Handler for Observer { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } +} +struct Quiet; +impl Actor for Quiet {} + +/// OTP `demonitor(Ref, [flush])`: a `demonitor` on another thread returns +/// while the observer is dispatching the monitor's `Down` (paused before the +/// flush check); the `Down` never reaches `Handler`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn demonitor_flushes_a_down_in_dispatch() { + let _scenario = FailScenario::setup(); + let handled = Arc::new(AtomicUsize::new(0)); + let observer = Observer(handled.clone()).start(); + let target = Quiet.start(); + let monitor = observer + .request(WatchDown(target.child_handle())) + .await + .unwrap(); + failpoints::configure( + "monitor.down_before_dispatch", + FailPoint::new(Action::Delay(Duration::from_millis(300))).times(1), + ) + .unwrap(); + target.context().stop(); + target.join().await; + reached("monitor.down_before_dispatch", 1).await; + observer.context().demonitor(monitor); + // FIFO, one turn at a time: the Down's dispatch finished before this. + observer.request(Ping).await.unwrap(); + assert_eq!( + handled.load(Ordering::SeqCst), + 0, + "a Down was handled after demonitor returned" + ); + observer.context().stop(); + observer.join().await; +} + mod native { use super::*; use spawned_concurrency::threads::{ @@ -1305,4 +1381,71 @@ mod native { std::thread::sleep(Duration::from_millis(2)); } } + + struct NativeObserver(Arc); + impl NativeActor for NativeObserver {} + impl NativeHandler for NativeObserver { + fn handle( + &mut self, + msg: WatchDown, + reply: spawned_concurrency::reply::PendingReply<'_, spawned_concurrency::MonitorRef>, + ctx: &NativeContext, + ) { + reply.reply(ctx.monitor(&msg.0)); + } + } + impl NativeHandler for NativeObserver { + fn handle( + &mut self, + _: Down, + _: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &NativeContext, + ) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + impl NativeHandler for NativeObserver { + fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &NativeContext, + ) { + reply.reply(()); + } + } + struct NativeQuiet; + impl NativeActor for NativeQuiet {} + + /// Threads mode: a `demonitor` on another thread returns while the + /// observer dispatches the monitor's `Down`; it is never handled. + #[test] + fn demonitor_flushes_a_down_in_dispatch() { + let _scenario = FailScenario::setup(); + let handled = Arc::new(AtomicUsize::new(0)); + let observer = NativeObserver(handled.clone()).start(); + let target = NativeQuiet.start(); + let monitor = observer.request(WatchDown(target.child_handle())).unwrap(); + failpoints::configure( + "monitor.down_before_dispatch", + FailPoint::new(Action::Delay(Duration::from_millis(300))).times(1), + ) + .unwrap(); + target.context().stop(); + target.join(); + let began = Instant::now(); + while failpoints::stats("monitor.down_before_dispatch").hits < 1 { + assert!(began.elapsed() < LIMIT, "the Down was not dispatched"); + std::thread::sleep(Duration::from_millis(2)); + } + observer.context().demonitor(monitor); + observer.request(Ping).unwrap(); + assert_eq!( + handled.load(Ordering::SeqCst), + 0, + "a Down was handled after demonitor returned" + ); + observer.context().stop(); + observer.join(); + } } diff --git a/concurrency/tests/link_fence_loom.rs b/concurrency/tests/link_fence_loom.rs index a80075bf..acedd876 100644 --- a/concurrency/tests/link_fence_loom.rs +++ b/concurrency/tests/link_fence_loom.rs @@ -429,7 +429,8 @@ impl Observer { } /// `demonitor` racing the target's death and the waiter: once it returns no -/// `Down` is enqueued; if it found no entry, the `Down` was delivered. +/// `Down` is enqueued; if it found an active entry none was, otherwise the +/// `Down` was delivered and is flushed (OTP `demonitor(Ref, [flush])`). #[test] fn demonitor_fences_the_down() { model("demonitor_fences_the_down", || { @@ -440,7 +441,7 @@ fn demonitor_fences_the_down() { let target = target.clone(); spawn(move || target.die()) }; - let removed = fence::remove(&observer.monitors, &1, || {}); + let removed = fence::demonitor(&observer.monitors, &1, || {}); observer.fenced.store(true, Ordering::Release); if removed.is_some() { observer.cancelled.store(true, Ordering::Release); @@ -454,6 +455,38 @@ fn demonitor_fences_the_down() { vec![ExitReason::Panic] }; assert_eq!(observer.downs(), expected); + assert!( + !fence::take_queued(&observer.monitors, &1), + "a queued Down survived demonitor" + ); + }); +} + +/// The observer's dispatch of a delivered `Down` racing `demonitor` on +/// another thread (a cloned context): a dispatch that begins after +/// `demonitor` returned never takes the `Down`; one that took it first +/// handles it (it was received before the flush). +#[test] +fn demonitor_flushes_a_down_in_dispatch() { + model("demonitor_flushes_a_down_in_dispatch", || { + let (observer, target) = (Observer::new(), Target::new()); + assert_eq!(observer.register(&target), fence::Registration::Active); + target.die(); + let waiter = observer.waiter(&target); + assert!(waiter.join().unwrap(), "the Down was delivered"); + let dispatcher = { + let observer = observer.clone(); + spawn(move || { + let fenced = observer.fenced.load(Ordering::Acquire); + let took = fence::take_queued(&observer.monitors, &1); + assert!(!(fenced && took), "a Down handled after demonitor returned"); + }) + }; + assert!(fence::demonitor(&observer.monitors, &1, || {}).is_none()); + observer.fenced.store(true, Ordering::Release); + dispatcher.join().unwrap(); + // Either the dispatch took it first, or the flush removed it. + assert!(!fence::take_queued(&observer.monitors, &1)); }); } diff --git a/docs/GUIDE.md b/docs/GUIDE.md index a9ee8d33..8287bf7f 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -129,7 +129,7 @@ State that must be released on every exit, including kills and runtime drops, be | `ctx.actor_ref()` | Get an `ActorRef` from the context | | `ctx.id()` | This actor's `ActorId` | | `ctx.monitor(&child_handle)` | Receive a `Down` message when the target exits; returns a `MonitorRef` | -| `ctx.demonitor(monitor_ref)` | Cancel a monitor (a fence: no later `Down`) | +| `ctx.demonitor(monitor_ref)` | Cancel a monitor (a fence with OTP's `flush`: no `Down` afterwards, not even one already queued) | | `ctx.link(&child_handle)` / `ctx.unlink(&child_handle)` | Bidirectional exit propagation (see [Links, monitors and exit signals](#links-monitors-and-exit-signals)) | | `ctx.try_link(&child_handle)` | Link, or `Err(NoProc)` if the target is already gone | | `ctx.exit(&child_handle, reason)` | Send an exit signal from this actor (OTP `exit/2`) | @@ -976,9 +976,10 @@ death point. ### Unlink and demonitor are fences Once `ctx.unlink(&h)` returns, that link has no further effect on the caller; -once `ctx.demonitor(r)` returns, no `Down` of that monitor is queued. An -`Exit` or `Down` already in the mailbox stays. Linking again creates a new -link that gets its own notice. +once `ctx.demonitor(r)` returns, no `Down` of that monitor is queued or +handled: one already in the mailbox is dropped unhandled, as OTP's +`demonitor(Ref, [flush])`. An `Exit` already in the mailbox stays. Linking +again creates a new link that gets its own notice. ### Runtime drop diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index f1e61dc9..da4f1448 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -50,6 +50,11 @@ table is the full list). What to edit: - **`match` on `ExitReason`**: add a wildcard arm (`#[non_exhaustive]`, new `Killed` and `NoProc`). +- **`demonitor` flushes** (OTP `demonitor(Ref, [flush])`): a `Down` of that + monitor already in the mailbox is no longer handled. Code that demonitored + and then expected a racing `Down` (or tolerated one) needs no change; code + that relied on a queued `Down` arriving after `demonitor` must stop + demonitoring, or read `ChildHandle::exit_reason()`. - **`max_restarts` with OneForAll/RestForOne counts rounds** (rank 5f): one recovery round takes one grant, however many children it rebuilds (v0.7 charged each rebuilt child). A cohort supervisor now tolerates more diff --git a/docs/TESTING.md b/docs/TESTING.md index aa0b42cd..a53798af 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -825,8 +825,8 @@ input and, for random schedules, its tape: This is generated, model-checked scenario testing on real runtimes, not an exhaustive scheduler; the coverage-guided fuzzer above runs on the simulator only. It does not cover bounded mailboxes or supervisor restart policies. -Demonitor races permit an already-queued notification; tests use bounded -observation windows and do not claim exhaustive delivery verification. +Demonitor flushes a queued `Down` (X2); these campaigns still observe it +through bounded windows and do not claim exhaustive delivery verification. Do not interpret a passing campaign as proof of race freedom. ## Additional checks @@ -1296,6 +1296,7 @@ and prints the seed when a test fails. Configuration is process-global. | `actor.exit_before_deliver` | no | A cloned context's exit signal racing its sender's death (tasks and threads) | | `link.before_install` | no | Peer exiting while a link is installed (tasks and threads) | | `monitor.before_install` | no | Target exiting while a monitor is installed (tasks and threads) | +| `monitor.down_before_dispatch` | no | `demonitor` on another thread returning while the monitor's `Down` is in dispatch: it is flushed (tasks and threads) | `tests/failpoints.rs` drives every site and checks invariants through public APIs: restarts follow the budget (generation count equals injected failures diff --git a/docs/design/otp-exit-signals.md b/docs/design/otp-exit-signals.md index 11864150..e9c22e2a 100644 --- a/docs/design/otp-exit-signals.md +++ b/docs/design/otp-exit-signals.md @@ -37,7 +37,7 @@ maintainer settled every open question (see [Decisions](#decisions)). | E3 | a trapping gen_server that gets `'EXIT'` from its parent runs `terminate(Reason)` | supervisor shutdown cancels a trapping child directly; a `start_linked` parent's exit reaches `exit_received` | a signal from the actor's parent is queued and, when dispatched, runs normal termination; it never reaches `exit_received` | | E4 | `link`/`monitor` on a process that no longer exists give `noproc` | the target's real exit reason | `NoProc` after D | | E5 | exit reasons are any term | closed four-variant enum | `#[non_exhaustive]`, adds `Killed` and `NoProc` | -| E6 | `unlink`/`demonitor` stop every later effect of the edge | a notice already taken for delivery still arrives | fenced at the receiver; only a message already queued stays | +| E6 | `unlink`/`demonitor` stop every later effect of the edge (`demonitor(Ref, [flush])` also removes a queued `'DOWN'`) | a notice already taken for delivery still arrives | fenced at the receiver; an `Exit` already queued stays, a queued `Down` is flushed | | E7 | signals between a pair of processes keep their order | death notices follow earlier messages | unchanged for death notices; direct signals are per-sender FIFO with messages | | S1 | `start_link` aborts on any child start error; exits of started children wait until `init` returns | Permanent/Transient children retry a failed first start; a ready child's exit during startup is handled at once | any first-start failure aborts; ready children's exits are processed after the initial start completes | | S2 | `start_link` returns its error after the failed child exited and started children were terminated | `start()` can return `Err` before the failed child is `Failed` | `start()` returns `Err` after rollback (or its deadlines) | @@ -514,8 +514,13 @@ signal in flight"). pending, removes it, and enqueues the `Exit`/`Down` (or, for a non-trapping receiver, requests its exit) **inside that critical section**. - `unlink` and `demonitor` remove the record under the same lock, live or - pending. After they return nothing of that edge takes effect; a message + pending. After they return nothing of that edge takes effect; an `Exit` already queued stays. +- **Flush (0.8).** A monitor's record outlives its delivery: delivery marks + it *queued*, and the `Down`'s dispatch takes it (a private envelope, so no + user-sent `Down` is affected). `demonitor` removes a queued record too, so + the dispatch then skips the `Down`, as `demonitor(Ref, [flush])`. Ordinary + messages pay nothing; a `Down` pays one table lock at dispatch. - A late `monitor` or late `link` leaves a pending record for its `NoProc` notice, so `demonitor` or `unlink` cancels it, as OTP keeps the caller's link record while the `noproc` response is pending. diff --git a/docs/design/semantics.md b/docs/design/semantics.md index a269be64..654fe39a 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -325,15 +325,28 @@ Checked by: `Lifecycle!DownAtMostOnce`, `Lifecycle!NoProcIffLate`, `Lifecycle!Do [`otp_exit_signals::tasks_mode::registrations_after_death_report_noproc`][otp_exit_signals], [`cancellation_safety::monitor_and_link_setup_requests_are_cancel_safe`][cancellation_safety]. -**X2. Demonitor and late registration.** `demonitor` before the target exits -suppresses `Down`; a `Down` already queued may still arrive. A monitor +**X2. Demonitor and late registration.** `demonitor` flushes, as OTP's +`demonitor(Ref, [flush])`: once it returns, in both modes, no `Down` of that +monitor is enqueued or handled, including one already in the mailbox (the +late `NoProc` included); a `Down` whose dispatch began before it returned +was already received and is handled. A `Down` that user code sends itself is +an ordinary message. A monitor registered by an actor that is already stopping is inert (no waiter, no `Down`). Likewise `ctx.link`/`ctx.try_link` called through a `Context` whose actor's exit already began, or which is past its death point (for example from a helper that outlives the actor), is inert: no record, no notice, no `NoProc`, and `try_link` returns `Ok(())` (a dead process cannot call `link` in OTP; design otp-exit-signals.md, "Sender lifetime"). -Checked by: `src/*/actor.rs::demonitor_before_target_dies_suppresses_down`, +Checked by: `Lifecycle!CancelFenced` (TLC, `MCLifecycleFence`; the +`NoDemonitorFlush` control runs a flushed `Down`); +`src/*/actor.rs::{demonitor_before_target_dies_suppresses_down, demonitor_flushes_a_queued_down}`, +`src/monitor.rs::tests::demonitor_flushes_a_queued_down`; +[`link_fence_loom`][link_fence_loom]`::demonitor_flushes_a_down_in_dispatch` +(a dispatch racing `demonitor` on another thread); +[`failpoints`][failpoints]`::{demonitor_flushes_a_down_in_dispatch, native::demonitor_flushes_a_down_in_dispatch}` +(site `monitor.down_before_dispatch`: `demonitor` returns while the `Down` +is in dispatch; the window is cross-thread, so a failpoint, not the +simulator, reaches it); `link::tests::link_from_an_exiting_or_dead_caller_is_inert`, [`otp_exit_signals::tasks_mode::link_from_a_dead_actor_is_inert`][otp_exit_signals], `closed_monitor_table_rejects_late_registration`, [`leaks::monitor_waiters_released_on_demonitor_owner_exit_and_target_exit`][leaks]. @@ -388,7 +401,8 @@ relationship and delivers one `NoProc` notice: a trapping caller gets gets exactly one notice, the real reason or `NoProc`. Repeating a link (including one whose notice is pending) and linking to oneself are no-ops. `unlink` and `demonitor` are fences: once they return, no notice of that edge -takes effect; one already queued stays. Each edge has a generation, so a stale +takes effect; an `Exit` already queued stays, a `Down` already queued is +flushed (X2). Each edge has a generation, so a stale notice never applies to a later link. `start_linked` registers the link before the child's `started()`. Checked by: `Lifecycle!NoProcIffLate`, `Lifecycle!NoEdgeToDead`, `Lifecycle!CancelFenced` (TLC, `MCLifecycleFence`); diff --git a/verification/tla/Lifecycle.tla b/verification/tla/Lifecycle.tla index a9eb9d79..7e3863f3 100644 --- a/verification/tla/Lifecycle.tla +++ b/verification/tla/Lifecycle.tla @@ -73,6 +73,7 @@ Past(a, s) == s \in {"gone", "dead"} (* SplitDeathPoint registration checks publication instead of D *) (* NoUnlinkFence unlink does not cancel a link notice in flight *) (* NoDemonitorFence demonitor does not cancel a Down in flight *) +(* NoDemonitorFlush a Down already queued runs after demonitor (v0.7) *) (* EarlyStopped a panicking request is answered before D *) (* PanicOverKill a panic during exit replaces Killed *) (* DropKeepsNormal a runtime drop keeps a requested Normal *) @@ -338,10 +339,14 @@ Monitor(a, b, m) == req, sends, signals, psent, afterStop, sigAfterD>> (* Demonitor: no Down is enqueued after it returns; one already queued stays. *) +(* With OTP's flush (demonitor(Ref, [flush])), also while the monitor's Down *) +(* is queued in the observer's mailbox: its dispatch then has no effect. *) Demonitor(a, m) == LET pending == {n \in inflight : n.k = "down" /\ n.m = m} + queued == \E i \in 1..Len(mbox[a]) : mbox[a][i].k = "down" /\ mbox[a][i].m = m IN /\ st[a] = "run" /\ mon[m].o = a - /\ mstate[m] \in {"active", "fired", "late"} /\ (mstate[m] = "active" \/ pending # {}) + /\ mstate[m] \in {"active", "fired", "late"} + /\ (mstate[m] = "active" \/ pending # {} \/ queued) /\ mstate' = [mstate EXCEPT ![m] = "cancelled"] /\ cancelled' = cancelled \cup pending /\ inflight' = IF Fault = "NoDemonitorFence" THEN inflight ELSE inflight \ pending @@ -371,7 +376,12 @@ Dispatch(a) == /\ UNCHANGED req [] OTHER -> \* msg or down (a panic here is SelfExit) UNCHANGED <> - /\ UNCHANGED <> + \* A flushed Down (its monitor cancelled by demonitor) is skipped: its + \* handler takes no effect. NoDemonitorFlush runs it anyway. + /\ LET h == Head(mbox[a]) IN + afterCancel' = (afterCancel \/ (h.k = "down" /\ Fault = "NoDemonitorFlush" + /\ mstate[h.m] = "cancelled")) + /\ UNCHANGED <> (* stopped() returned: the death point D. *) Die(a) == diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 28dcd5c4..bbecf473 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -240,6 +240,7 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCLifecycle", "KillNoSupersede", "KilledFromKill"), ("MCLifecycleFence", "NoUnlinkFence", "CancelFenced"), ("MCLifecycleFence", "NoDemonitorFence", "CancelFenced"), + ("MCLifecycleFence", "NoDemonitorFlush", "CancelFenced"), ("MCLifecycle", "EarlyStopped", "StoppedAtDeath"), ("MCLifecycle", "PanicOverKill", "KilledFromKill"), ("MCLifecycle", "DropKeepsNormal", "RuntimeDropReason"), From 04342a13aff2eaeee8ec5eeddf80df870176cfc8 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:13:46 -0300 Subject: [PATCH 154/208] refactor!: non_exhaustive enums and read-only structs; Down names the actor (0.8 API pass) SystemError, HelperError, RegistryError, SupervisorStrategy, ExecutionProfile, ChildState, Backend, spawned_test::Outcome) and on the read-only structs (Down, Exit, ChildSnapshot, SupervisorReport, ActorExit, metrics::Snapshot, HistogramSnapshot). Down gains actor, the monitored ActorId (OTP's Object in {'DOWN', Ref, process, Object, Info}); Exit::new and Down::new build notices for test doubles. Tests match with wildcard arms. --- CHANGELOG.md | 7 ++++ concurrency/src/link.rs | 9 ++++++ concurrency/src/metrics.rs | 2 ++ concurrency/src/monitor.rs | 21 ++++++++++-- concurrency/src/registry.rs | 1 + concurrency/src/supervision.rs | 36 +++++++++++++++++++++ concurrency/src/system.rs | 4 +++ concurrency/src/tasks/actor.rs | 3 ++ concurrency/src/threads/actor.rs | 2 ++ concurrency/tests/cross_backend.rs | 1 + concurrency/tests/execution_profiles.rs | 1 + concurrency/tests/otp_exit_signals.rs | 35 ++++---------------- concurrency/tests/supervision_strategies.rs | 1 + concurrency/tests/supervisor_policies.rs | 32 ++++++++++++++---- docs/GUIDE.md | 5 ++- docs/MIGRATION.md | 3 ++ docs/design/semantics.md | 2 +- testing/src/inbox.rs | 1 + 18 files changed, 127 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdd3ac88..82e82272 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,13 @@ `ShutdownReport::wait` and the test kit's waits are named `grace` or `limit` (a deadline is an instant). `PendingReplies` and `Context::deferred_outstanding` are documented as experimental. + - `#[non_exhaustive]` on the enums that will grow (`SupervisorError`, + `SystemError`, `HelperError`, `RegistryError`, `SupervisorStrategy`, + `ExecutionProfile`, `ChildState`, `Backend`, `spawned_test::Outcome`) + and on the read-only structs (`Down`, `Exit`, `ChildSnapshot`, + `SupervisorReport`, `ActorExit`, `metrics::Snapshot`, + `HistogramSnapshot`). `Down` gains `actor`, the monitored `ActorId` + (OTP's `Object` in `{'DOWN', Ref, process, Object, Info}`). - Deferred replies and jobs, fixes from the 0.8 pre-release review: a job deadline still fires while the result waits for scheduled emits, and the job keeps its `MAX_OUTSTANDING` credit until its terminal message; a job diff --git a/concurrency/src/link.rs b/concurrency/src/link.rs index 9712f2eb..da16c90b 100644 --- a/concurrency/src/link.rs +++ b/concurrency/src/link.rs @@ -17,6 +17,7 @@ use std::sync::{Arc, Mutex}; /// /// [`Context::trap_exit(true)`]: crate::threads::Context::trap_exit #[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] pub struct Exit { /// The actor that died. pub from: ActorId, @@ -24,6 +25,14 @@ pub struct Exit { pub reason: ExitReason, } +impl Exit { + /// An exit notice from `from` (for test doubles and comparisons; the + /// framework builds the ones it delivers). + pub fn new(from: ActorId, reason: ExitReason) -> Self { + Self { from, reason } + } +} + // --------------------------------------------------------------------------- // Internal types // --------------------------------------------------------------------------- diff --git a/concurrency/src/metrics.rs b/concurrency/src/metrics.rs index af4a3e09..c9520220 100644 --- a/concurrency/src/metrics.rs +++ b/concurrency/src/metrics.rs @@ -109,6 +109,7 @@ pub const BUCKET_BOUNDS_NS: [u64; 32] = { bounds }; #[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] pub struct HistogramSnapshot { pub buckets: [u64; 32], pub count: u64, @@ -159,6 +160,7 @@ macro_rules! counters { /// Process aggregates. Reads are independent atomic samples, not a /// transaction. Conservation equations hold after operations quiesce. #[derive(Debug, Clone)] + #[non_exhaustive] pub struct Snapshot { $( pub $name: u64, )* pub queue_wait: HistogramSnapshot, pub handler_duration: HistogramSnapshot, pub request_latency: HistogramSnapshot, pub deferred_reply_latency: HistogramSnapshot } pub fn snapshot() -> Snapshot { Snapshot { $( $name: COUNTERS.$name.load(Relaxed), )* queue_wait: QUEUE_WAIT.snapshot(), handler_duration: HANDLER_DURATION.snapshot(), request_latency: REQUEST_LATENCY.snapshot(), deferred_reply_latency: DEFERRED_REPLY_LATENCY.snapshot() } } fn render_counters(out: &mut String, s: &Snapshot) { diff --git a/concurrency/src/monitor.rs b/concurrency/src/monitor.rs index a022e17d..aeef9662 100644 --- a/concurrency/src/monitor.rs +++ b/concurrency/src/monitor.rs @@ -39,18 +39,35 @@ impl std::fmt::Display for MonitorRef { // Down // --------------------------------------------------------------------------- -/// Message delivered to a monitoring actor when its target stops. +/// Message delivered to a monitoring actor when its target stops: OTP's +/// `{'DOWN', Ref, process, Object, Info}`. /// /// To monitor another actor, call `ctx.monitor(&child_handle)` and implement -/// `Handler` on the monitoring actor. +/// `Handler` on the monitoring actor. Non-exhaustive: match it with +/// `Down { monitor_ref, actor, reason, .. }`. #[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] pub struct Down { /// The monitor that triggered this notification. pub monitor_ref: MonitorRef, + /// The monitored actor (OTP's `Object`). + pub actor: crate::ActorId, /// Why the monitored actor stopped. pub reason: ExitReason, } +impl Down { + /// A down notice for monitor `monitor_ref` of `actor` (for test doubles + /// and comparisons; the framework builds the ones it delivers). + pub fn new(monitor_ref: MonitorRef, actor: crate::ActorId, reason: ExitReason) -> Self { + Self { + monitor_ref, + actor, + reason, + } + } +} + impl Message for Down { type Result = (); } diff --git a/concurrency/src/registry.rs b/concurrency/src/registry.rs index eeb55405..c9ec1d6b 100644 --- a/concurrency/src/registry.rs +++ b/concurrency/src/registry.rs @@ -17,6 +17,7 @@ fn global_store() -> &'static Store { /// Errors that can occur when registering a value in the registry. #[derive(Debug, thiserror::Error)] +#[non_exhaustive] pub enum RegistryError { /// A value with this name is already registered. #[error("name '{0}' is already registered")] diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs index 750c8109..ec4caa1f 100644 --- a/concurrency/src/supervision.rs +++ b/concurrency/src/supervision.rs @@ -137,6 +137,7 @@ pub(crate) fn jitter_seed(seed: u64, name: &[u8]) -> u64 { /// The affected children are fully joined before ordered reconstruction. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[non_exhaustive] pub enum SupervisorStrategy { #[default] OneForOne, @@ -168,8 +169,40 @@ impl Default for SupervisorConfig { } } } +impl SupervisorConfig { + /// OneForOne, 5 restarts per 30 s, 1024 children, the default backoff. + pub fn new() -> Self { + Self::default() + } + /// Set [`strategy`](Self::strategy). + pub fn strategy(mut self, strategy: SupervisorStrategy) -> Self { + self.strategy = strategy; + self + } + /// Set [`max_restarts`](Self::max_restarts) (OTP's intensity). + pub fn max_restarts(mut self, max_restarts: usize) -> Self { + self.max_restarts = max_restarts; + self + } + /// Set [`restart_window`](Self::restart_window) (OTP's period). + pub fn restart_window(mut self, restart_window: Duration) -> Self { + self.restart_window = restart_window; + self + } + /// Set [`max_children`](Self::max_children). + pub fn max_children(mut self, max_children: usize) -> Self { + self.max_children = max_children; + self + } + /// Set [`backoff`](Self::backoff). + pub fn backoff(mut self, backoff: Backoff) -> Self { + self.backoff = backoff; + self + } +} #[derive(Debug, Clone, thiserror::Error)] +#[non_exhaustive] pub enum SupervisorError { #[error(transparent)] System(#[from] SystemError), @@ -205,6 +238,7 @@ pub enum SupervisorError { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum ChildState { Waiting, Starting, @@ -216,6 +250,7 @@ pub enum ChildState { } #[derive(Debug, Clone)] +#[non_exhaustive] pub struct ChildSnapshot { pub name: String, pub state: ChildState, @@ -1381,6 +1416,7 @@ pub struct Supervisor { core: Arc, } #[derive(Debug, Clone)] +#[non_exhaustive] pub struct SupervisorReport { pub complete: bool, pub failure: Option, diff --git a/concurrency/src/system.rs b/concurrency/src/system.rs index 9d1b43b1..f2f38f4e 100644 --- a/concurrency/src/system.rs +++ b/concurrency/src/system.rs @@ -30,6 +30,7 @@ use group::GroupInner; /// Resource models are opt-in; thread pools reserve one native thread per live /// actor, while pinned shards multiplex actors cooperatively on fixed CPUs. #[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum ExecutionProfile { AsyncPool { workers: usize, @@ -90,6 +91,7 @@ fn next_admission() -> Result { } #[derive(Debug, Clone, thiserror::Error)] +#[non_exhaustive] pub enum SystemError { #[error("no free persistent actor thread; stop/join a scope before admission")] Capacity, @@ -116,6 +118,7 @@ pub enum SystemError { } #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] pub enum HelperError { #[error("owned helper was cancelled before completion")] Cancelled, @@ -681,6 +684,7 @@ pub struct ShutdownReport { pub unfinished_jobs: Vec<(crate::ActorId, usize)>, } #[derive(Debug, Clone)] +#[non_exhaustive] pub struct ActorExit { pub id: crate::ActorId, pub reason: ExitReason, diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index cf36b71f..1c51c934 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -51,6 +51,7 @@ pub use jobs::{BlockingJobPermit, JobCtx, JobPermit}; /// account for those threads and reactor descriptors when sizing populations. /// Actor completion does not wait for detached helpers or runtime destruction. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[non_exhaustive] pub enum Backend { #[default] Async, @@ -846,6 +847,7 @@ impl Context { &self.shared.replies, Down { monitor_ref, + actor: target.id(), reason: ExitReason::NoProc, }, ); @@ -892,6 +894,7 @@ impl Context { &actor_ref.shared.replies, Down { monitor_ref, + actor: target.id(), reason, }, ); diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index b979ad1d..10f06995 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -721,6 +721,7 @@ impl Context { &self.replies, Down { monitor_ref, + actor: target.id(), reason: ExitReason::NoProc, }, ); @@ -770,6 +771,7 @@ impl Context { &actor_ref.replies, Down { monitor_ref, + actor: target.id(), reason, }, ); diff --git a/concurrency/tests/cross_backend.rs b/concurrency/tests/cross_backend.rs index 90bf9afa..186336ee 100644 --- a/concurrency/tests/cross_backend.rs +++ b/concurrency/tests/cross_backend.rs @@ -1001,6 +1001,7 @@ fn affected(strategy: SupervisorStrategy, n: usize, crashed: usize) -> std::ops: SupervisorStrategy::OneForOne => crashed..crashed + 1, SupervisorStrategy::OneForAll => 0..n, SupervisorStrategy::RestForOne => crashed..n, + other => unreachable!("strategy {other:?} not under test"), } } fn supervisor_config(strategy: SupervisorStrategy) -> SupervisorConfig { diff --git a/concurrency/tests/execution_profiles.rs b/concurrency/tests/execution_profiles.rs index d508e20f..d4cab99b 100644 --- a/concurrency/tests/execution_profiles.rs +++ b/concurrency/tests/execution_profiles.rs @@ -288,6 +288,7 @@ async fn each_strategy_recovers_on_a_full_native_pool_without_overlapping_slots( SupervisorStrategy::OneForOne => 4, SupervisorStrategy::OneForAll => 6, SupervisorStrategy::RestForOne => 5, + other => unreachable!("strategy {other:?} not under test"), } ); assert!(supervisor.shutdown(LIMIT).await.complete); diff --git a/concurrency/tests/otp_exit_signals.rs b/concurrency/tests/otp_exit_signals.rs index 980fab9f..299878b1 100644 --- a/concurrency/tests/otp_exit_signals.rs +++ b/concurrency/tests/otp_exit_signals.rs @@ -231,10 +231,7 @@ mod tasks_mode { settle(&trapper).await; assert_eq!( trapper_seen.exits(), - vec![Exit { - from: t.id(), - reason: ExitReason::Killed - }] + vec![Exit::new(t.id(), ExitReason::Killed)] ); settle(&watcher).await; assert_eq!(watcher_seen.downs().len(), 1); @@ -302,14 +299,8 @@ mod tasks_mode { assert_eq!( trap_seen.exits(), vec![ - Exit { - from: ActorId::EXTERNAL, - reason: ExitReason::Normal - }, - Exit { - from: ActorId::EXTERNAL, - reason: ExitReason::Shutdown - }, + Exit::new(ActorId::EXTERNAL, ExitReason::Normal), + Exit::new(ActorId::EXTERNAL, ExitReason::Shutdown), ] ); assert!(trapper.child_handle().is_alive()); @@ -340,10 +331,7 @@ mod tasks_mode { settle(&b).await; assert_eq!( b_seen.exits(), - vec![Exit { - from: a.id(), - reason: ExitReason::Panic("from a".into()) - }] + vec![Exit::new(a.id(), ExitReason::Panic("from a".into()))] ); let bh3 = bh.clone(); on(&a, move |ctx| ctx.exit(&bh3, ExitReason::Kill)).await; @@ -378,10 +366,7 @@ mod tasks_mode { settle(&trapper).await; assert_eq!( trap_seen.exits(), - vec![Exit { - from: d.id(), - reason: ExitReason::NoProc - }] + vec![Exit::new(d.id(), ExitReason::NoProc)] ); let downs = trap_seen.downs(); assert_eq!(downs.len(), 1); @@ -589,10 +574,7 @@ mod tasks_mode { assert_eq!(actor.request(Ping).await.unwrap(), 7); assert_eq!( seen.exits(), - vec![Exit { - from: ActorId::EXTERNAL, - reason: ExitReason::Shutdown - }] + vec![Exit::new(ActorId::EXTERNAL, ExitReason::Shutdown)] ); actor.child_handle().stop(); ended(&actor.child_handle()).await; @@ -1300,10 +1282,7 @@ mod threads_mode { assert_eq!(peer.request(Ping).unwrap(), 7); assert_eq!( peer_seen.exits(), - vec![Exit { - from: target.id(), - reason: ExitReason::Killed - }] + vec![Exit::new(target.id(), ExitReason::Killed)] ); // NoProc for a late link and try_link. assert_eq!(peer.context().try_link(&target.child_handle()), Err(NoProc)); diff --git a/concurrency/tests/supervision_strategies.rs b/concurrency/tests/supervision_strategies.rs index df9bbf72..ca15c6e8 100644 --- a/concurrency/tests/supervision_strategies.rs +++ b/concurrency/tests/supervision_strategies.rs @@ -170,6 +170,7 @@ async fn strategies_select_affected_children_and_reconstruct_in_dependency_order SupervisorStrategy::OneForOne => index == 1, SupervisorStrategy::OneForAll => true, SupervisorStrategy::RestForOne => index >= 1, + other => unreachable!("strategy {other:?} not under test"), }; if affected { let next = diff --git a/concurrency/tests/supervisor_policies.rs b/concurrency/tests/supervisor_policies.rs index 29caf049..9c304d28 100644 --- a/concurrency/tests/supervisor_policies.rs +++ b/concurrency/tests/supervisor_policies.rs @@ -100,6 +100,7 @@ fn apply( SupervisorStrategy::OneForOne => index == origin, SupervisorStrategy::OneForAll => true, SupervisorStrategy::RestForOne => index >= origin, + other => unreachable!("strategy {other:?} not modelled"), }; if affected && child.live { if modes[index] == RestartMode::Temporary { @@ -110,7 +111,27 @@ fn apply( } } } -fn unsettled(actual: &[ChildSnapshot], expected: &[Expected]) -> Vec { +/// The snapshot fields the checker reads (`ChildSnapshot` is +/// `#[non_exhaustive]`, so the negative test below builds these instead). +#[derive(Debug)] +struct Seen { + name: String, + state: ChildState, + generation: u64, +} +impl From for Seen { + fn from(s: ChildSnapshot) -> Self { + Seen { + name: s.name, + state: s.state, + generation: s.generation, + } + } +} +fn seen(children: &[SupervisedChild]) -> Vec { + children.iter().map(|c| c.snapshot().into()).collect() +} +fn unsettled(actual: &[Seen], expected: &[Expected]) -> Vec { assert_eq!(actual.len(), expected.len()); let mut pending = Vec::new(); // Do not short-circuit: a later child may have been over-reconstructed even @@ -140,7 +161,7 @@ fn unsettled(actual: &[ChildSnapshot], expected: &[Expected]) -> Vec { async fn observe(children: &[SupervisedChild], expected: &[Expected]) { let outcome = spawned_rt::tasks::timeout(LIMIT, async { loop { - let actual: Vec<_> = children.iter().map(SupervisedChild::snapshot).collect(); + let actual = seen(children); let pending = unsettled(&actual, expected); if pending.is_empty() { break; @@ -177,7 +198,7 @@ async fn observe(children: &[SupervisedChild], expected: &[Expected]) { }) .await; if outcome.is_err() { - let actual: Vec<_> = children.iter().map(SupervisedChild::snapshot).collect(); + let actual = seen(children); // Preserve the concrete over-reconstruction diagnosis at the watchdog // boundary too, rather than hiding it behind a generic timeout. let pending = unsettled(&actual, expected); @@ -346,13 +367,10 @@ fn retired_temporary_and_transient_slots_stay_retired_across_later_recoveries() #[test] #[should_panic(expected = "unexpected extra reconstruction: child-1")] fn a_waiting_earlier_child_cannot_hide_a_later_extra_reconstruction() { - let snapshot = |name: &str, state, generation| ChildSnapshot { + let snapshot = |name: &str, state, generation| Seen { name: name.into(), state, generation, - actor_id: None, - last_exit: None, - last_error: None, }; let actual = [ snapshot("child-0", ChildState::Backoff, 1), diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 8287bf7f..a3cae1b0 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -929,7 +929,10 @@ impl Watcher { } ``` -The hook is optional and defaults to doing nothing. `Exit`, `Down` and +The hook is optional and defaults to doing nothing. `Down` carries the +monitor's `MonitorRef`, the monitored actor's `ActorId` (`actor`, OTP's +`Object`) and its reason; `Exit` carries `from` and `reason`. Both are +`#[non_exhaustive]`: match them with `..`. `Exit`, `Down` and parent exits are ordinary mailbox messages: they queue behind messages already there and never interrupt a running handler. A death notice follows every message the dying actor sent earlier to the same receiver. A link notice is diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index da4f1448..a8c9e77f 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -42,6 +42,9 @@ every change; this guide covers what you need to edit when upgrading. | `InsertError>` *(0.8 pre-release)* | `InsertError` (`#[non_exhaustive]`) | | `DEFAULT_MAX_OUTSTANDING` *(0.8 pre-release)* | removed from the public API (`Actor::MAX_OUTSTANDING` defaults to 1024) | | `system.shutdown(deadline)`, `supervisor.shutdown(deadline)`, `report.wait(deadline)` | unchanged calls; the `Duration` parameter is named `grace` (a deadline is an instant in 0.8) | +| `Down { monitor_ref, reason }` (pattern) | `Down { monitor_ref, reason, .. }`; `Down` gains `actor: ActorId` (the monitored actor) and, like `Exit`, is `#[non_exhaustive]` | +| exhaustive `match` on `SupervisorError`, `SystemError`, `HelperError`, `RegistryError`, `SupervisorStrategy`, `ExecutionProfile`, `ChildState`, `Backend`, `spawned_test::Outcome` | add a wildcard arm (all `#[non_exhaustive]`) | +| struct literal or exhaustive pattern of `ChildSnapshot`, `SupervisorReport`, `ActorExit`, `metrics::Snapshot`, `HistogramSnapshot` | read the fields; patterns need `..` (all `#[non_exhaustive]`) | ### Exit signals, startup and runtime drops as in OTP (ranks 5e, 5f, 5g, targets 0.8, in progress) diff --git a/docs/design/semantics.md b/docs/design/semantics.md index 654fe39a..95c7e27d 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -312,7 +312,7 @@ outgoing monitors cancelled before publication. ## 5. Links, monitors and trapped exits **X1. Down exactly once.** `ctx.monitor(&handle)` (requires `Handler`) -yields exactly one `Down { monitor_ref, reason }` per monitor that is not +yields exactly one `Down { monitor_ref, actor, reason }` per monitor that is not cancelled: the target's final reason if registered before its death point D, `NoProc` (at once) if after it. Each call returns a distinct `MonitorRef`; the target is unaffected. diff --git a/testing/src/inbox.rs b/testing/src/inbox.rs index 3d08ab7f..99be74f5 100644 --- a/testing/src/inbox.rs +++ b/testing/src/inbox.rs @@ -8,6 +8,7 @@ use std::time::{Duration, Instant}; /// What a mock or probe did with one message it received. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum Outcome { /// Received, but its handling has not finished: the reply closure or a /// `delay` is still running. `expect` and `take_all` skip it until it is From 21081b194703000bcbfb12e6f9723fc4aae25a2d Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:15:30 -0300 Subject: [PATCH 155/208] refactor!: builders for SupervisorConfig, Backoff and metrics::Config (0.8 API pass) The three client-constructed config structs are non_exhaustive, so a later field (OTP added auto_shutdown to the supervisor flags in OTP 24) is not a breaking change. Each gains new() and one setter per field (Backoff::seed takes the seed); the public fields stay readable and assignable. Struct literals in tests, examples, benches and the fuzz driver become builder chains; GUIDE, MIGRATION and CHANGELOG follow. --- CHANGELOG.md | 3 + concurrency/benches/metrics_overhead.rs | 6 +- concurrency/examples/panic_abort.rs | 15 +-- concurrency/src/metrics.rs | 14 +++ concurrency/src/supervision.rs | 40 ++++++ concurrency/tests/canary_feedback.rs | 40 +++--- concurrency/tests/cancellation_safety.rs | 18 ++- concurrency/tests/cohort_trace.rs | 59 +++++---- concurrency/tests/coverage_trace.rs | 20 ++- concurrency/tests/cross_backend.rs | 24 ++-- concurrency/tests/execution_profiles.rs | 22 ++-- concurrency/tests/exit_signals_sim.rs | 5 +- concurrency/tests/failpoints.rs | 20 ++- concurrency/tests/foreign_panics.rs | 18 ++- concurrency/tests/leaks.rs | 36 +++--- concurrency/tests/metrics.rs | 5 +- concurrency/tests/metrics_composable.rs | 20 +-- concurrency/tests/metrics_counts_only.rs | 9 +- concurrency/tests/metrics_deferred.rs | 5 +- concurrency/tests/metrics_sampling.rs | 13 +- concurrency/tests/nested_supervision.rs | 16 +-- concurrency/tests/otp_exit_signals.rs | 5 +- concurrency/tests/panic_safety.rs | 18 ++- concurrency/tests/semantics_gaps.rs | 37 +++--- concurrency/tests/sim.rs | 53 ++++---- concurrency/tests/sim_explore.rs | 24 ++-- concurrency/tests/stack_budget.rs | 18 ++- concurrency/tests/supervision.rs | 119 +++++++----------- concurrency/tests/supervision_strategies.rs | 20 ++- concurrency/tests/supervision_trace.rs | 20 ++- concurrency/tests/supervisor_policies.rs | 26 ++-- concurrency/tests/tree_trace.rs | 30 ++--- docs/GUIDE.md | 17 ++- docs/MIGRATION.md | 3 + .../src/bin/coordinated_shutdown.rs | 5 +- .../capabilities/src/bin/framework_metrics.rs | 5 +- .../src/bin/registrations_and_fakes.rs | 17 ++- .../capabilities/src/bin/supervision_tree.rs | 15 +-- examples/capabilities/src/bin/test_kit.rs | 17 ++- fuzz/src/driver.rs | 29 +++-- fuzz/src/scenario.rs | 2 +- testing/tests/tasks_kit.rs | 51 ++++---- 42 files changed, 438 insertions(+), 501 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82e82272..a36c8c24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,9 @@ `SupervisorReport`, `ActorExit`, `metrics::Snapshot`, `HistogramSnapshot`). `Down` gains `actor`, the monitored `ActorId` (OTP's `Object` in `{'DOWN', Ref, process, Object, Info}`). + - `SupervisorConfig`, `Backoff` and `metrics::Config` are + `#[non_exhaustive]` with builders (`new()` plus one setter per field; + `Backoff::seed(u64)`). - Deferred replies and jobs, fixes from the 0.8 pre-release review: a job deadline still fires while the result waits for scheduled emits, and the job keeps its `MAX_OUTSTANDING` credit until its terminal message; a job diff --git a/concurrency/benches/metrics_overhead.rs b/concurrency/benches/metrics_overhead.rs index 2648a176..6cfecb08 100644 --- a/concurrency/benches/metrics_overhead.rs +++ b/concurrency/benches/metrics_overhead.rs @@ -54,9 +54,9 @@ fn main() { assert!(actors > 0 && actors <= 1024 && producers > 0 && producers <= 1024); assert!(duration.as_secs() > 0 && duration.as_secs() <= 60 && work <= 1_000_000); #[cfg(feature = "metrics")] - spawned_concurrency::metrics::configure(spawned_concurrency::metrics::Config { - timing_sample_every: _sample_every, - }) + spawned_concurrency::metrics::configure( + spawned_concurrency::metrics::Config::default().timing_sample_every(_sample_every), + ) .unwrap(); let (requests, elapsed) = if mode == "async" { use tasks::ActorStart; diff --git a/concurrency/examples/panic_abort.rs b/concurrency/examples/panic_abort.rs index 81263ee0..9a217fd1 100644 --- a/concurrency/examples/panic_abort.rs +++ b/concurrency/examples/panic_abort.rs @@ -103,15 +103,12 @@ mod tasks { tokio::runtime::Runtime::new().unwrap().block_on(async { let system = ActorSystem::new().unwrap(); let group = system.async_group("abort", 1).unwrap(); - let config = SupervisorConfig { - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(1), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }; + let config = SupervisorConfig::default().backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(1)) + .jitter(false), + ); let mut tree = system.supervisor(config).unwrap(); let child = tree .add( diff --git a/concurrency/src/metrics.rs b/concurrency/src/metrics.rs index c9520220..8a0f53c5 100644 --- a/concurrency/src/metrics.rs +++ b/concurrency/src/metrics.rs @@ -25,7 +25,9 @@ use spawned_rt::time::Instant; /// Timing samples are deterministic, one in every N observations per category. /// Zero disables clocks/histograms while retaining counters and gauges. +/// Non-exhaustive: `Config::new().timing_sample_every(n)`. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub struct Config { pub timing_sample_every: u64, } @@ -36,6 +38,18 @@ impl Default for Config { } } } +impl Config { + /// Time every observation (`timing_sample_every = 1`). + pub fn new() -> Self { + Self::default() + } + /// Time one in every `n` observations per category; 0 keeps counters + /// and gauges only. + pub fn timing_sample_every(mut self, n: u64) -> Self { + self.timing_sample_every = n; + self + } +} struct SamplingPolicy { // Only explicit requirements compose. Implicit default instrumentation must // not prevent an embedded component from configuring sampling later. diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs index ec4caa1f..a4eb3d88 100644 --- a/concurrency/src/supervision.rs +++ b/concurrency/src/supervision.rs @@ -63,7 +63,10 @@ fn exit_class(reason: &ExitReason, cleanup_failed: bool) -> &'static str { } } +/// Restart delay policy. Non-exhaustive: start from `Backoff::default()` (or +/// `new()`) and chain the setters, or assign the public fields of a value. #[derive(Debug, Clone)] +#[non_exhaustive] pub struct Backoff { /// First retry's delay cap, not its minimum. With jitter enabled the delay /// can be anywhere from zero through this value (1s permits a 150ms retry). @@ -93,6 +96,37 @@ impl Default for Backoff { } } impl Backoff { + /// The default policy: 10 ms initial cap, 5 s maximum, stable after + /// 30 s, jitter on, no fixed seed. + pub fn new() -> Self { + Self::default() + } + /// Set [`initial`](Self::initial). + pub fn initial(mut self, initial: Duration) -> Self { + self.initial = initial; + self + } + /// Set [`maximum`](Self::maximum). + pub fn maximum(mut self, maximum: Duration) -> Self { + self.maximum = maximum; + self + } + /// Set [`stable_after`](Self::stable_after). + pub fn stable_after(mut self, stable_after: Duration) -> Self { + self.stable_after = stable_after; + self + } + /// Set [`jitter`](Self::jitter). + pub fn jitter(mut self, jitter: bool) -> Self { + self.jitter = jitter; + self + } + /// Seed the jitter for reproducible tests. + pub fn seed(mut self, seed: u64) -> Self { + self.seed = Some(seed); + self + } + /// Capped exponential delay before jitter. pub(crate) fn bound(&self, failures: u32) -> Duration { self.initial @@ -145,7 +179,13 @@ pub enum SupervisorStrategy { RestForOne, } +/// Supervisor flags (OTP's `strategy`, `intensity` and `period`, plus a +/// child cap and a restart backoff). Non-exhaustive: start from +/// `SupervisorConfig::default()` (or `new()`) and chain the setters, +/// `SupervisorConfig::new().strategy(SupervisorStrategy::OneForAll).max_restarts(3)`, +/// or assign the public fields of a value. #[derive(Debug, Clone)] +#[non_exhaustive] pub struct SupervisorConfig { pub strategy: SupervisorStrategy, /// Supervisor-wide restarts per `restart_window` (OTP's intensity and diff --git a/concurrency/tests/canary_feedback.rs b/concurrency/tests/canary_feedback.rs index e1867c6e..e796d87a 100644 --- a/concurrency/tests/canary_feedback.rs +++ b/concurrency/tests/canary_feedback.rs @@ -201,16 +201,14 @@ async fn supervision_logs_restart_budget_exhaustion_and_give_up() { let system = ActorSystem::new().unwrap(); let group = system.async_group("canary-tracing", 1).unwrap(); let mut builder = system - .supervisor(SupervisorConfig { - max_restarts: 1, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default().max_restarts(1).backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false), + ), + ) .unwrap(); let child = builder .add( @@ -257,17 +255,17 @@ async fn supervision_logs_restart_budget_exhaustion_and_give_up() { // 60-second backoff, and lookups must report owner failure even while // the healthy sibling's owned cleanup is pending. let mut cohort = system - .supervisor(SupervisorConfig { - strategy: SupervisorStrategy::OneForAll, - max_restarts: 0, - backoff: Backoff { - initial: Duration::from_secs(60), - maximum: Duration::from_secs(60), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(SupervisorStrategy::OneForAll) + .max_restarts(0) + .backoff( + Backoff::default() + .initial(Duration::from_secs(60)) + .maximum(Duration::from_secs(60)) + .jitter(false), + ), + ) .unwrap(); let (release, gate) = std::sync::mpsc::channel(); let gate = Arc::new(Mutex::new(Some(gate))); diff --git a/concurrency/tests/cancellation_safety.rs b/concurrency/tests/cancellation_safety.rs index 68c4e4ba..a1f3576e 100644 --- a/concurrency/tests/cancellation_safety.rs +++ b/concurrency/tests/cancellation_safety.rs @@ -580,17 +580,13 @@ async fn group_ready_wait_is_cancel_safe() { // --------------------------------------------------------------------------- fn config() -> SupervisorConfig { - SupervisorConfig { - max_restarts: 100, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - seed: Some(1), - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default().max_restarts(100).backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false) + .seed(1), + ) } /// SupervisorBuilder::start: a dropped startup requests cleanup of every diff --git a/concurrency/tests/cohort_trace.rs b/concurrency/tests/cohort_trace.rs index 6323c54e..22554d5b 100644 --- a/concurrency/tests/cohort_trace.rs +++ b/concurrency/tests/cohort_trace.rs @@ -81,17 +81,17 @@ async fn trace_cohort_selection_and_temporary_retirement() { let system = ActorSystem::new().unwrap(); let group = system.async_group("cohort-trace", 2).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - strategy, - max_restarts: 30, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(2), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(30) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(2)) + .jitter(false), + ), + ) .unwrap(); let id = tree.trace_id(); let mut children = Vec::new(); @@ -142,11 +142,11 @@ async fn trace_superseded_round_with_parked_temporary_tail() { let group = system.async_group("cohort-superseded-trace", 2).unwrap(); let entered = Arc::new(tokio::sync::Notify::new()); let mut tree = system - .supervisor(SupervisorConfig { - strategy, - max_restarts: 30, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(30), + ) .unwrap(); let id = tree.trace_id(); let mut children = Vec::new(); @@ -215,10 +215,7 @@ async fn trace_normally_retired_transient_is_skipped_by_later_cohorts() { let system = ActorSystem::new().unwrap(); let group = system.async_group("retired-transient-trace", 2).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - strategy, - ..SupervisorConfig::default() - }) + .supervisor(SupervisorConfig::default().strategy(strategy)) .unwrap(); let mut children = Vec::new(); for i in 0..3 { @@ -294,17 +291,17 @@ async fn trace_failed_replacement_starts_a_new_round() { let system = ActorSystem::new().unwrap(); let group = system.async_group("cohort-pre-ready-trace", 2).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - strategy, - max_restarts: 30, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(2), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(30) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(2)) + .jitter(false), + ), + ) .unwrap(); let id = tree.trace_id(); tree.add(ChildSpec::new("a", &group, |_| async { diff --git a/concurrency/tests/coverage_trace.rs b/concurrency/tests/coverage_trace.rs index 9573ef3a..971f4a41 100644 --- a/concurrency/tests/coverage_trace.rs +++ b/concurrency/tests/coverage_trace.rs @@ -34,17 +34,15 @@ mod tree_record; const LIMIT: Duration = Duration::from_secs(4); fn config(max_restarts: usize) -> SupervisorConfig { - SupervisorConfig { - max_restarts, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(4), - jitter: false, - seed: Some(7), - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default() + .max_restarts(max_restarts) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(4)) + .jitter(false) + .seed(7), + ) } fn mode_name(mode: RestartMode) -> &'static str { diff --git a/concurrency/tests/cross_backend.rs b/concurrency/tests/cross_backend.rs index 186336ee..82c9144d 100644 --- a/concurrency/tests/cross_backend.rs +++ b/concurrency/tests/cross_backend.rs @@ -1005,19 +1005,17 @@ fn affected(strategy: SupervisorStrategy, n: usize, crashed: usize) -> std::ops: } } fn supervisor_config(strategy: SupervisorStrategy) -> SupervisorConfig { - SupervisorConfig { - strategy, - max_restarts: 10_000, - restart_window: Duration::from_secs(60), - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - seed: Some(1), - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(10_000) + .restart_window(Duration::from_secs(60)) + .backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false) + .seed(1), + ) } fn supervisor_workload( env: &mut Env, diff --git a/concurrency/tests/execution_profiles.rs b/concurrency/tests/execution_profiles.rs index d4cab99b..ac146ab8 100644 --- a/concurrency/tests/execution_profiles.rs +++ b/concurrency/tests/execution_profiles.rs @@ -235,17 +235,17 @@ async fn each_strategy_recovers_on_a_full_native_pool_without_overlapping_slots( let system = ActorSystem::new().unwrap(); let group = system.thread_group("supervised-roots", 3).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - strategy, - max_restarts: 20, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(2), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(20) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(2)) + .jitter(false), + ), + ) .unwrap(); let calls = Arc::new(AtomicUsize::new(0)); let mut children = Vec::new(); diff --git a/concurrency/tests/exit_signals_sim.rs b/concurrency/tests/exit_signals_sim.rs index 811e2b6a..9e2564af 100644 --- a/concurrency/tests/exit_signals_sim.rs +++ b/concurrency/tests/exit_signals_sim.rs @@ -471,10 +471,7 @@ async fn deferred_exit_vs_commit(seed: u64) { let log = Log::default(); let system = ActorSystem::new().unwrap(); let group = system.async_group("sim-s1", 2).unwrap(); - let config = SupervisorConfig { - strategy: SupervisorStrategy::OneForOne, - ..SupervisorConfig::default() - }; + let config = SupervisorConfig::default().strategy(SupervisorStrategy::OneForOne); let mut tree = system.supervisor(config).unwrap(); let (l, starts) = (log.clone(), Arc::new(Mutex::new(0u32))); let first = tree diff --git a/concurrency/tests/failpoints.rs b/concurrency/tests/failpoints.rs index ab2063be..64a1f4a9 100644 --- a/concurrency/tests/failpoints.rs +++ b/concurrency/tests/failpoints.rs @@ -27,17 +27,15 @@ use std::{ const LIMIT: Duration = Duration::from_secs(4); fn config(max_restarts: usize) -> SupervisorConfig { - SupervisorConfig { - max_restarts, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(4), - jitter: false, - seed: Some(7), - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default() + .max_restarts(max_restarts) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(4)) + .jitter(false) + .seed(7), + ) } /// Wait until a site has been reached `n` times. diff --git a/concurrency/tests/foreign_panics.rs b/concurrency/tests/foreign_panics.rs index aba3f359..a22ac4b6 100644 --- a/concurrency/tests/foreign_panics.rs +++ b/concurrency/tests/foreign_panics.rs @@ -440,17 +440,13 @@ mod env { let Some(group) = self.group.clone() else { return true; }; - let config = SupervisorConfig { - max_restarts: 10, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - seed: Some(1), - ..Backoff::default() - }, - ..SupervisorConfig::default() - }; + let config = SupervisorConfig::default().max_restarts(10).backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false) + .seed(1), + ); let mut tree = self.system.supervisor(config).unwrap(); let stopped = stopped.clone(); let child = tree diff --git a/concurrency/tests/leaks.rs b/concurrency/tests/leaks.rs index d5a615a9..e3822ee3 100644 --- a/concurrency/tests/leaks.rs +++ b/concurrency/tests/leaks.rs @@ -480,16 +480,14 @@ mod async_leaks { let system = ActorSystem::new().unwrap(); let group = system.async_group("leaks", 1).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - max_restarts: 100, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default().max_restarts(100).backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false), + ), + ) .unwrap(); let mut children = Vec::new(); for name in names { @@ -885,16 +883,14 @@ mod sim_leaks { let system = ActorSystem::new().unwrap(); let group = system.async_group("leaks-sim-sup", 1).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - max_restarts: 100, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default().max_restarts(100).backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false), + ), + ) .unwrap(); let child = tree .add(ChildSpec::new(name, &group, move |_| { diff --git a/concurrency/tests/metrics.rs b/concurrency/tests/metrics.rs index 7a5424a8..4867e21f 100644 --- a/concurrency/tests/metrics.rs +++ b/concurrency/tests/metrics.rs @@ -91,10 +91,7 @@ fn runtime() -> tokio::runtime::Runtime { // reset, and these assertions must not race unrelated actors in other tests. #[test] fn metrics_conserve_work_across_all_lifecycle_paths() { - metrics::configure(metrics::Config { - timing_sample_every: 1, - }) - .unwrap(); + metrics::configure(metrics::Config::default().timing_sample_every(1)).unwrap(); runtime().block_on(async { let (release, gate) = oneshot::channel(); let actor = Probe { gate: Some(gate) }.start(); diff --git a/concurrency/tests/metrics_composable.rs b/concurrency/tests/metrics_composable.rs index 5bcdf9fd..6e04c7ec 100644 --- a/concurrency/tests/metrics_composable.rs +++ b/concurrency/tests/metrics_composable.rs @@ -29,25 +29,15 @@ fn embedded_components_can_add_sampling_requirements_after_actors_start() { actor.request(Ping).unwrap(); assert_eq!(metrics::snapshot().queue_wait.count, 1); // The implicit default did not register an application requirement. - metrics::configure_composable(Config { - timing_sample_every: 0, - }); + metrics::configure_composable(Config::default().timing_sample_every(0)); actor.request(Ping).unwrap(); assert_eq!(metrics::snapshot().queue_wait.count, 1); // Two libraries initialize independently, after the host is already running. - metrics::configure_composable(Config { - timing_sample_every: 8, - }); - metrics::configure_composable(Config { - timing_sample_every: 2, - }); - metrics::configure_composable(Config { - timing_sample_every: 0, - }); - metrics::configure_composable(Config { - timing_sample_every: 16, - }); + metrics::configure_composable(Config::default().timing_sample_every(8)); + metrics::configure_composable(Config::default().timing_sample_every(2)); + metrics::configure_composable(Config::default().timing_sample_every(0)); + metrics::configure_composable(Config::default().timing_sample_every(16)); assert_eq!(metrics::configuration().timing_sample_every, 2); for _ in 0..32 { actor.request(Ping).unwrap(); diff --git a/concurrency/tests/metrics_counts_only.rs b/concurrency/tests/metrics_counts_only.rs index 44fe4c79..e3504150 100644 --- a/concurrency/tests/metrics_counts_only.rs +++ b/concurrency/tests/metrics_counts_only.rs @@ -22,10 +22,7 @@ impl Handler for Probe { } #[test] fn zero_timing_rate_retains_counters_without_timing_samples() { - metrics::configure(metrics::Config { - timing_sample_every: 0, - }) - .unwrap(); + metrics::configure(metrics::Config::default().timing_sample_every(0)).unwrap(); let actor = Probe.start(); for _ in 0..16 { actor.request(Ping).unwrap(); @@ -43,9 +40,7 @@ fn zero_timing_rate_retains_counters_without_timing_samples() { s.actors_active + s.handlers_active + s.requests_active + s.mailbox_depth, 0 ); - metrics::configure_composable(metrics::Config { - timing_sample_every: 8, - }); + metrics::configure_composable(metrics::Config::default().timing_sample_every(8)); assert_eq!(metrics::configuration().timing_sample_every, 8); assert!(metrics::configure(metrics::Config::default()).is_err()); } diff --git a/concurrency/tests/metrics_deferred.rs b/concurrency/tests/metrics_deferred.rs index 3ebb48a9..f75fedfd 100644 --- a/concurrency/tests/metrics_deferred.rs +++ b/concurrency/tests/metrics_deferred.rs @@ -169,10 +169,7 @@ async fn settle(actor: &ActorRef) { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn deferred_and_job_families_count_each_outcome_once() { - metrics::configure(Config { - timing_sample_every: 1, - }) - .unwrap(); + metrics::configure(Config::default().timing_sample_every(1)).unwrap(); let before = metrics::snapshot(); let actor = Owner::default().start(); diff --git a/concurrency/tests/metrics_sampling.rs b/concurrency/tests/metrics_sampling.rs index 44f02eca..3577fa7d 100644 --- a/concurrency/tests/metrics_sampling.rs +++ b/concurrency/tests/metrics_sampling.rs @@ -23,10 +23,7 @@ impl Handler for Probe { } #[test] fn sampling_retains_every_counter_and_scales_only_timing_observations() { - metrics::configure(metrics::Config { - timing_sample_every: 4, - }) - .unwrap(); + metrics::configure(metrics::Config::default().timing_sample_every(4)).unwrap(); let rt = tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -49,13 +46,9 @@ fn sampling_retains_every_counter_and_scales_only_timing_observations() { s.actors_active + s.handlers_active + s.requests_active + s.mailbox_depth, 0 ); - metrics::configure_composable(metrics::Config { - timing_sample_every: 8, - }); + metrics::configure_composable(metrics::Config::default().timing_sample_every(8)); assert_eq!(metrics::configuration().timing_sample_every, 4); - metrics::configure_composable(metrics::Config { - timing_sample_every: 2, - }); + metrics::configure_composable(metrics::Config::default().timing_sample_every(2)); assert_eq!(metrics::configuration().timing_sample_every, 2); assert!(metrics::configure(metrics::Config::default()).is_err()); } diff --git a/concurrency/tests/nested_supervision.rs b/concurrency/tests/nested_supervision.rs index 40ab1f78..d6891167 100644 --- a/concurrency/tests/nested_supervision.rs +++ b/concurrency/tests/nested_supervision.rs @@ -22,16 +22,12 @@ use std::{ mod tree_record; const LIMIT: Duration = Duration::from_secs(5); fn policy(retries: usize) -> SupervisorConfig { - SupervisorConfig { - max_restarts: retries, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(4), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default().max_restarts(retries).backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(4)) + .jitter(false), + ) } struct Next; impl Message for Next { diff --git a/concurrency/tests/otp_exit_signals.rs b/concurrency/tests/otp_exit_signals.rs index 299878b1..393aaa44 100644 --- a/concurrency/tests/otp_exit_signals.rs +++ b/concurrency/tests/otp_exit_signals.rs @@ -1095,10 +1095,7 @@ mod tasks_mode { ] { let system = ActorSystem::new().unwrap(); let group = system.async_group("s1", 1).unwrap(); - let config = SupervisorConfig { - strategy, - ..SupervisorConfig::default() - }; + let config = SupervisorConfig::default().strategy(strategy); let mut tree = system.supervisor(config).unwrap(); let seen = Seen::default(); let first = tree diff --git a/concurrency/tests/panic_safety.rs b/concurrency/tests/panic_safety.rs index 44da247a..b8fd1637 100644 --- a/concurrency/tests/panic_safety.rs +++ b/concurrency/tests/panic_safety.rs @@ -885,17 +885,13 @@ mod tasks_mode { } fn config() -> SupervisorConfig { - SupervisorConfig { - max_restarts: 10, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - seed: Some(1), - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default().max_restarts(10).backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false) + .seed(1), + ) } /// Under supervision the first generation fails in each callback; the diff --git a/concurrency/tests/semantics_gaps.rs b/concurrency/tests/semantics_gaps.rs index 5932738a..87e7792f 100644 --- a/concurrency/tests/semantics_gaps.rs +++ b/concurrency/tests/semantics_gaps.rs @@ -479,15 +479,12 @@ mod tasks { async fn timers_are_not_carried_to_a_supervised_replacement() { let system = ActorSystem::new().unwrap(); let group = system.async_group("timers", 1).unwrap(); - let config = SupervisorConfig { - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(1), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }; + let config = SupervisorConfig::default().backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(1)) + .jitter(false), + ); let (tx, mut rx) = unbounded_channel(); let first = Arc::new(AtomicBool::new(true)); let mut tree = system.supervisor(config).unwrap(); @@ -1893,18 +1890,16 @@ mod supervision_contract { }; fn config(strategy: SupervisorStrategy, initial: Duration) -> SupervisorConfig { - SupervisorConfig { - strategy, - max_restarts: 20, - backoff: Backoff { - initial, - maximum: Duration::from_secs(10), - jitter: false, - seed: Some(7), - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(20) + .backoff( + Backoff::default() + .initial(initial) + .maximum(Duration::from_secs(10)) + .jitter(false) + .seed(7), + ) } struct Hold { diff --git a/concurrency/tests/sim.rs b/concurrency/tests/sim.rs index 1bdddaaa..a3cc1bac 100644 --- a/concurrency/tests/sim.rs +++ b/concurrency/tests/sim.rs @@ -39,19 +39,17 @@ fn push(log: &Log, event: String) { } fn config(strategy: SupervisorStrategy) -> SupervisorConfig { - SupervisorConfig { - strategy, - max_restarts: 3, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(4), - jitter: false, - // Fixed jitter seed: otherwise it comes from SystemTime. - seed: Some(7), - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(3) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(4)) + .jitter(false) + // Fixed jitter seed: otherwise it comes from SystemTime. + .seed(7), + ) } struct Crash; @@ -415,20 +413,21 @@ async fn restart_window(out: Arc>>) { let system = ActorSystem::new().unwrap(); let group = system.async_group("sim-window", 1).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - strategy: SupervisorStrategy::OneForOne, - max_restarts: 3, - restart_window: Duration::from_secs(60), - backoff: Backoff { - initial: Duration::from_secs(1), - maximum: Duration::from_secs(8), - stable_after: Duration::from_secs(3600), - jitter: true, - // Left to spawned_rt::time::entropy: seeded by the simulation. - seed: None, - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(SupervisorStrategy::OneForOne) + .max_restarts(3) + .restart_window(Duration::from_secs(60)) + // No fixed seed: jitter is left to spawned_rt::time::entropy, + // seeded by the simulation. + .backoff( + Backoff::default() + .initial(Duration::from_secs(1)) + .maximum(Duration::from_secs(8)) + .stable_after(Duration::from_secs(3600)) + .jitter(true), + ), + ) .unwrap(); let child: SupervisedChild = tree .add(ChildSpec::new("flappy", &group, |_| async { diff --git a/concurrency/tests/sim_explore.rs b/concurrency/tests/sim_explore.rs index b05062e9..e4bf78e6 100644 --- a/concurrency/tests/sim_explore.rs +++ b/concurrency/tests/sim_explore.rs @@ -228,18 +228,18 @@ async fn initial_failure() -> Vec<(String, u64)> { let system = ActorSystem::new().unwrap(); let group = system.async_group("explore-initial", 2).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - strategy: SupervisorStrategy::OneForAll, - max_restarts: 3, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(4), - jitter: false, - seed: Some(7), - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(SupervisorStrategy::OneForAll) + .max_restarts(3) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(4)) + .jitter(false) + .seed(7), + ), + ) .unwrap(); let children: Vec<_> = (0..3) .map(|index| { diff --git a/concurrency/tests/stack_budget.rs b/concurrency/tests/stack_budget.rs index 0a34620b..11d16d0b 100644 --- a/concurrency/tests/stack_budget.rs +++ b/concurrency/tests/stack_budget.rs @@ -257,17 +257,13 @@ mod tasks_chain { system.async_group("stack-async", 2).unwrap(), system.thread_group("stack-thread", 4).unwrap(), ] { - let config = SupervisorConfig { - max_restarts: 10, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - seed: Some(1), - ..Backoff::default() - }, - ..SupervisorConfig::default() - }; + let config = SupervisorConfig::default().max_restarts(10).backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false) + .seed(1), + ); let mut tree = system.supervisor(config).unwrap(); let child = tree .add(ChildSpec::new("flaky", &group, |ctx| { diff --git a/concurrency/tests/supervision.rs b/concurrency/tests/supervision.rs index 038faedf..7ca256e9 100644 --- a/concurrency/tests/supervision.rs +++ b/concurrency/tests/supervision.rs @@ -17,17 +17,15 @@ use std::{ }; const LIMIT: Duration = Duration::from_secs(4); fn config(max_restarts: usize) -> SupervisorConfig { - SupervisorConfig { - max_restarts, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(8), - jitter: false, - seed: Some(7), - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default() + .max_restarts(max_restarts) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(8)) + .jitter(false) + .seed(7), + ) } struct Next; impl Message for Next { @@ -1164,17 +1162,16 @@ async fn backoff_exponent_resets_only_after_a_stable_generation() { let system = ActorSystem::new().unwrap(); let group = system.async_group("backoff-reset", 1).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - max_restarts: 20, - backoff: Backoff { - initial: Duration::from_millis(20), - maximum: Duration::from_secs(2), - stable_after: Duration::from_millis(300), - jitter: false, - seed: Some(7), - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default().max_restarts(20).backoff( + Backoff::default() + .initial(Duration::from_millis(20)) + .maximum(Duration::from_secs(2)) + .stable_after(Duration::from_millis(300)) + .jitter(false) + .seed(7), + ), + ) .unwrap(); let child = tree .add(ChildSpec::new("flappy", &group, |_| async { @@ -1461,18 +1458,18 @@ async fn one_for_all_restart_waits_for_its_backoff() { let system = ActorSystem::new().unwrap(); let group = system.async_group("ofa-backoff", 1).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - strategy: SupervisorStrategy::OneForAll, - max_restarts: 5, - backoff: Backoff { - initial: Duration::from_millis(300), - maximum: Duration::from_secs(2), - jitter: false, - seed: Some(1), - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(SupervisorStrategy::OneForAll) + .max_restarts(5) + .backoff( + Backoff::default() + .initial(Duration::from_millis(300)) + .maximum(Duration::from_secs(2)) + .jitter(false) + .seed(1), + ), + ) .unwrap(); let child = tree .add(ChildSpec::new("a", &group, |_| async { @@ -1507,43 +1504,19 @@ fn supervisor_config_limits_are_validated_at_their_boundaries() { }; let base = SupervisorConfig::default; assert!(accepts(base())); - assert!(rejects(SupervisorConfig { - restart_window: Duration::ZERO, - ..base() - })); - assert!(rejects(SupervisorConfig { - max_children: 0, - ..base() - })); - assert!(accepts(SupervisorConfig { - max_children: 65536, - ..base() - })); - assert!(rejects(SupervisorConfig { - max_children: 65537, - ..base() - })); - assert!(accepts(SupervisorConfig { - max_restarts: 65536, - ..base() - })); - assert!(rejects(SupervisorConfig { - max_restarts: 65537, - ..base() - })); - let backoff = |initial, maximum| Backoff { - initial: Duration::from_millis(initial), - maximum: Duration::from_millis(maximum), - ..Backoff::default() + assert!(rejects(base().restart_window(Duration::ZERO))); + assert!(rejects(base().max_children(0))); + assert!(accepts(base().max_children(65536))); + assert!(rejects(base().max_children(65537))); + assert!(accepts(base().max_restarts(65536))); + assert!(rejects(base().max_restarts(65537))); + let backoff = |initial, maximum| { + Backoff::default() + .initial(Duration::from_millis(initial)) + .maximum(Duration::from_millis(maximum)) }; - assert!(accepts(SupervisorConfig { - backoff: backoff(10, 10), - ..base() - })); - assert!(rejects(SupervisorConfig { - backoff: backoff(11, 10), - ..base() - })); + assert!(accepts(base().backoff(backoff(10, 10)))); + assert!(rejects(base().backoff(backoff(11, 10)))); } /// T4/V5: on native threads (`thread_group`) the restart window runs on the @@ -1564,11 +1537,7 @@ fn native_restart_window_expires_on_the_monotonic_clock() { let system = ActorSystem::new().unwrap(); let group = system.thread_group("window", 4).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - max_restarts: 1, - restart_window: window, - ..config(1) - }) + .supervisor(config(1).max_restarts(1).restart_window(window)) .unwrap(); let child = tree .add(ChildSpec::new("native", &group, |_| async { diff --git a/concurrency/tests/supervision_strategies.rs b/concurrency/tests/supervision_strategies.rs index ca15c6e8..dd92266f 100644 --- a/concurrency/tests/supervision_strategies.rs +++ b/concurrency/tests/supervision_strategies.rs @@ -18,17 +18,15 @@ use std::{ }; const LIMIT: Duration = Duration::from_secs(5); fn policy(strategy: SupervisorStrategy) -> SupervisorConfig { - SupervisorConfig { - strategy, - max_restarts: 100, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(4), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(100) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(4)) + .jitter(false), + ) } struct Next; impl Message for Next { diff --git a/concurrency/tests/supervision_trace.rs b/concurrency/tests/supervision_trace.rs index 072abe8e..730c8a71 100644 --- a/concurrency/tests/supervision_trace.rs +++ b/concurrency/tests/supervision_trace.rs @@ -23,17 +23,15 @@ use std::{ const LIMIT: Duration = Duration::from_secs(4); fn config(max_restarts: usize) -> SupervisorConfig { - SupervisorConfig { - max_restarts, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(8), - jitter: false, - seed: Some(7), - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default() + .max_restarts(max_restarts) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(8)) + .jitter(false) + .seed(7), + ) } fn mode_name(mode: RestartMode) -> &'static str { diff --git a/concurrency/tests/supervisor_policies.rs b/concurrency/tests/supervisor_policies.rs index 9c304d28..20e671ae 100644 --- a/concurrency/tests/supervisor_policies.rs +++ b/concurrency/tests/supervisor_policies.rs @@ -231,19 +231,19 @@ async fn scenario( let system = ActorSystem::new().unwrap(); let group = system.async_group("policy-oracle", 1).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - strategy, - max_restarts: 128, - restart_window: Duration::from_secs(3600), - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - seed: Some(42), - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(128) + .restart_window(Duration::from_secs(3600)) + .backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false) + .seed(42), + ), + ) .unwrap(); let mut children = Vec::new(); for (index, restart) in modes.iter().enumerate() { diff --git a/concurrency/tests/tree_trace.rs b/concurrency/tests/tree_trace.rs index 093c7833..2283d85a 100644 --- a/concurrency/tests/tree_trace.rs +++ b/concurrency/tests/tree_trace.rs @@ -30,17 +30,15 @@ impl Handler for Worker { } } fn config(strategy: SupervisorStrategy) -> SupervisorConfig { - SupervisorConfig { - strategy, - max_restarts: 10, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(2), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(10) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(2)) + .jitter(false), + ) } #[path = "support/tree_record.rs"] mod tree_record; @@ -61,10 +59,7 @@ async fn mixed_strategy_three_level_shutdown_and_subtree_replacement() { .add_supervisor(SupervisorSpec::new( "branch", &group, - SupervisorConfig { - max_restarts: 0, - ..config(SupervisorStrategy::RestForOne) - }, + config(SupervisorStrategy::RestForOne).max_restarts(0), move |middle| { let placement = placement.clone(); let expose = expose.clone(); @@ -72,10 +67,7 @@ async fn mixed_strategy_three_level_shutdown_and_subtree_replacement() { middle.add_supervisor(SupervisorSpec::new( "nested", &control, - SupervisorConfig { - max_restarts: 0, - ..config(SupervisorStrategy::OneForAll) - }, + config(SupervisorStrategy::OneForAll).max_restarts(0), move |inner| { *expose.lock().unwrap() = Some(inner.add(ChildSpec::new("leaf", &placement, |_| async { diff --git a/docs/GUIDE.md b/docs/GUIDE.md index a3cae1b0..b99e7377 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -1276,6 +1276,19 @@ zero through one second, including 0.15–0.8s. Subsequent caps double up to ready lifetime resets that child's exponent. Cleanup time is not healthy uptime. Use `Backoff::seed` for reproducible policy tests, not security randomness. +`SupervisorConfig` and `Backoff` are `#[non_exhaustive]` (OTP's supervisor +flags grew in OTP 24, and so will these): build them with the setters, + +```rust +let config = SupervisorConfig::new() + .strategy(SupervisorStrategy::OneForAll) + .max_restarts(3) + .restart_window(Duration::from_secs(10)) + .backoff(Backoff::new().initial(Duration::from_millis(50)).jitter(false)); +``` + +or assign the public fields of a `default()` value. + ### Recovery strategies and nested trees Set `SupervisorConfig::strategy` explicitly; OneForOne remains the default. @@ -1425,7 +1438,7 @@ Configure before starting actors or constructing observed requests: ```rust use spawned_concurrency::metrics::{self, Config}; -metrics::configure(Config { timing_sample_every: 16 }).unwrap(); +metrics::configure(Config::new().timing_sample_every(16)).unwrap(); // Counters include every operation; one in 16 observations gets timing samples. let snapshot = metrics::snapshot(); let prometheus_text = metrics::render_prometheus(); @@ -1437,7 +1450,7 @@ histogram observations. Configuration is process-wide. The one-time `configure` requirements later, including with a policy previously set by `configure`. For embedded components that initialize independently, use -`metrics::configure_composable(Config { timing_sample_every: 16 })` instead. +`metrics::configure_composable(Config::new().timing_sample_every(16))` instead. It can be called after actors start. The smallest nonzero interval requested by any component wins; zero means “I only require counters” and never disables another component's timings. Requirements last for the process lifetime. Instrumentation initially samples every observation until an explicit policy diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index a8c9e77f..38f9ddbf 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -45,6 +45,9 @@ every change; this guide covers what you need to edit when upgrading. | `Down { monitor_ref, reason }` (pattern) | `Down { monitor_ref, reason, .. }`; `Down` gains `actor: ActorId` (the monitored actor) and, like `Exit`, is `#[non_exhaustive]` | | exhaustive `match` on `SupervisorError`, `SystemError`, `HelperError`, `RegistryError`, `SupervisorStrategy`, `ExecutionProfile`, `ChildState`, `Backend`, `spawned_test::Outcome` | add a wildcard arm (all `#[non_exhaustive]`) | | struct literal or exhaustive pattern of `ChildSnapshot`, `SupervisorReport`, `ActorExit`, `metrics::Snapshot`, `HistogramSnapshot` | read the fields; patterns need `..` (all `#[non_exhaustive]`) | +| `SupervisorConfig { strategy: s, max_restarts: n, ..Default::default() }` | `SupervisorConfig::new().strategy(s).max_restarts(n)` (or assign fields of `SupervisorConfig::default()`) | +| `Backoff { initial: a, jitter: false, seed: Some(7), ..Default::default() }` | `Backoff::new().initial(a).jitter(false).seed(7)` | +| `metrics::Config { timing_sample_every: n }` | `metrics::Config::new().timing_sample_every(n)` | ### Exit signals, startup and runtime drops as in OTP (ranks 5e, 5f, 5g, targets 0.8, in progress) diff --git a/examples/capabilities/src/bin/coordinated_shutdown.rs b/examples/capabilities/src/bin/coordinated_shutdown.rs index 97be3919..fd2f1878 100644 --- a/examples/capabilities/src/bin/coordinated_shutdown.rs +++ b/examples/capabilities/src/bin/coordinated_shutdown.rs @@ -104,10 +104,7 @@ async fn main() { let stops: Stops = Arc::default(); // RestForOne rebuilds dependents retaining an earlier instance's reference. let mut tree = system - .supervisor(SupervisorConfig { - strategy: SupervisorStrategy::RestForOne, - ..SupervisorConfig::default() - }) + .supervisor(SupervisorConfig::default().strategy(SupervisorStrategy::RestForOne)) .unwrap(); let events = stops.clone(); let peers = tree diff --git a/examples/capabilities/src/bin/framework_metrics.rs b/examples/capabilities/src/bin/framework_metrics.rs index e389a941..076fddef 100644 --- a/examples/capabilities/src/bin/framework_metrics.rs +++ b/examples/capabilities/src/bin/framework_metrics.rs @@ -22,10 +22,7 @@ impl Handler for Worker { } #[tokio::main(flavor = "multi_thread", worker_threads = 2)] async fn main() { - metrics::configure(Config { - timing_sample_every: 16, - }) - .unwrap(); + metrics::configure(Config::default().timing_sample_every(16)).unwrap(); let actor = Worker.start(); for _ in 0..32 { actor.request(Ping).await.unwrap(); diff --git a/examples/capabilities/src/bin/registrations_and_fakes.rs b/examples/capabilities/src/bin/registrations_and_fakes.rs index 35466802..861c1997 100644 --- a/examples/capabilities/src/bin/registrations_and_fakes.rs +++ b/examples/capabilities/src/bin/registrations_and_fakes.rs @@ -56,15 +56,14 @@ async fn main() { let system = ActorSystem::new().unwrap(); let group = system.async_group("named-service", 1).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default().backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false), + ), + ) .unwrap(); let service = tree .add(ChildSpec::new("prices", &group, |_| async { diff --git a/examples/capabilities/src/bin/supervision_tree.rs b/examples/capabilities/src/bin/supervision_tree.rs index d098e3b5..2bcc7ee1 100644 --- a/examples/capabilities/src/bin/supervision_tree.rs +++ b/examples/capabilities/src/bin/supervision_tree.rs @@ -40,11 +40,11 @@ async fn main() { let controls = system.async_group("tree-control", 1).unwrap(); let workers = system.thread_group("tree-workers", 3).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - strategy, - max_restarts: 20, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(20), + ) .unwrap(); let earlier = tree .add(ChildSpec::new("earlier", &workers, |_| async { @@ -58,10 +58,7 @@ async fn main() { .add_supervisor(SupervisorSpec::new( "subsystem", &controls, - SupervisorConfig { - max_restarts: 0, - ..SupervisorConfig::default() - }, + SupervisorConfig::default().max_restarts(0), move |subtree| { let child = subtree.add(ChildSpec::new("leaf", &placement, |_| async { Ok::<_, String>(Worker) diff --git a/examples/capabilities/src/bin/test_kit.rs b/examples/capabilities/src/bin/test_kit.rs index 20ad7467..f8151eb2 100644 --- a/examples/capabilities/src/bin/test_kit.rs +++ b/examples/capabilities/src/bin/test_kit.rs @@ -110,15 +110,14 @@ async fn main() { let test = kit::TestSystem::new(); let mut tree = test .system() - .supervisor(SupervisorConfig { - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default().backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false), + ), + ) .unwrap(); let rates_ref = rates.to_rates_ref(); let probe = out.clone(); diff --git a/fuzz/src/driver.rs b/fuzz/src/driver.rs index b843fdf4..ee70ec20 100644 --- a/fuzz/src/driver.rs +++ b/fuzz/src/driver.rs @@ -247,26 +247,25 @@ fn restart_mode(v: u8) -> RestartMode { fn config(p: &Policy, seed: u64) -> SupervisorConfig { let initial = Duration::from_millis(p.backoff as u64); - SupervisorConfig { - strategy: match p.strategy % 3 { + SupervisorConfig::default() + .strategy(match p.strategy % 3 { 0 => SupervisorStrategy::OneForOne, 1 => SupervisorStrategy::OneForAll, _ => SupervisorStrategy::RestForOne, - }, - max_restarts: (p.max_restarts % 7) as usize, + }) + .max_restarts((p.max_restarts % 7) as usize) // At least 50 ms, so a fast restart loop exhausts the budget // instead of restarting forever as virtual time creeps. - restart_window: Duration::from_millis(50 + 2 * p.window as u64), - backoff: Backoff { - initial, - maximum: initial * 4, - stable_after: Duration::from_millis(10 * p.stable as u64 + 1), - jitter: p.jitter, - // Never the default (wall-clock) jitter seed. - seed: Some(seed), - }, - ..SupervisorConfig::default() - } + .restart_window(Duration::from_millis(50 + 2 * p.window as u64)) + .backoff( + Backoff::default() + .initial(initial) + .maximum(initial * 4) + .stable_after(Duration::from_millis(10 * p.stable as u64 + 1)) + .jitter(p.jitter) + // Never the default (wall-clock) jitter seed. + .seed(seed), + ) } fn cfg_strategy(p: &Policy) -> SupervisorStrategy { diff --git a/fuzz/src/scenario.rs b/fuzz/src/scenario.rs index 2f011237..ac35e9fa 100644 --- a/fuzz/src/scenario.rs +++ b/fuzz/src/scenario.rs @@ -255,7 +255,7 @@ pub enum Op { DropSystem { sys: u8, }, - /// `request_raw`, then drop the reply receiver (R4: not a cancellation). + /// `send_request`, then drop the reply receiver (R4: not a cancellation). DropReply { to: u8, cmd: Cmd, diff --git a/testing/tests/tasks_kit.rs b/testing/tests/tasks_kit.rs index 48a9eb6b..06182a45 100644 --- a/testing/tests/tasks_kit.rs +++ b/testing/tests/tasks_kit.rs @@ -304,15 +304,14 @@ async fn supervised_child_restarts_after_injected_crash() { let test = TestSystem::new(); let mut tree = test .system() - .supervisor(SupervisorConfig { - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default().backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false), + ), + ) .unwrap(); let child = tree .add(ChildSpec::new("counter", test.group(), |_| async { @@ -508,15 +507,14 @@ async fn restart_child_waits_out_a_restart_in_progress() { let test = TestSystem::new(); let mut tree = test .system() - .supervisor(SupervisorConfig { - backoff: Backoff { - initial: Duration::from_millis(100), - maximum: Duration::from_millis(100), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default().backoff( + Backoff::default() + .initial(Duration::from_millis(100)) + .maximum(Duration::from_millis(100)) + .jitter(false), + ), + ) .unwrap(); let child = tree .add(ChildSpec::new("counter", test.group(), |_| async { @@ -569,15 +567,14 @@ async fn restart_child_has_one_deadline_for_the_whole_call() { let test = TestSystem::new(); let mut tree = test .system() - .supervisor(SupervisorConfig { - backoff: Backoff { - initial: Duration::from_millis(400), - maximum: Duration::from_millis(400), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default().backoff( + Backoff::default() + .initial(Duration::from_millis(400)) + .maximum(Duration::from_millis(400)) + .jitter(false), + ), + ) .unwrap(); let child = tree .add(ChildSpec::new("counter", test.group(), |_| async { From 64eca940a1971d9798695ca4e4a4cb5e314cbdc6 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:18:01 -0300 Subject: [PATCH 156/208] test(exit): run the stopped_skipped log test in its own process The test installed a thread-scoped subscriber while other test threads registered the same callsites; the interest cache could settle on "never" and drop the debug events, so it failed under load (twice in four validation runs on a loaded host). The child process installs the capture as the global subscriber. --- concurrency/tests/otp_exit_signals.rs | 30 ++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/concurrency/tests/otp_exit_signals.rs b/concurrency/tests/otp_exit_signals.rs index 393aaa44..f61fd8a6 100644 --- a/concurrency/tests/otp_exit_signals.rs +++ b/concurrency/tests/otp_exit_signals.rs @@ -43,6 +43,9 @@ impl Message for Ping { type Result = u8; } +#[path = "support/drop_subprocess.rs"] +mod subprocess; + mod tasks_mode { use super::*; use spawned_concurrency::tasks::{Actor, ActorRef, ActorStart, Context, Handler}; @@ -736,10 +739,10 @@ mod tasks_mode { // Decision 2 is easy to miss: a skipped stopped() is logged at debug // (`event = "stopped_skipped"`), for a kill, an untrapped signal and a - // runtime drop. A current-thread runtime keeps the scoped subscriber on - // the actors' thread. + // runtime drop. mod stopped_skipped_log { use super::*; + use crate::subprocess; use std::collections::BTreeMap; type Fields = BTreeMap; #[derive(Clone, Default)] @@ -794,10 +797,26 @@ mod tasks_mode { } } + /// Runs in its own process with `Capture` as the global subscriber: + /// a thread-scoped subscriber raced other test threads registering + /// the `stopped_skipped` callsite, which could cache its interest as + /// "never" and drop the events (a flaky failure under load). #[test] fn skipped_stopped_is_logged_at_debug() { + subprocess::run( + "tasks_mode::stopped_skipped_log::child", + "skipped stopped() logged", + ); + } + + #[test] + #[ignore = "subprocess helper exercised by the enabled parent"] + fn child() { + if std::env::var_os("SPAWNED_MAILBOX_DROP_CHILD").is_none() { + return; + } let capture = Capture::default(); - let _subscriber = tracing::subscriber::set_default(capture.clone()); + tracing::subscriber::set_global_default(capture.clone()).unwrap(); let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -806,10 +825,6 @@ mod tasks_mode { let seen = Seen::default(); let killed = Node::new(&seen, true).start(); settle(&killed).await; - // Another test thread may have been registering the - // `stopped_skipped` callsite while `set_default` rebuilt the - // interest cache, leaving it cached as "never". - tracing::callsite::rebuild_interest_cache(); killed.child_handle().kill(); assert_eq!(ended(&killed.child_handle()).await, ExitReason::Killed); let signalled = Node::new(&seen, false).start(); @@ -845,6 +860,7 @@ mod tasks_mode { capture.skipped(dropped), vec![("runtime_drop".into(), "killed".into())] ); + println!("skipped stopped() logged"); } } From 83d409869b3877a7997bb63a0fab06c05dbf664d Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:32:17 -0300 Subject: [PATCH 157/208] test(kit): restart_child names its Duration a limit The send_request change renamed the test kit's Duration parameters from deadline to limit (a deadline is an instant in 0.8), and with them restart_child's panic text; the should_panic expectation follows. --- testing/tests/tasks_kit.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/tests/tasks_kit.rs b/testing/tests/tasks_kit.rs index 06182a45..ff1829ab 100644 --- a/testing/tests/tasks_kit.rs +++ b/testing/tests/tasks_kit.rs @@ -562,7 +562,7 @@ async fn exit_injection_never_blocks_a_current_thread_runtime() { // Regression: `deadline` bounds the whole call, not each of its phases. Each // phase here waits out a 400 ms backoff, so two phases exceed 600 ms. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[should_panic(expected = "one deadline for the whole call")] +#[should_panic(expected = "one limit for the whole call")] async fn restart_child_has_one_deadline_for_the_whole_call() { let test = TestSystem::new(); let mut tree = test From 617c993d9bbe484d8efd8e77586917ae4b11849a Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:32:34 -0300 Subject: [PATCH 158/208] fix(bench): await fan-out ReplyReceivers through into_future ReplyReceiver is IntoFuture, not Future, so join_all over send_request receivers did not compile with --all-features. --- concurrency/benches/workload_research.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/concurrency/benches/workload_research.rs b/concurrency/benches/workload_research.rs index 0c2223b9..deba48c9 100644 --- a/concurrency/benches/workload_research.rs +++ b/concurrency/benches/workload_research.rs @@ -271,8 +271,11 @@ impl Handler for Node { .children .iter() .map(|c| { - c.send_request(msg.clone(), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) - .unwrap() + // A `ReplyReceiver` is `IntoFuture`; `join_all` takes futures. + std::future::IntoFuture::into_future( + c.send_request(msg.clone(), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(), + ) }) .collect(); let mut nodes = 1; From 7aaf93fb2fd23d6828925c9f35514c9810de08d3 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:39:15 -0300 Subject: [PATCH 159/208] refactor!: seal Receiver, hide the reply module and test hooks, check #[send_handler] (0.8 API pass) Receiver is sealed (spawned-test's probe implements it through __private::Sealed). The reply module is doc(hidden): its types are at the crate root and under tasks/threads, and macros emit root paths. The test hooks (__in_turn, __discharge_stopped, reply_pair, PendingReply::detached) move into a doc(hidden) __private module with no semver promise. lifecycle_trace is public only with its feature; spawned_rt::time's tick plumbing is doc(hidden). #[handler] is removed, and a #[send_handler] method returning a value is a compile error instead of silently answering a request (two new trybuild cases). The experiment benches use the ReplyReceiver API. Tests, examples and docs name root paths. --- CHANGELOG.md | 13 ++++ README.md | 2 +- concurrency/benches/baseline.rs | 4 +- concurrency/benches/designs.rs | 26 ++++--- concurrency/benches/lifecycle_soak.rs | 10 +-- concurrency/benches/metrics_overhead.rs | 4 +- concurrency/benches/pinned_pipeline.rs | 6 +- concurrency/benches/runtime_isolation.rs | 4 +- concurrency/benches/workload_research.rs | 18 +++-- concurrency/examples/panic_abort.rs | 4 +- concurrency/src/lib.rs | 68 ++++++++++++++++++- concurrency/src/reply/mod.rs | 11 ++- concurrency/src/response.rs | 2 +- concurrency/src/tasks/actor.rs | 14 ++-- concurrency/src/threads/actor.rs | 14 ++-- concurrency/tests/actor_scenarios.rs | 8 +-- concurrency/tests/backend_experiments.rs | 6 +- concurrency/tests/backend_resource_limits.rs | 2 +- concurrency/tests/borrowed_runtime.rs | 4 +- concurrency/tests/canary_feedback.rs | 6 +- concurrency/tests/cancellation_safety.rs | 6 +- concurrency/tests/cfg_gated_protocol.rs | 4 +- concurrency/tests/cohort_trace.rs | 4 +- concurrency/tests/context_exit.rs | 4 +- concurrency/tests/coverage_trace.rs | 4 +- concurrency/tests/cross_backend.rs | 18 ++--- concurrency/tests/deadline_service.rs | 64 +++++++++-------- concurrency/tests/death_boundary.rs | 47 +++++++------ concurrency/tests/dedicated_runtime.rs | 8 +-- concurrency/tests/deferred_api.rs | 5 +- concurrency/tests/deferred_replies.rs | 7 +- concurrency/tests/execution_groups.rs | 10 +-- concurrency/tests/execution_profiles.rs | 10 +-- concurrency/tests/exit_signal_matrix.rs | 6 +- concurrency/tests/failpoints.rs | 22 +++--- concurrency/tests/foreign_panics.rs | 16 ++--- concurrency/tests/full_actor_shards.rs | 23 +++---- concurrency/tests/future_layout.rs | 4 +- concurrency/tests/kill_links.rs | 4 +- concurrency/tests/leaks.rs | 36 +++++----- concurrency/tests/mailbox_drop_panic.rs | 8 +-- concurrency/tests/metrics.rs | 14 ++-- concurrency/tests/metrics_composable.rs | 2 +- concurrency/tests/metrics_counts_only.rs | 2 +- concurrency/tests/metrics_sampling.rs | 2 +- concurrency/tests/native_dispatcher.rs | 2 +- concurrency/tests/native_mailbox_drop.rs | 4 +- concurrency/tests/nested_supervision.rs | 6 +- concurrency/tests/otp_exit_signals.rs | 36 +++++----- concurrency/tests/panic_safety.rs | 40 +++++------ concurrency/tests/process_failures.rs | 2 +- concurrency/tests/protocol_timeouts.rs | 22 +++--- concurrency/tests/resource_cleanup.rs | 4 +- concurrency/tests/semantics_gaps.rs | 26 +++---- concurrency/tests/shutdown_publication.rs | 2 +- concurrency/tests/sim.rs | 2 +- concurrency/tests/sim_explore.rs | 8 +-- concurrency/tests/stack_budget.rs | 8 +-- concurrency/tests/startup_admission.rs | 2 +- concurrency/tests/supervision.rs | 16 ++--- concurrency/tests/supervision_strategies.rs | 8 +-- concurrency/tests/supervision_trace.rs | 12 ++-- concurrency/tests/supervisor_policies.rs | 4 +- concurrency/tests/support/failure_actor.rs | 8 +-- concurrency/tests/support/store_actor.rs | 4 +- concurrency/tests/tree_trace.rs | 2 +- .../tests/ui-pass/actor_tasks_lifecycle.rs | 4 +- .../tests/ui-pass/protocol_edge_cases.rs | 2 +- .../tests/ui/actor_exit_with_handler.rs | 2 +- concurrency/tests/ui/actor_handler_generic.rs | 2 +- concurrency/tests/ui/actor_handler_removed.rs | 17 +++++ .../tests/ui/actor_handler_removed.stderr | 13 ++++ .../ui/actor_send_handler_returns_value.rs | 17 +++++ .../actor_send_handler_returns_value.stderr | 13 ++++ .../ui/actor_two_handler_attributes.stderr | 2 +- docs/GUIDE.md | 2 +- docs/MIGRATION.md | 11 ++- .../src/bin/byte_budget_pipeline.rs | 6 +- .../src/bin/coordinated_shutdown.rs | 8 +-- .../capabilities/src/bin/dedicated_thread.rs | 2 +- .../capabilities/src/bin/execution_groups.rs | 4 +- .../src/bin/execution_profiles.rs | 2 +- .../capabilities/src/bin/framework_metrics.rs | 2 +- .../capabilities/src/bin/graceful_drain.rs | 4 +- .../capabilities/src/bin/native_lifecycle.rs | 4 +- .../capabilities/src/bin/protocol_timeouts.rs | 8 +-- .../src/bin/registrations_and_fakes.rs | 2 +- .../capabilities/src/bin/runtime_isolation.rs | 4 +- .../src/bin/shutdown_diagnostics.rs | 4 +- .../src/bin/streams_and_recipients.rs | 8 +-- .../src/bin/supervision_recovery.rs | 4 +- .../capabilities/src/bin/supervision_tree.rs | 2 +- .../capabilities/src/bin/timeout_outcomes.rs | 4 +- examples/exit_reason/src/main.rs | 10 +-- examples/thread_only/src/main.rs | 4 +- macros/src/lib.rs | 27 +++++++- rt/src/time.rs | 4 ++ testing/src/inbox.rs | 6 +- testing/src/lib.rs | 8 +-- testing/src/probe.rs | 3 + testing/src/shared.rs | 8 +-- testing/src/tasks.rs | 4 +- testing/src/threads.rs | 4 +- testing/tests/tasks_kit.rs | 6 +- testing/tests/threads_kit.rs | 2 +- 105 files changed, 599 insertions(+), 414 deletions(-) create mode 100644 concurrency/tests/ui/actor_handler_removed.rs create mode 100644 concurrency/tests/ui/actor_handler_removed.stderr create mode 100644 concurrency/tests/ui/actor_send_handler_returns_value.rs create mode 100644 concurrency/tests/ui/actor_send_handler_returns_value.stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index a36c8c24..18ec2b92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,19 @@ - `SupervisorConfig`, `Backoff` and `metrics::Config` are `#[non_exhaustive]` with builders (`new()` plus one setter per field; `Backoff::seed(u64)`). + - Hidden and sealed surface: `Receiver` is sealed (it gained a + required method in 0.8; sealing lets the next one be added freely). The + `reply` module is `#[doc(hidden)]` (its types are at the crate root and + under `tasks`/`threads`; macros emit root paths). Test hooks + (`ActorRef::__in_turn`, `PendingReply::__discharge_stopped`, + `reply_pair`, `PendingReply::detached`) move into + `#[doc(hidden)] __private`, which spawned-test uses and which carries no + semver promise. `lifecycle_trace` is public only with the + `lifecycle-trace` feature; `spawned_rt::time`'s tick functions are + `#[doc(hidden)]`. + - Macros: `#[handler]` is removed (use `#[request_handler]` or + `#[send_handler]`), and a `#[send_handler]` method returning a value is + a compile error (it used to answer a `request` with it). - Deferred replies and jobs, fixes from the 0.8 pre-release review: a job deadline still fires while the result waits for scheduled emits, and the job keeps its `MAX_OUTSTANDING` credit until its terminal message; a job diff --git a/README.md b/README.md index fdb2ade1..6c7f7c71 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ The return type on each protocol method determines the message kind: - `Result<(), ActorError>` — fire-and-forget send (both modes), returns the send result - No return / `-> ()` — fire-and-forget send (both modes), discards the send result -**Actors** implement message handlers with `#[actor]`. Each handler method is annotated with `#[request_handler]`, `#[send_handler]`, or `#[handler]` and receives a single message struct plus a `Context`. The macro generates the `Actor` trait impl and one `Handler` impl per method. +**Actors** implement message handlers with `#[actor]`. Each handler method is annotated with `#[request_handler]` (answers with its return value), `#[send_handler]` (returns `()`) or `#[deferred_handler]` (answers through its `PendingReply`, now or later) and receives a single message struct plus a `Context`. The macro generates the `Actor` trait impl and one `Handler` impl per method. **Using an actor** is just calling trait methods on its `ActorRef`. Because `#[protocol]` generates `impl NameServerProtocol for ActorRef`, you write `ns.add(...)` and `ns.find(...)` — the macro handles message construction and mailbox dispatch behind the scenes. diff --git a/concurrency/benches/baseline.rs b/concurrency/benches/baseline.rs index 994c534d..22ff4d74 100644 --- a/concurrency/benches/baseline.rs +++ b/concurrency/benches/baseline.rs @@ -102,7 +102,7 @@ impl tasks::Handler for Sink { async fn handle( &mut self, work: Work, - reply: spawned_concurrency::reply::PendingReply<'_, Ack>, + reply: spawned_concurrency::PendingReply<'_, Ack>, _: &tasks::Context, ) { reply.reply(process(work)); @@ -113,7 +113,7 @@ impl threads::Handler for Sink { fn handle( &mut self, work: Work, - reply: spawned_concurrency::reply::PendingReply<'_, Ack>, + reply: spawned_concurrency::PendingReply<'_, Ack>, _: &threads::Context, ) { reply.reply(process(work)); diff --git a/concurrency/benches/designs.rs b/concurrency/benches/designs.rs index c03883d8..45c9799c 100644 --- a/concurrency/benches/designs.rs +++ b/concurrency/benches/designs.rs @@ -125,7 +125,7 @@ impl tasks::Handler for Sink { async fn handle( &mut self, work: Work, - reply: spawned_concurrency::reply::PendingReply<'_, Ack>, + reply: spawned_concurrency::PendingReply<'_, Ack>, _: &tasks::Context, ) { let __reply_value: Ack = { @@ -166,7 +166,7 @@ impl threads::Handler for Sink { fn handle( &mut self, work: Work, - reply: spawned_concurrency::reply::PendingReply<'_, Ack>, + reply: spawned_concurrency::PendingReply<'_, Ack>, _: &threads::Context, ) { reply.reply(process(work)); @@ -203,7 +203,7 @@ impl tasks::Handler for Sink { async fn handle( &mut self, probe: Probe, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &tasks::Context, ) { { @@ -216,7 +216,7 @@ impl threads::Handler for Sink { fn handle( &mut self, probe: Probe, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &threads::Context, ) { { @@ -292,6 +292,9 @@ enum Target { enum Reply { Tasks(tokio::sync::oneshot::Receiver), Threads(mpsc::Receiver), + /// A framework request's receiver (its deadline is fixed at send). + FrameworkTasks(spawned_concurrency::ReplyReceiver), + FrameworkThreads(spawned_concurrency::ReplyReceiver), } impl Target { fn request(&self, work: Work, kind: &str) -> Reply { @@ -310,14 +313,14 @@ impl Target { return rx; } match self { - Self::Tasks(actor) => Reply::Tasks( + Self::Tasks(actor) => Reply::FrameworkTasks( actor - .send_request(work, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .send_request(work, Duration::from_secs(10)) .expect("actor stopped"), ), - Self::Threads(actor) => Reply::Threads( + Self::Threads(actor) => Reply::FrameworkThreads( actor - .send_request(work, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .send_request(work, Duration::from_secs(10)) .expect("actor stopped"), ), Self::Tokio(tx) => { @@ -336,6 +339,11 @@ impl Reply { return match self { Self::Tasks(rx) => futures::executor::block_on(rx).expect("reply dropped"), Self::Threads(rx) => rx.recv().expect("reply dropped"), + Self::FrameworkTasks(rx) => { + futures::executor::block_on(std::future::IntoFuture::into_future(rx)) + .expect("reply failed") + } + Self::FrameworkThreads(rx) => rx.recv().expect("reply failed"), }; } match self { @@ -348,6 +356,8 @@ impl Reply { Self::Threads(rx) => rx .recv_timeout(Duration::from_secs(10)) .expect("reply timeout/disconnected"), + Self::FrameworkTasks(rx) => rt.block_on(async { rx.await.expect("reply failed") }), + Self::FrameworkThreads(rx) => rx.recv().expect("reply failed"), } } } diff --git a/concurrency/benches/lifecycle_soak.rs b/concurrency/benches/lifecycle_soak.rs index a0cc5dea..716a2432 100644 --- a/concurrency/benches/lifecycle_soak.rs +++ b/concurrency/benches/lifecycle_soak.rs @@ -64,7 +64,7 @@ impl Handler for Worker { async fn handle( &mut self, msg: Step, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { @@ -96,7 +96,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { panic!("soak injected handler failure"); @@ -115,7 +115,7 @@ impl Handler for Watcher { async fn handle( &mut self, msg: Watch, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { for (child, panic) in msg.0 { @@ -128,7 +128,7 @@ impl Handler for Watcher { async fn handle( &mut self, msg: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { let panic = self @@ -151,7 +151,7 @@ impl Handler for Watcher { async fn handle( &mut self, _: Remaining, - reply: spawned_concurrency::reply::PendingReply<'_, usize>, + reply: spawned_concurrency::PendingReply<'_, usize>, _: &Context, ) { reply.reply(self.pending.len()); diff --git a/concurrency/benches/metrics_overhead.rs b/concurrency/benches/metrics_overhead.rs index 6cfecb08..7b67de52 100644 --- a/concurrency/benches/metrics_overhead.rs +++ b/concurrency/benches/metrics_overhead.rs @@ -21,7 +21,7 @@ impl tasks::Handler for Sink { async fn handle( &mut self, work: Work, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &tasks::Context, ) { reply.reply(compute(work)); @@ -32,7 +32,7 @@ impl threads::Handler for Sink { fn handle( &mut self, work: Work, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &threads::Context, ) { reply.reply(compute(work)); diff --git a/concurrency/benches/pinned_pipeline.rs b/concurrency/benches/pinned_pipeline.rs index 65d012b6..2e1378d6 100644 --- a/concurrency/benches/pinned_pipeline.rs +++ b/concurrency/benches/pinned_pipeline.rs @@ -52,7 +52,7 @@ mod linux { async fn handle( &mut self, m: Wire, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.next = m.0; @@ -72,7 +72,7 @@ mod linux { async fn handle( &mut self, mut m: Work, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { @@ -100,7 +100,7 @@ mod linux { async fn handle( &mut self, _: Audit, - reply: spawned_concurrency::reply::PendingReply<'_, (String, String, u64)>, + reply: spawned_concurrency::PendingReply<'_, (String, String, u64)>, _: &Context, ) { let __reply_value: (String, String, u64) = { diff --git a/concurrency/benches/runtime_isolation.rs b/concurrency/benches/runtime_isolation.rs index 2b1491e2..641b7c5e 100644 --- a/concurrency/benches/runtime_isolation.rs +++ b/concurrency/benches/runtime_isolation.rs @@ -49,7 +49,7 @@ mod linux { async fn handle( &mut self, m: Work, - reply: spawned_concurrency::reply::PendingReply<'_, (u64, u64, Instant)>, + reply: spawned_concurrency::PendingReply<'_, (u64, u64, Instant)>, _: &Context, ) { let __reply_value: (u64, u64, Instant) = async { @@ -79,7 +79,7 @@ mod linux { async fn handle( &mut self, _: Audit, - reply: spawned_concurrency::reply::PendingReply<'_, (String, u64)>, + reply: spawned_concurrency::PendingReply<'_, (String, u64)>, _: &Context, ) { let __reply_value: (String, u64) = { diff --git a/concurrency/benches/workload_research.rs b/concurrency/benches/workload_research.rs index deba48c9..de88d1c9 100644 --- a/concurrency/benches/workload_research.rs +++ b/concurrency/benches/workload_research.rs @@ -1,4 +1,5 @@ //! Full actors, explicit DAG placement and independently scheduled offered load. +use std::future::IntoFuture as _; #[path = "support/args.rs"] mod cli; #[path = "support/profiling.rs"] @@ -233,7 +234,7 @@ impl Handler for Node { async fn handle( &mut self, msg: Wire, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { if std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").as_deref() == Ok("sharded") { @@ -257,7 +258,7 @@ impl Handler for Node { async fn handle( &mut self, msg: Route, - reply: spawned_concurrency::reply::PendingReply<'_, Ack>, + reply: spawned_concurrency::PendingReply<'_, Ack>, _: &Context, ) { let __reply_value: Ack = { @@ -372,7 +373,7 @@ struct Meta { } struct Pending { meta: Meta, - reply: tokio::sync::oneshot::Receiver, + reply: spawned_concurrency::ReplyReceiver, } fn sample( p: Meta, @@ -612,7 +613,8 @@ fn main() { }, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, ) - .unwrap(), + .unwrap() + .into_future(), ) .unwrap(); } @@ -628,7 +630,7 @@ fn main() { spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, ) .unwrap(); - runtime.block_on(reply).unwrap(); + runtime.block_on(reply.into_future()).unwrap(); } let done = Arc::new(AtomicBool::new(false)); let observed = Arc::new(AtomicU64::new(0)); @@ -822,7 +824,8 @@ fn main() { if let Some(tx) = &tx { tx.send(Pending { meta, reply }).unwrap(); } else { - let ack = futures::executor::block_on(reply).unwrap(); + let ack = + futures::executor::block_on(reply.into_future()).unwrap(); samples.push(sample( meta, Some(ack), @@ -882,7 +885,8 @@ fn main() { }, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, ) - .unwrap(), + .unwrap() + .into_future(), ) .unwrap(); } diff --git a/concurrency/examples/panic_abort.rs b/concurrency/examples/panic_abort.rs index 9a217fd1..3b862d13 100644 --- a/concurrency/examples/panic_abort.rs +++ b/concurrency/examples/panic_abort.rs @@ -46,7 +46,7 @@ mod native { fn handle( &mut self, _: Boom, - _reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { panic!("{PANIC_TEXT}"); @@ -74,7 +74,7 @@ mod tasks { async fn handle( &mut self, _: Boom, - _reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { panic!("{PANIC_TEXT}"); diff --git a/concurrency/src/lib.rs b/concurrency/src/lib.rs index deefc428..54927cb9 100644 --- a/concurrency/src/lib.rs +++ b/concurrency/src/lib.rs @@ -41,7 +41,7 @@ //! - No return type — fire-and-forget send //! //! **Actors** — `#[actor]` on an impl block generates `impl Actor` and `Handler` -//! impls from annotated methods (`#[request_handler]`, `#[send_handler]`, `#[handler]`). +//! impls from annotated methods (`#[request_handler]`, `#[send_handler]`, `#[deferred_handler]`). //! Lifecycle hooks use `#[started]` and `#[stopped]`. //! //! **Type-Erased References** — Each protocol generates an `XRef` type alias @@ -90,12 +90,19 @@ pub mod failpoints; pub mod child_handle; pub mod error; -#[cfg_attr(not(feature = "async"), allow(dead_code))] +// Test-only: public (and unstable) with the `lifecycle-trace` feature. +#[cfg(feature = "lifecycle-trace")] pub mod lifecycle_trace; +#[cfg(not(feature = "lifecycle-trace"))] +#[allow(dead_code)] +pub(crate) mod lifecycle_trace; pub mod link; pub mod message; pub mod monitor; pub mod registry; +// Reached through the crate root, `tasks` and `threads`; the module path +// is not part of the API. +#[doc(hidden)] pub mod reply; pub mod response; #[cfg(feature = "async")] @@ -133,6 +140,63 @@ mod native_link_regression_tests; mod actor_cleanup; mod mailbox; +/// Hooks for `spawned-test` and macro-generated code. Not part of the public +/// API: no semver promise, may change in any release. +#[doc(hidden)] +pub mod __private { + use crate::reply::{Deadline, PendingReply, ReplyReceiver}; + use crate::ActorError; + + /// Seals [`Receiver`](crate::tasks::Receiver): implemented by + /// `ActorRef`, `Context` and spawned-test's probe only. + pub trait Sealed {} + + /// A `PendingReply` not tied to an actor turn, and its receiver + /// (`spawned_test::reply_pair`). + pub fn reply_pair( + deadline: Deadline, + ) -> (PendingReply<'static, R>, ReplyReceiver) { + crate::reply::reply_pair(deadline) + } + + /// The `PendingReply` of a sent message (`spawned_test::detached_reply`). + pub fn detached_reply() -> PendingReply<'static, R> { + PendingReply::detached() + } + + /// Answer `ActorStopped` (a test double standing in for an absent actor). + pub fn discharge_stopped(reply: PendingReply<'_, R>) { + reply.discharge_stopped() + } + + /// Run `f` in one of the actor's turns (`spawned_test::tasks::in_actor`). + #[cfg(feature = "async")] + pub fn in_turn( + actor: &crate::tasks::ActorRef, + f: F, + ) -> Result, ActorError> + where + A: crate::tasks::Actor, + R: Send + 'static, + F: FnOnce(&mut A, &crate::tasks::Context) -> R + Send + 'static, + { + actor.in_turn(f) + } + + /// Run `f` in one of the actor's turns (`spawned_test::threads::in_actor`). + pub fn in_turn_threads( + actor: &crate::threads::ActorRef, + f: F, + ) -> Result, ActorError> + where + A: crate::threads::Actor, + R: Send + 'static, + F: FnOnce(&mut A, &crate::threads::Context) -> R + Send + 'static, + { + actor.in_turn(f) + } +} + mod instrumentation; #[cfg(feature = "metrics")] pub mod metrics; diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 626f96ff..a0436a80 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -284,9 +284,8 @@ impl<'t, R: Send + 'static> PendingReply<'t, R> { } /// A `PendingReply` for a sent message: there is no caller. For tests - /// calling a handler by hand, use `spawned_test::detached_reply()`. - #[doc(hidden)] - pub fn detached() -> Self { + /// calling a handler by hand, `spawned_test::detached_reply()`. + pub(crate) fn detached() -> Self { Self { slot: None, registry: None, @@ -356,8 +355,7 @@ impl<'t, R: Send + 'static> PendingReply<'t, R> { /// Answer `ActorStopped` (test doubles standing in for an absent actor: /// without an owning actor the reason is `NoProc`). - #[doc(hidden)] - pub fn __discharge_stopped(mut self) { + pub(crate) fn discharge_stopped(mut self) { if let Some(slot) = self.slot.take() { slot.terminate(Terminal::Stopped); } @@ -698,8 +696,7 @@ impl std::future::Future for ReplyFuture { /// A `PendingReply` not tied to an actor turn, and its receiver, for calling /// a manual `Handler::handle` directly in tests (`spawned_test::reply_pair`). /// Await the receiver only after driving the turns the reply depends on. -#[doc(hidden)] -pub fn reply_pair( +pub(crate) fn reply_pair( deadline: Deadline, ) -> (PendingReply<'static, R>, ReplyReceiver) { let (mut handle, receiver) = reply_slot(deadline, None); diff --git a/concurrency/src/response.rs b/concurrency/src/response.rs index 2d09af4d..87f31a6d 100644 --- a/concurrency/src/response.rs +++ b/concurrency/src/response.rs @@ -48,7 +48,7 @@ enum ResponseState { /// Concrete wrapper for protocol request-response methods that works in both /// `tasks` (async) and `threads` (blocking) modes. /// -/// - **Tasks mode**: wraps a [`ReplyReceiver`](crate::reply::ReplyReceiver); +/// - **Tasks mode**: wraps a [`ReplyReceiver`](crate::ReplyReceiver); /// `.await` returns `Result`. The request's deadline was fixed /// when it was sent. /// - **Threads mode**: wraps a pre-computed result; use `.unwrap()` / `.expect()` directly diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 1c51c934..7476906d 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -1112,6 +1112,9 @@ impl Context { } } +impl crate::__private::Sealed for Context {} +impl crate::__private::Sealed for ActorRef {} + // Bridge: Context implements Receiver for any M that A handles impl Receiver for Context where @@ -1138,8 +1141,9 @@ where /// Object-safe trait for sending a single message type to an actor. /// /// Implemented automatically by `ActorRef` and `Context` for any -/// message type that `A` handles. -pub trait Receiver: Send + Sync { +/// message type that `A` handles. Sealed: it cannot be implemented outside +/// spawned (a method can then be added without a breaking change). +pub trait Receiver: Send + Sync + crate::__private::Sealed { fn send(&self, msg: M) -> Result<(), ActorError>; /// Send a request without waiting (`gen_server:send_request/3`); its /// deadline is fixed now (the slot carries it). On a [`Recipient`], @@ -1278,9 +1282,9 @@ impl ActorRef { } /// Run `f` in one of the actor's turns, with its state and `Context`, - /// and receive its result. For tests: use `spawned_test::tasks::in_actor`. - #[doc(hidden)] - pub fn __in_turn(&self, f: F) -> Result, ActorError> + /// and receive its result. For tests: `spawned_test::tasks::in_actor` (through + /// `__private`). + pub(crate) fn in_turn(&self, f: F) -> Result, ActorError> where R: Send + 'static, F: FnOnce(&mut A, &Context) -> R + Send + 'static, diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 10f06995..15272c72 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -905,6 +905,9 @@ impl Context { } } +impl crate::__private::Sealed for Context {} +impl crate::__private::Sealed for ActorRef {} + // Bridge: Context implements Receiver for any M that A handles impl Receiver for Context where @@ -931,8 +934,9 @@ where /// Object-safe trait for sending a single message type to an actor. /// /// Implemented automatically by `ActorRef` and `Context` for any -/// message type that `A` handles. -pub trait Receiver: Send + Sync { +/// message type that `A` handles. Sealed: it cannot be implemented outside +/// spawned (a method can then be added without a breaking change). +pub trait Receiver: Send + Sync + crate::__private::Sealed { fn send(&self, msg: M) -> Result<(), ActorError>; /// Send a request without waiting (`gen_server:send_request/3`); its /// deadline is fixed now (the slot carries it). On a [`Recipient`], @@ -1066,9 +1070,9 @@ impl ActorRef { } /// Run `f` in one of the actor's turns, with its state and `Context`, - /// and receive its result. For tests: use `spawned_test::threads::in_actor`. - #[doc(hidden)] - pub fn __in_turn(&self, f: F) -> Result, ActorError> + /// and receive its result. For tests: `spawned_test::threads::in_actor` (through + /// `__private`). + pub(crate) fn in_turn(&self, f: F) -> Result, ActorError> where R: Send + 'static, F: FnOnce(&mut A, &Context) -> R + Send + 'static, diff --git a/concurrency/tests/actor_scenarios.rs b/concurrency/tests/actor_scenarios.rs index 33345c1b..d88414da 100644 --- a/concurrency/tests/actor_scenarios.rs +++ b/concurrency/tests/actor_scenarios.rs @@ -40,7 +40,7 @@ mod tasks { async fn handle( &mut self, command: Command, - reply: spawned_concurrency::reply::PendingReply<'_, Reply>, + reply: spawned_concurrency::PendingReply<'_, Reply>, ctx: &Context, ) { reply.reply(match command { @@ -65,7 +65,7 @@ mod tasks { async fn handle( &mut self, event: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { let _ = self.0.events.send(event); @@ -83,7 +83,7 @@ mod threads { fn handle( &mut self, command: Command, - reply: spawned_concurrency::reply::PendingReply<'_, Reply>, + reply: spawned_concurrency::PendingReply<'_, Reply>, ctx: &Context, ) { reply.reply(match command { @@ -108,7 +108,7 @@ mod threads { fn handle( &mut self, event: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { let _ = self.0.events.send(event); diff --git a/concurrency/tests/backend_experiments.rs b/concurrency/tests/backend_experiments.rs index bba0d989..846f4e1d 100644 --- a/concurrency/tests/backend_experiments.rs +++ b/concurrency/tests/backend_experiments.rs @@ -20,7 +20,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Probe, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { @@ -37,7 +37,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Nested, - reply: spawned_concurrency::reply::PendingReply<'_, bool>, + reply: spawned_concurrency::PendingReply<'_, bool>, _: &Context, ) { let __reply_value: bool = { @@ -82,7 +82,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Network, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { diff --git a/concurrency/tests/backend_resource_limits.rs b/concurrency/tests/backend_resource_limits.rs index 4c1602a4..bcba0b07 100644 --- a/concurrency/tests/backend_resource_limits.rs +++ b/concurrency/tests/backend_resource_limits.rs @@ -33,7 +33,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); diff --git a/concurrency/tests/borrowed_runtime.rs b/concurrency/tests/borrowed_runtime.rs index dc23b504..278a6187 100644 --- a/concurrency/tests/borrowed_runtime.rs +++ b/concurrency/tests/borrowed_runtime.rs @@ -23,7 +23,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Identity, - reply: spawned_concurrency::reply::PendingReply<'_, ThreadId>, + reply: spawned_concurrency::PendingReply<'_, ThreadId>, _: &Context, ) { reply.reply(std::thread::current().id()); @@ -33,7 +33,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Ambient, - reply: spawned_concurrency::reply::PendingReply<'_, ActorRef>, + reply: spawned_concurrency::PendingReply<'_, ActorRef>, _: &Context, ) { reply.reply(Worker.start()); diff --git a/concurrency/tests/canary_feedback.rs b/concurrency/tests/canary_feedback.rs index e796d87a..c875d274 100644 --- a/concurrency/tests/canary_feedback.rs +++ b/concurrency/tests/canary_feedback.rs @@ -36,7 +36,7 @@ impl Handler for Worker { async fn handle( &mut self, hold: Hold, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { let _ = hold.entered.send(()); @@ -48,7 +48,7 @@ impl Handler for Worker { async fn handle( &mut self, quit: Quit, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { ctx.stop_with_reason(quit.0); @@ -408,7 +408,7 @@ async fn panicked_turn_is_cleared_before_stopped_hook() { async fn handle( &mut self, _: Quit, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { panic!("diagnostic turn cleanup regression"); diff --git a/concurrency/tests/cancellation_safety.rs b/concurrency/tests/cancellation_safety.rs index a1f3576e..94c7ee68 100644 --- a/concurrency/tests/cancellation_safety.rs +++ b/concurrency/tests/cancellation_safety.rs @@ -197,7 +197,7 @@ impl Handler for Probe { async fn handle( &mut self, cmd: Cmd, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, ctx: &Context, ) { let __reply_value: u64 = { @@ -235,7 +235,7 @@ impl Handler for Probe { async fn handle( &mut self, _: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.counters.downs.fetch_add(1, Ordering::SeqCst); @@ -251,7 +251,7 @@ impl Handler for Probe { async fn handle( &mut self, _: bumper::Bump, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = diff --git a/concurrency/tests/cfg_gated_protocol.rs b/concurrency/tests/cfg_gated_protocol.rs index f85b271a..374818bd 100644 --- a/concurrency/tests/cfg_gated_protocol.rs +++ b/concurrency/tests/cfg_gated_protocol.rs @@ -26,7 +26,7 @@ impl Handler for MixedActor { async fn handle( &mut self, _msg: mixed_protocol::GetValue, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _ctx: &Context, ) { reply.reply(self.value); @@ -38,7 +38,7 @@ impl Handler for MixedActor { async fn handle( &mut self, _msg: mixed_protocol::TestOnlyIncrement, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _ctx: &Context, ) { let __reply_value: u64 = { diff --git a/concurrency/tests/cohort_trace.rs b/concurrency/tests/cohort_trace.rs index 22554d5b..61e7167f 100644 --- a/concurrency/tests/cohort_trace.rs +++ b/concurrency/tests/cohort_trace.rs @@ -18,7 +18,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { panic!("trace cohort failure"); @@ -273,7 +273,7 @@ impl Handler for MaybeStop { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { panic!("trace cohort failure"); diff --git a/concurrency/tests/context_exit.rs b/concurrency/tests/context_exit.rs index 9cb59432..817b883b 100644 --- a/concurrency/tests/context_exit.rs +++ b/concurrency/tests/context_exit.rs @@ -23,7 +23,7 @@ fn native_explicit_exit_is_cooperative_and_first_reason_wins() { fn handle( &mut self, exit: Exit, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { ctx.stop_with_reason(exit.0); @@ -59,7 +59,7 @@ async fn async_explicit_exit_is_cooperative_and_first_reason_wins() { async fn handle( &mut self, exit: Exit, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { ctx.stop_with_reason(exit.0); diff --git a/concurrency/tests/coverage_trace.rs b/concurrency/tests/coverage_trace.rs index 971f4a41..e511b26a 100644 --- a/concurrency/tests/coverage_trace.rs +++ b/concurrency/tests/coverage_trace.rs @@ -92,7 +92,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { panic!("injected child failure"); @@ -310,7 +310,7 @@ impl Handler for QuitInStarted { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { panic!("injected child failure"); diff --git a/concurrency/tests/cross_backend.rs b/concurrency/tests/cross_backend.rs index 82c9144d..38616a94 100644 --- a/concurrency/tests/cross_backend.rs +++ b/concurrency/tests/cross_backend.rs @@ -172,7 +172,7 @@ mod tasks_node { async fn handle( &mut self, cmd: Cmd, - reply: spawned_concurrency::reply::PendingReply<'_, Reply>, + reply: spawned_concurrency::PendingReply<'_, Reply>, ctx: &Context, ) { let __reply_value: Reply = async { @@ -241,7 +241,7 @@ mod tasks_node { async fn handle( &mut self, down: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.state.log.push(Obs::Down { @@ -256,7 +256,7 @@ mod tasks_node { async fn handle( &mut self, Tick(n): Tick, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { self.state.ticks_seen += 1; @@ -274,7 +274,7 @@ mod tasks_node { async fn handle( &mut self, _: Pulse, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { let __reply_value: () = async { @@ -331,7 +331,7 @@ mod threads_node { fn handle( &mut self, cmd: Cmd, - reply: spawned_concurrency::reply::PendingReply<'_, Reply>, + reply: spawned_concurrency::PendingReply<'_, Reply>, ctx: &Context, ) { let __reply_value: Reply = (|| { @@ -407,7 +407,7 @@ mod threads_node { fn handle( &mut self, down: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.state.log.push(Obs::Down { @@ -422,7 +422,7 @@ mod threads_node { fn handle( &mut self, Tick(n): Tick, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { self.state.ticks_seen += 1; @@ -440,7 +440,7 @@ mod threads_node { fn handle( &mut self, _: Pulse, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { let __reply_value: () = (|| { @@ -1368,7 +1368,7 @@ mod early_stop { async fn handle( &mut self, block: Block, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &spawned_concurrency::tasks::Context, ) { // Deliberately blocks the only async worker of the group. diff --git a/concurrency/tests/deadline_service.rs b/concurrency/tests/deadline_service.rs index d54fd817..4e31c990 100644 --- a/concurrency/tests/deadline_service.rs +++ b/concurrency/tests/deadline_service.rs @@ -73,23 +73,22 @@ fn deadline_service_child() { .unwrap(); runtime.block_on(async { let owner = Owner::default().start(); - owner - .__in_turn(|_, ctx| { - let opts = JobOptions::new().deadline(Duration::from_millis(20)); - ctx.pipe_blocking( - opts, - |job| { - let at = Instant::now() + Duration::from_millis(40); - job.emit_at_blocking(at, Bomb(true)).unwrap(); - std::thread::sleep(Duration::from_millis(150)); - }, - |_: JobOutcome<()>| Done, - ) - .unwrap(); - }) - .unwrap() - .await + spawned_concurrency::__private::in_turn(&owner, |_, ctx| { + let opts = JobOptions::new().deadline(Duration::from_millis(20)); + ctx.pipe_blocking( + opts, + |job| { + let at = Instant::now() + Duration::from_millis(40); + job.emit_at_blocking(at, Bomb(true)).unwrap(); + std::thread::sleep(Duration::from_millis(150)); + }, + |_: JobOutcome<()>| Done, + ) .unwrap(); + }) + .unwrap() + .await + .unwrap(); // The deadline (20 ms) took the notification; the emit is discarded // at 40 ms on the service thread and its destructor panics. tokio::time::sleep(Duration::from_millis(100)).await; @@ -105,24 +104,23 @@ fn deadline_service_child() { Err(_) => panic!("the deadline service stopped: a parked request never expired"), } // A job deadline still fires too. - let fired = other - .__in_turn(|_, ctx| { - let (tx, rx) = std::sync::mpsc::channel(); - let opts = JobOptions::new().deadline(Duration::from_millis(20)); - ctx.pipe_blocking( - opts, - |_| std::thread::sleep(Duration::from_millis(300)), - move |outcome: JobOutcome<()>| { - let _ = tx.send(outcome.is_err()); - Done - }, - ) - .unwrap(); - rx - }) - .unwrap() - .await + let fired = spawned_concurrency::__private::in_turn(&other, |_, ctx| { + let (tx, rx) = std::sync::mpsc::channel(); + let opts = JobOptions::new().deadline(Duration::from_millis(20)); + ctx.pipe_blocking( + opts, + |_| std::thread::sleep(Duration::from_millis(300)), + move |outcome: JobOutcome<()>| { + let _ = tx.send(outcome.is_err()); + Done + }, + ) .unwrap(); + rx + }) + .unwrap() + .await + .unwrap(); let deadline = tokio::task::spawn_blocking(move || fired.recv_timeout(Duration::from_secs(5))) .await diff --git a/concurrency/tests/death_boundary.rs b/concurrency/tests/death_boundary.rs index 88ecac82..fcb208df 100644 --- a/concurrency/tests/death_boundary.rs +++ b/concurrency/tests/death_boundary.rs @@ -181,31 +181,30 @@ mod tasks_mode { }; let (scheduled_tx, scheduled) = mpsc::channel(); runtime.block_on(async { - actor - .__in_turn(move |_, ctx| { - ctx.pipe( - JobOptions::new(), - move |job| async move { - let far = spawned_concurrency::Instant::now() + LIMIT * 10; - if held { - // The result waits for this emit: held. - job.emit_at(far, Queued(inert())).await.unwrap(); - scheduled_tx.send(()).unwrap(); - Some(payload) - } else { - job.emit_at(far, Waiting(payload)).await.unwrap(); - scheduled_tx.send(()).unwrap(); - job.cancelled().await; - None - } - }, - |_| Queued(inert()), - ) - .unwrap(); - }) - .unwrap() - .await + spawned_concurrency::__private::in_turn(&actor, move |_, ctx| { + ctx.pipe( + JobOptions::new(), + move |job| async move { + let far = spawned_concurrency::Instant::now() + LIMIT * 10; + if held { + // The result waits for this emit: held. + job.emit_at(far, Queued(inert())).await.unwrap(); + scheduled_tx.send(()).unwrap(); + Some(payload) + } else { + job.emit_at(far, Waiting(payload)).await.unwrap(); + scheduled_tx.send(()).unwrap(); + job.cancelled().await; + None + } + }, + |_| Queued(inert()), + ) .unwrap(); + }) + .unwrap() + .await + .unwrap(); }); scheduled.recv_timeout(LIMIT).unwrap(); let (unblock, blocked) = mpsc::channel(); diff --git a/concurrency/tests/dedicated_runtime.rs b/concurrency/tests/dedicated_runtime.rs index 57335d05..b8a52873 100644 --- a/concurrency/tests/dedicated_runtime.rs +++ b/concurrency/tests/dedicated_runtime.rs @@ -95,7 +95,7 @@ impl Handler for Probe { async fn handle( &mut self, operation: Inspect, - reply: spawned_concurrency::reply::PendingReply<'_, Observation>, + reply: spawned_concurrency::PendingReply<'_, Observation>, _: &Context, ) { let __reply_value: Observation = { @@ -158,7 +158,7 @@ impl Handler for Child { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { @@ -176,7 +176,7 @@ impl Handler for Probe { async fn handle( &mut self, _: Nested, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { @@ -202,7 +202,7 @@ impl Handler for Probe { async fn handle( &mut self, message: Exchange, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { diff --git a/concurrency/tests/deferred_api.rs b/concurrency/tests/deferred_api.rs index 4aeff771..08113ce1 100644 --- a/concurrency/tests/deferred_api.rs +++ b/concurrency/tests/deferred_api.rs @@ -48,7 +48,10 @@ mod tasks_mode { R: Send + 'static, F: FnOnce(&mut Owner, &Context) -> R + Send + 'static, { - actor.__in_turn(f).unwrap().await.unwrap() + spawned_concurrency::__private::in_turn(&actor, f) + .unwrap() + .await + .unwrap() } pub struct Owner { diff --git a/concurrency/tests/deferred_replies.rs b/concurrency/tests/deferred_replies.rs index d73e4eda..0c9cfbea 100644 --- a/concurrency/tests/deferred_replies.rs +++ b/concurrency/tests/deferred_replies.rs @@ -426,7 +426,7 @@ mod tasks_mode { /// waits. #[spawned_macros::sim_test] async fn closed_follows_a_shortened_deadline() { - let (reply, _rx) = spawned_concurrency::reply::reply_pair::(Deadline::Never); + let (reply, _rx) = spawned_concurrency::__private::reply_pair::(Deadline::Never); let token = std::sync::Arc::new(reply.defer().unwrap()); let watcher = { let token = token.clone(); @@ -527,9 +527,10 @@ mod tasks_mode { let mut parker = Parker::new(seen); let actor = Parker::new(Arc::new(Seen::default())).start(); let ctx = actor.context(); - let (reply, rx) = spawned_concurrency::reply::reply_pair::(Deadline::Never); + let (reply, rx) = spawned_concurrency::__private::reply_pair::(Deadline::Never); parker.handle(Park(5), reply, &ctx).await; - let (release, released) = spawned_concurrency::reply::reply_pair::(Deadline::Never); + let (release, released) = + spawned_concurrency::__private::reply_pair::(Deadline::Never); parker.handle(Release(1), release, &ctx).await; assert_eq!(released.await.unwrap(), 1); assert_eq!(rx.await.unwrap(), 6); diff --git a/concurrency/tests/execution_groups.rs b/concurrency/tests/execution_groups.rs index fe5eab65..0beb9d16 100644 --- a/concurrency/tests/execution_groups.rs +++ b/concurrency/tests/execution_groups.rs @@ -26,7 +26,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, String>, + reply: spawned_concurrency::PendingReply<'_, String>, _: &Context, ) { reply.reply( @@ -41,7 +41,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); @@ -364,7 +364,7 @@ impl Handler for FailedStartup { async fn handle( &mut self, msg: Queued, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.handled.fetch_add(1, Ordering::SeqCst); @@ -578,7 +578,7 @@ impl Handler for SnapshotHeld { async fn handle( &mut self, msg: Hold, - _: spawned_concurrency::reply::PendingReply<'_, ()>, + _: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { let _ = msg.0.send(()); @@ -625,7 +625,7 @@ impl Handler for HelperAware { async fn handle( &mut self, _: CheckAdmission, - reply: spawned_concurrency::reply::PendingReply<'_, bool>, + reply: spawned_concurrency::PendingReply<'_, bool>, ctx: &Context, ) { reply.reply(ctx.spawn_helper(async {}).is_err()); diff --git a/concurrency/tests/execution_profiles.rs b/concurrency/tests/execution_profiles.rs index ac146ab8..d963f1a9 100644 --- a/concurrency/tests/execution_profiles.rs +++ b/concurrency/tests/execution_profiles.rs @@ -25,7 +25,7 @@ impl Handler for Probe { async fn handle( &mut self, _: Audit, - reply: spawned_concurrency::reply::PendingReply<'_, ::Result>, + reply: spawned_concurrency::PendingReply<'_, ::Result>, _: &Context, ) { let __reply_value: ::Result = { @@ -46,7 +46,7 @@ impl Handler for Probe { async fn handle( &mut self, _: StartPeer, - reply: spawned_concurrency::reply::PendingReply<'_, ::Result>, + reply: spawned_concurrency::PendingReply<'_, ::Result>, ctx: &Context, ) { let __reply_value: ::Result = @@ -97,7 +97,7 @@ impl Handler for Probe { async fn handle( &mut self, msg: Block, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { tokio::task::block_in_place(|| { @@ -115,7 +115,7 @@ impl Handler for Probe { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { @@ -372,7 +372,7 @@ mod tls_join { async fn handle( &mut self, msg: Install, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { PROBE.with(|p| *p.borrow_mut() = Some(msg.0)); diff --git a/concurrency/tests/exit_signal_matrix.rs b/concurrency/tests/exit_signal_matrix.rs index b721b750..5dedc8f0 100644 --- a/concurrency/tests/exit_signal_matrix.rs +++ b/concurrency/tests/exit_signal_matrix.rs @@ -28,7 +28,7 @@ macro_rules! actors { struct Idle; impl Actor for Idle {} impl Handler for Idle { - $($asynchronous)? fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + $($asynchronous)? fn handle(&mut self, _: Ping, reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context) { reply.reply(()); } } @@ -39,12 +39,12 @@ macro_rules! actors { } } impl Handler for Observer { - $($asynchronous)? fn handle(&mut self, _: Ping, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + $($asynchronous)? fn handle(&mut self, _: Ping, reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context) { reply.reply(()); } } impl Handler for Observer { - $($asynchronous)? fn handle(&mut self, down: Down, reply: spawned_concurrency::reply::PendingReply<'_, ()>, _: &Context) { + $($asynchronous)? fn handle(&mut self, down: Down, reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context) { self.0.send(Notice::Down(down.reason)).unwrap(); reply.reply(()); diff --git a/concurrency/tests/failpoints.rs b/concurrency/tests/failpoints.rs index 64a1f4a9..0ec035a3 100644 --- a/concurrency/tests/failpoints.rs +++ b/concurrency/tests/failpoints.rs @@ -100,7 +100,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { panic!("injected child failure"); @@ -110,7 +110,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); @@ -672,7 +672,7 @@ impl Handler for Linker { async fn handle( &mut self, msg: LinkTo, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { ctx.link(&msg.0); @@ -683,7 +683,7 @@ impl Handler for Linker { async fn handle( &mut self, msg: Watch, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { ctx.monitor(&msg.0); @@ -694,7 +694,7 @@ impl Handler for Linker { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); @@ -704,7 +704,7 @@ impl Handler for Linker { async fn handle( &mut self, msg: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { let _ = self.downs.send(msg.reason); @@ -1176,7 +1176,7 @@ mod native { fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &NativeContext, ) { panic!("injected native failure"); @@ -1189,7 +1189,7 @@ mod native { fn handle( &mut self, msg: Hold, - reply: spawned_concurrency::reply::PendingReply<'_, u32>, + reply: spawned_concurrency::PendingReply<'_, u32>, _: &NativeContext, ) { msg.0.send(reply.defer().unwrap()).unwrap(); @@ -1199,7 +1199,7 @@ mod native { fn handle( &mut self, _: Ping, - _: spawned_concurrency::reply::PendingReply<'_, ()>, + _: spawned_concurrency::PendingReply<'_, ()>, _: &NativeContext, ) { } @@ -1260,7 +1260,7 @@ mod native { fn handle( &mut self, msg: LinkTo, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &NativeContext, ) { ctx.link(&msg.0); @@ -1315,7 +1315,7 @@ mod native { fn handle( &mut self, msg: LinkTo, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &NativeContext, ) { ctx.link(&msg.0); diff --git a/concurrency/tests/foreign_panics.rs b/concurrency/tests/foreign_panics.rs index a22ac4b6..978cdf00 100644 --- a/concurrency/tests/foreign_panics.rs +++ b/concurrency/tests/foreign_panics.rs @@ -185,7 +185,7 @@ mod task_actors { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0.on_crash(); @@ -196,7 +196,7 @@ mod task_actors { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { reply.reply(7); @@ -206,7 +206,7 @@ mod task_actors { async fn handle( &mut self, _: Gift, - reply: spawned_concurrency::reply::PendingReply<'_, Bomb>, + reply: spawned_concurrency::PendingReply<'_, Bomb>, _: &Context, ) { let __reply_value: Bomb = { @@ -220,7 +220,7 @@ mod task_actors { async fn handle( &mut self, _: Stash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0.stash.push(Bomb(Some(REPLY_PANIC))); @@ -245,7 +245,7 @@ mod thread_actors { fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0.on_crash(); @@ -256,7 +256,7 @@ mod thread_actors { fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { reply.reply(7); @@ -266,7 +266,7 @@ mod thread_actors { fn handle( &mut self, _: Gift, - reply: spawned_concurrency::reply::PendingReply<'_, Bomb>, + reply: spawned_concurrency::PendingReply<'_, Bomb>, _: &Context, ) { let __reply_value: Bomb = { @@ -280,7 +280,7 @@ mod thread_actors { fn handle( &mut self, _: Stash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0.stash.push(Bomb(Some(REPLY_PANIC))); diff --git a/concurrency/tests/full_actor_shards.rs b/concurrency/tests/full_actor_shards.rs index 3b2bc898..6aac1c9e 100644 --- a/concurrency/tests/full_actor_shards.rs +++ b/concurrency/tests/full_actor_shards.rs @@ -36,7 +36,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Thread, - reply: spawned_concurrency::reply::PendingReply<'_, String>, + reply: spawned_concurrency::PendingReply<'_, String>, _: &Context, ) { reply.reply( @@ -58,7 +58,7 @@ impl Handler for Worker { async fn handle( &mut self, msg: Hold, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { @@ -77,7 +77,7 @@ impl Handler for Worker { async fn handle( &mut self, msg: Nested, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { @@ -273,7 +273,7 @@ impl Handler for Worker { async fn handle( &mut self, msg: Freeze, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { let _ = msg.started.send(()); @@ -357,7 +357,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Tick, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { @@ -375,10 +375,7 @@ impl Handler for Worker { async fn handle( &mut self, _: SpawnBlockingChild, - reply: spawned_concurrency::reply::PendingReply< - '_, - spawned_concurrency::tasks::ActorRef, - >, + reply: spawned_concurrency::PendingReply<'_, spawned_concurrency::tasks::ActorRef>, _: &Context, ) { reply.reply( @@ -395,7 +392,7 @@ impl Handler for Worker { async fn handle( &mut self, msg: Helper, - reply: spawned_concurrency::reply::PendingReply<'_, tokio::task::JoinHandle>, + reply: spawned_concurrency::PendingReply<'_, tokio::task::JoinHandle>, _: &Context, ) { reply.reply(tokio::spawn(async move { @@ -444,7 +441,7 @@ impl Handler for Worker { async fn handle( &mut self, msg: InstallMonitor, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { msg.observer.monitor(&msg.target); @@ -455,7 +452,7 @@ impl Handler for Worker { async fn handle( &mut self, _: spawned_concurrency::Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0.lock().unwrap().push("down"); @@ -470,7 +467,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Events, - reply: spawned_concurrency::reply::PendingReply<'_, Vec<&'static str>>, + reply: spawned_concurrency::PendingReply<'_, Vec<&'static str>>, _: &Context, ) { reply.reply(self.0.lock().unwrap().clone()); diff --git a/concurrency/tests/future_layout.rs b/concurrency/tests/future_layout.rs index 3483134f..f9b7ac09 100644 --- a/concurrency/tests/future_layout.rs +++ b/concurrency/tests/future_layout.rs @@ -26,7 +26,7 @@ impl Handler for Worker { async fn handle( &mut self, _: probe_protocol::Notify, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); @@ -36,7 +36,7 @@ impl Handler for Worker { async fn handle( &mut self, query: probe_protocol::Query, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { reply.reply(query.value); diff --git a/concurrency/tests/kill_links.rs b/concurrency/tests/kill_links.rs index 8e8eaf59..729347ea 100644 --- a/concurrency/tests/kill_links.rs +++ b/concurrency/tests/kill_links.rs @@ -25,7 +25,7 @@ fn native_propagated_kill_is_trappable_but_direct_kill_is_not() { fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, u8>, + reply: spawned_concurrency::PendingReply<'_, u8>, _: &Context, ) { reply.reply(7); @@ -86,7 +86,7 @@ async fn async_propagated_kill_is_trappable_but_direct_kill_is_not() { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, u8>, + reply: spawned_concurrency::PendingReply<'_, u8>, _: &Context, ) { reply.reply(7); diff --git a/concurrency/tests/leaks.rs b/concurrency/tests/leaks.rs index e3822ee3..95aef408 100644 --- a/concurrency/tests/leaks.rs +++ b/concurrency/tests/leaks.rs @@ -67,7 +67,7 @@ impl threads::Handler for NativeProbe { fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &threads::Context, ) { reply.reply(()); @@ -112,7 +112,7 @@ impl threads::Handler>> for NativeA { fn handle( &mut self, msg: SetPeer>, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &threads::Context, ) { self.peer = Some(msg.0); @@ -123,7 +123,7 @@ impl threads::Handler>> for NativeB { fn handle( &mut self, msg: SetPeer>, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &threads::Context, ) { self.peer = Some(msg.0); @@ -221,7 +221,7 @@ mod async_leaks { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); @@ -235,7 +235,7 @@ mod async_leaks { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { @@ -247,7 +247,7 @@ mod async_leaks { async fn handle( &mut self, _: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); @@ -342,7 +342,7 @@ mod async_leaks { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); @@ -352,7 +352,7 @@ mod async_leaks { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); @@ -362,7 +362,7 @@ mod async_leaks { async fn handle( &mut self, msg: SetPeer<(ActorRef, tasks::Recipient)>, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.peer = Some(msg.0 .0); @@ -374,7 +374,7 @@ mod async_leaks { async fn handle( &mut self, msg: SetPeer<(ActorRef, tasks::Recipient)>, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.peer = Some(msg.0 .0); @@ -628,7 +628,7 @@ mod sim_leaks { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); @@ -638,7 +638,7 @@ mod sim_leaks { async fn handle( &mut self, _: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); @@ -661,7 +661,7 @@ mod sim_leaks { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); @@ -671,7 +671,7 @@ mod sim_leaks { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); @@ -681,7 +681,7 @@ mod sim_leaks { async fn handle( &mut self, msg: SetPeer<(tasks::ActorRef, tasks::Recipient)>, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.peer = Some(msg.0); @@ -692,7 +692,7 @@ mod sim_leaks { async fn handle( &mut self, msg: SetPeer<(tasks::ActorRef, tasks::Recipient)>, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.peer = Some(msg.0); @@ -830,7 +830,7 @@ mod sim_leaks { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { panic!("injected failure for leak test"); @@ -941,7 +941,7 @@ mod sim_leaks { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { panic!("injected failure for leak test"); diff --git a/concurrency/tests/mailbox_drop_panic.rs b/concurrency/tests/mailbox_drop_panic.rs index 2fce7ed1..ad1e3349 100644 --- a/concurrency/tests/mailbox_drop_panic.rs +++ b/concurrency/tests/mailbox_drop_panic.rs @@ -45,7 +45,7 @@ impl Handler for Victim { async fn handle( &mut self, msg: Hold, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { msg.0.send(()).unwrap(); @@ -58,7 +58,7 @@ impl Handler for Victim { async fn handle( &mut self, _: Bomb, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0.fetch_add(1, Ordering::SeqCst); @@ -79,7 +79,7 @@ impl Handler for Observer { async fn handle( &mut self, down: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0.send((false, down.reason)).unwrap(); @@ -90,7 +90,7 @@ impl Handler for Observer { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); diff --git a/concurrency/tests/metrics.rs b/concurrency/tests/metrics.rs index 4867e21f..3d95a8c3 100644 --- a/concurrency/tests/metrics.rs +++ b/concurrency/tests/metrics.rs @@ -42,7 +42,7 @@ impl Handler for Probe { async fn handle( &mut self, _: metrics_protocol::Query, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { reply.reply(9); @@ -52,7 +52,7 @@ impl Handler for Probe { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); @@ -62,7 +62,7 @@ impl Handler for Probe { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { panic!("metric panic regression"); @@ -72,7 +72,7 @@ impl Handler for Probe { async fn handle( &mut self, hold: Hold, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { let _ = hold.entered.send(()); @@ -258,7 +258,7 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { fn handle( &mut self, _: metrics_protocol::Query, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &ThreadContext, ) { reply.reply(9); @@ -268,7 +268,7 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &ThreadContext, ) { reply.reply(()); @@ -278,7 +278,7 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &ThreadContext, ) { panic!("native metric panic"); diff --git a/concurrency/tests/metrics_composable.rs b/concurrency/tests/metrics_composable.rs index 6e04c7ec..ee0e9b39 100644 --- a/concurrency/tests/metrics_composable.rs +++ b/concurrency/tests/metrics_composable.rs @@ -16,7 +16,7 @@ impl Handler for Probe { fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); diff --git a/concurrency/tests/metrics_counts_only.rs b/concurrency/tests/metrics_counts_only.rs index e3504150..01c06796 100644 --- a/concurrency/tests/metrics_counts_only.rs +++ b/concurrency/tests/metrics_counts_only.rs @@ -14,7 +14,7 @@ impl Handler for Probe { fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); diff --git a/concurrency/tests/metrics_sampling.rs b/concurrency/tests/metrics_sampling.rs index 3577fa7d..f7f96e31 100644 --- a/concurrency/tests/metrics_sampling.rs +++ b/concurrency/tests/metrics_sampling.rs @@ -15,7 +15,7 @@ impl Handler for Probe { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); diff --git a/concurrency/tests/native_dispatcher.rs b/concurrency/tests/native_dispatcher.rs index 035ab9f1..e0d5f6ec 100644 --- a/concurrency/tests/native_dispatcher.rs +++ b/concurrency/tests/native_dispatcher.rs @@ -19,7 +19,7 @@ impl Handler for Healthy { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { diff --git a/concurrency/tests/native_mailbox_drop.rs b/concurrency/tests/native_mailbox_drop.rs index 0d472dc9..543bf624 100644 --- a/concurrency/tests/native_mailbox_drop.rs +++ b/concurrency/tests/native_mailbox_drop.rs @@ -43,7 +43,7 @@ impl Handler for Victim { fn handle( &mut self, _: Bomb, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { @@ -60,7 +60,7 @@ impl Handler for Victim { fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { diff --git a/concurrency/tests/nested_supervision.rs b/concurrency/tests/nested_supervision.rs index d6891167..38a3a31e 100644 --- a/concurrency/tests/nested_supervision.rs +++ b/concurrency/tests/nested_supervision.rs @@ -43,7 +43,7 @@ impl Handler for Counter { async fn handle( &mut self, _: Next, - reply: spawned_concurrency::reply::PendingReply<'_, usize>, + reply: spawned_concurrency::PendingReply<'_, usize>, _: &Context, ) { let __reply_value: usize = { @@ -57,7 +57,7 @@ impl Handler for Counter { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { @@ -543,7 +543,7 @@ impl Handler for MaybeStop { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { diff --git a/concurrency/tests/otp_exit_signals.rs b/concurrency/tests/otp_exit_signals.rs index f61fd8a6..6709e90f 100644 --- a/concurrency/tests/otp_exit_signals.rs +++ b/concurrency/tests/otp_exit_signals.rs @@ -61,7 +61,7 @@ mod tasks_mode { )>, pub panic_in_stopped: bool, /// Deferred tokens stored by `Hold`. - pub held: Vec>, + pub held: Vec>, /// stopped() drops the stored tokens before waiting on `in_stopped`. pub drop_held_in_stopped: bool, } @@ -104,7 +104,7 @@ mod tasks_mode { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, u8>, + reply: spawned_concurrency::PendingReply<'_, u8>, _: &Context, ) { reply.reply(7); @@ -114,7 +114,7 @@ mod tasks_mode { async fn handle( &mut self, down: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.seen.downs.lock().unwrap().push(down); @@ -130,7 +130,7 @@ mod tasks_mode { async fn handle( &mut self, f: Do, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { (f.0)(ctx); @@ -146,7 +146,7 @@ mod tasks_mode { async fn handle( &mut self, _: Hold, - reply: spawned_concurrency::reply::PendingReply<'_, u8>, + reply: spawned_concurrency::PendingReply<'_, u8>, _: &Context, ) { self.held.push(reply.defer().unwrap()); @@ -161,7 +161,7 @@ mod tasks_mode { async fn handle( &mut self, _: Boom, - _reply: spawned_concurrency::reply::PendingReply<'_, u8>, + _reply: spawned_concurrency::PendingReply<'_, u8>, _: &Context, ) { panic!("boom in a request handler"); @@ -176,7 +176,7 @@ mod tasks_mode { async fn handle( &mut self, msg: Block, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { let _ = msg.0.await; @@ -512,7 +512,7 @@ mod tasks_mode { async fn handle( &mut self, _: Spawn, - reply: spawned_concurrency::reply::PendingReply<'_, ChildHandle>, + reply: spawned_concurrency::PendingReply<'_, ChildHandle>, ctx: &Context, ) { let child = Node::new(&self.child_seen, true).start_linked(ctx); @@ -563,7 +563,7 @@ mod tasks_mode { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, u8>, + reply: spawned_concurrency::PendingReply<'_, u8>, _: &Context, ) { reply.reply(7); @@ -633,7 +633,7 @@ mod tasks_mode { async fn handle( &mut self, _: Probe, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { self.0.lock().unwrap().push(("probe", ctx.exit_pending())); @@ -649,7 +649,7 @@ mod tasks_mode { async fn handle( &mut self, _: AwaitShutdown, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { let deadline = tokio::time::Instant::now() + Duration::from_secs(3); @@ -871,9 +871,7 @@ mod tasks_mode { use spawned_concurrency::tasks::ActorRef; use spawned_concurrency::ActorError; - async fn answered( - rx: spawned_concurrency::reply::ReplyReceiver, - ) -> Result + async fn answered(rx: spawned_concurrency::ReplyReceiver) -> Result where R: Send + 'static, { @@ -1267,7 +1265,7 @@ mod threads_mode { fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, u8>, + reply: spawned_concurrency::PendingReply<'_, u8>, _: &Context, ) { reply.reply(7); @@ -1319,8 +1317,8 @@ mod threads_mode { // admission while exiting and NoProc after D. mod caller_errors { use super::*; - use spawned_concurrency::reply::{PendingReply, ReplyReceiver, ReplyTo}; use spawned_concurrency::ActorError; + use spawned_concurrency::{PendingReply, ReplyReceiver, ReplyTo}; use std::sync::mpsc; #[derive(Default)] @@ -1484,7 +1482,7 @@ mod threads_mode { fn handle( &mut self, _: Probe, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { self.0.lock().unwrap().push(("probe", ctx.exit_pending())); @@ -1499,7 +1497,7 @@ mod threads_mode { fn handle( &mut self, msg: Block, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { let _ = msg.0.recv(); @@ -1517,7 +1515,7 @@ mod threads_mode { fn handle( &mut self, _: Spawn, - reply: spawned_concurrency::reply::PendingReply<'_, ActorRef>, + reply: spawned_concurrency::PendingReply<'_, ActorRef>, ctx: &Context, ) { reply.reply(Late(self.0.clone()).start_linked(ctx)); diff --git a/concurrency/tests/panic_safety.rs b/concurrency/tests/panic_safety.rs index b8fd1637..0edab0b2 100644 --- a/concurrency/tests/panic_safety.rs +++ b/concurrency/tests/panic_safety.rs @@ -200,7 +200,7 @@ mod tasks_actors { async fn handle( &mut self, _: Poke, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0.fault(Fault::SendHandler, "injected handler panic"); @@ -211,7 +211,7 @@ mod tasks_actors { async fn handle( &mut self, _: Ask, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { reply.reply(self.0.ask()); @@ -221,7 +221,7 @@ mod tasks_actors { async fn handle( &mut self, _: Payload, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); @@ -231,7 +231,7 @@ mod tasks_actors { async fn handle( &mut self, _: Gift, - reply: spawned_concurrency::reply::PendingReply<'_, Bomb>, + reply: spawned_concurrency::PendingReply<'_, Bomb>, _: &Context, ) { reply.reply(self.0.gift()); @@ -241,7 +241,7 @@ mod tasks_actors { async fn handle( &mut self, _: Hold, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0.gate.wait(); @@ -252,7 +252,7 @@ mod tasks_actors { async fn handle( &mut self, setup: Setup, - reply: spawned_concurrency::reply::PendingReply<'_, Option>, + reply: spawned_concurrency::PendingReply<'_, Option>, ctx: &Context, ) { let __reply_value: Option = async { @@ -271,7 +271,7 @@ mod tasks_actors { async fn handle( &mut self, _: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0.fault(Fault::DownHandler, "injected Down panic"); @@ -292,7 +292,7 @@ mod tasks_actors { async fn handle( &mut self, down: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0 @@ -305,7 +305,7 @@ mod tasks_actors { async fn handle( &mut self, setup: Setup, - reply: spawned_concurrency::reply::PendingReply<'_, Option>, + reply: spawned_concurrency::PendingReply<'_, Option>, ctx: &Context, ) { let __reply_value: Option = async { @@ -324,7 +324,7 @@ mod tasks_actors { async fn handle( &mut self, _: Ask, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { reply.reply(7); @@ -355,7 +355,7 @@ mod thread_actors { fn handle( &mut self, _: Poke, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0.fault(Fault::SendHandler, "injected handler panic"); @@ -366,7 +366,7 @@ mod thread_actors { fn handle( &mut self, _: Ask, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { reply.reply(self.0.ask()); @@ -376,7 +376,7 @@ mod thread_actors { fn handle( &mut self, _: Payload, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); @@ -386,7 +386,7 @@ mod thread_actors { fn handle( &mut self, _: Gift, - reply: spawned_concurrency::reply::PendingReply<'_, Bomb>, + reply: spawned_concurrency::PendingReply<'_, Bomb>, _: &Context, ) { reply.reply(self.0.gift()); @@ -396,7 +396,7 @@ mod thread_actors { fn handle( &mut self, _: Hold, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0.gate.wait(); @@ -407,7 +407,7 @@ mod thread_actors { fn handle( &mut self, setup: Setup, - reply: spawned_concurrency::reply::PendingReply<'_, Option>, + reply: spawned_concurrency::PendingReply<'_, Option>, ctx: &Context, ) { let __reply_value: Option = (|| { @@ -425,7 +425,7 @@ mod thread_actors { fn handle( &mut self, _: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0.fault(Fault::DownHandler, "injected Down panic"); @@ -446,7 +446,7 @@ mod thread_actors { fn handle( &mut self, down: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0 @@ -459,7 +459,7 @@ mod thread_actors { fn handle( &mut self, setup: Setup, - reply: spawned_concurrency::reply::PendingReply<'_, Option>, + reply: spawned_concurrency::PendingReply<'_, Option>, ctx: &Context, ) { let __reply_value: Option = (|| { @@ -477,7 +477,7 @@ mod thread_actors { fn handle( &mut self, _: Ask, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { reply.reply(7); diff --git a/concurrency/tests/process_failures.rs b/concurrency/tests/process_failures.rs index d1ff53c9..2967c30b 100644 --- a/concurrency/tests/process_failures.rs +++ b/concurrency/tests/process_failures.rs @@ -33,7 +33,7 @@ impl Handler for Fixture { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { diff --git a/concurrency/tests/protocol_timeouts.rs b/concurrency/tests/protocol_timeouts.rs index a470ed7b..5fc61f44 100644 --- a/concurrency/tests/protocol_timeouts.rs +++ b/concurrency/tests/protocol_timeouts.rs @@ -61,7 +61,7 @@ mod asynchronous { async fn handle( &mut self, msg: timed_protocol::Work, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { @@ -75,7 +75,7 @@ mod asynchronous { async fn handle( &mut self, msg: timed_protocol::Brief, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { @@ -89,7 +89,7 @@ mod asynchronous { async fn handle( &mut self, msg: legacy_protocol::Legacy, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { @@ -157,7 +157,7 @@ mod synchronous { fn handle( &mut self, msg: timed_protocol::Work, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { @@ -171,7 +171,7 @@ mod synchronous { fn handle( &mut self, msg: timed_protocol::Brief, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { @@ -210,7 +210,7 @@ mod synchronous { fn handle( &mut self, _: zero_protocol::Zero, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); @@ -220,7 +220,7 @@ mod synchronous { fn handle( &mut self, _: naming_protocol::Timeout, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { reply.reply(31); @@ -230,7 +230,7 @@ mod synchronous { fn handle( &mut self, _: naming_protocol::ActorRef, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { reply.reply(32); @@ -240,7 +240,7 @@ mod synchronous { fn handle( &mut self, _: naming_protocol::TimeoutOverride, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { reply.reply(33); @@ -250,7 +250,7 @@ mod synchronous { fn handle( &mut self, _: naming_protocol::Dispatch0, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { reply.reply(34); @@ -275,7 +275,7 @@ mod synchronous { fn handle( &mut self, msg: owned_protocol::Owned, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { reply.reply(msg.__spawned_request_timeout.value); diff --git a/concurrency/tests/resource_cleanup.rs b/concurrency/tests/resource_cleanup.rs index c3be4ec3..6ae8415b 100644 --- a/concurrency/tests/resource_cleanup.rs +++ b/concurrency/tests/resource_cleanup.rs @@ -18,7 +18,7 @@ impl tasks::Handler for Sink { async fn handle( &mut self, msg: Probe, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &tasks::Context, ) { drop(msg.0); @@ -30,7 +30,7 @@ impl threads::Handler for Sink { fn handle( &mut self, msg: Probe, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &threads::Context, ) { drop(msg.0); diff --git a/concurrency/tests/semantics_gaps.rs b/concurrency/tests/semantics_gaps.rs index 87e7792f..b65aefaa 100644 --- a/concurrency/tests/semantics_gaps.rs +++ b/concurrency/tests/semantics_gaps.rs @@ -109,7 +109,7 @@ mod native { fn handle( &mut self, watch: Watch, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { ctx.trap_exit(true); @@ -122,7 +122,7 @@ mod native { fn handle( &mut self, ping: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { let _ = self.0.send(Event::Ping(ping.0)); @@ -133,7 +133,7 @@ mod native { fn handle( &mut self, down: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { let _ = self.0.send(Event::Down(down.reason)); @@ -147,7 +147,7 @@ mod native { fn handle( &mut self, burst: Burst>, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { for i in 0..burst.count { @@ -195,7 +195,7 @@ mod native { fn handle( &mut self, _: Boom, - _reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { panic!("request handler panic"); @@ -268,7 +268,7 @@ mod tasks { async fn handle( &mut self, watch: Watch, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { ctx.trap_exit(true); @@ -281,7 +281,7 @@ mod tasks { async fn handle( &mut self, ping: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { let _ = self.0.send(Event::Ping(ping.0)); @@ -292,7 +292,7 @@ mod tasks { async fn handle( &mut self, down: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { let _ = self.0.send(Event::Down(down.reason)); @@ -306,7 +306,7 @@ mod tasks { async fn handle( &mut self, burst: Burst>, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { for i in 0..burst.count { @@ -364,7 +364,7 @@ mod tasks { async fn handle( &mut self, _: Tick, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { let _ = self.0.send(Instant::now()); @@ -454,7 +454,7 @@ mod tasks { async fn handle( &mut self, stamp: Stamp, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { let _ = self.log.send((self.generation, stamp.0)); @@ -465,7 +465,7 @@ mod tasks { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { @@ -545,7 +545,7 @@ mod tasks { async fn handle( &mut self, _: Boom, - _reply: spawned_concurrency::reply::PendingReply<'_, u64>, + _reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { panic!("request handler panic"); diff --git a/concurrency/tests/shutdown_publication.rs b/concurrency/tests/shutdown_publication.rs index c8751c1d..1aaf4c40 100644 --- a/concurrency/tests/shutdown_publication.rs +++ b/concurrency/tests/shutdown_publication.rs @@ -69,7 +69,7 @@ impl Handler for Worker { async fn handle( &mut self, msg: Command, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { match msg { diff --git a/concurrency/tests/sim.rs b/concurrency/tests/sim.rs index a3cc1bac..d7500877 100644 --- a/concurrency/tests/sim.rs +++ b/concurrency/tests/sim.rs @@ -91,7 +91,7 @@ impl Handler for Node { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { panic!("injected crash after readiness"); diff --git a/concurrency/tests/sim_explore.rs b/concurrency/tests/sim_explore.rs index e4bf78e6..4d5e4c6a 100644 --- a/concurrency/tests/sim_explore.rs +++ b/concurrency/tests/sim_explore.rs @@ -59,7 +59,7 @@ impl Handler for Store { async fn handle( &mut self, _: Get, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { @@ -73,7 +73,7 @@ impl Handler for Store { async fn handle( &mut self, Set(value): Set, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.log.push(Access::Write(value)); @@ -85,7 +85,7 @@ impl Handler for Store { async fn handle( &mut self, _: Log, - reply: spawned_concurrency::reply::PendingReply<'_, Vec>, + reply: spawned_concurrency::PendingReply<'_, Vec>, _: &Context, ) { reply.reply(self.log.clone()); @@ -109,7 +109,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Bump, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { for _ in 0..self.noise { diff --git a/concurrency/tests/stack_budget.rs b/concurrency/tests/stack_budget.rs index 11d16d0b..6dd65ac5 100644 --- a/concurrency/tests/stack_budget.rs +++ b/concurrency/tests/stack_budget.rs @@ -123,7 +123,7 @@ mod tasks_chain { async fn handle( &mut self, msg: relay::Forward, - reply: spawned_concurrency::reply::PendingReply<'_, Block>, + reply: spawned_concurrency::PendingReply<'_, Block>, _: &Context, ) { let __reply_value: Block = { @@ -229,7 +229,7 @@ mod tasks_chain { async fn handle( &mut self, _: Crash, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { if self.generation == 1 { @@ -242,7 +242,7 @@ mod tasks_chain { async fn handle( &mut self, msg: relay::Forward, - reply: spawned_concurrency::reply::PendingReply<'_, Block>, + reply: spawned_concurrency::PendingReply<'_, Block>, _: &Context, ) { reply.reply(msg.block); @@ -316,7 +316,7 @@ mod threads_chain { fn handle( &mut self, msg: relay::Forward, - reply: spawned_concurrency::reply::PendingReply<'_, Block>, + reply: spawned_concurrency::PendingReply<'_, Block>, _: &Context, ) { let __reply_value: Block = { diff --git a/concurrency/tests/startup_admission.rs b/concurrency/tests/startup_admission.rs index 20cc4c02..158775e3 100644 --- a/concurrency/tests/startup_admission.rs +++ b/concurrency/tests/startup_admission.rs @@ -36,7 +36,7 @@ impl spawned_concurrency::tasks::Handler for Probe { async fn handle( &mut self, _: Queued, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { diff --git a/concurrency/tests/supervision.rs b/concurrency/tests/supervision.rs index 7ca256e9..ce95d759 100644 --- a/concurrency/tests/supervision.rs +++ b/concurrency/tests/supervision.rs @@ -41,7 +41,7 @@ impl Handler for Counter { async fn handle( &mut self, _: Next, - reply: spawned_concurrency::reply::PendingReply<'_, usize>, + reply: spawned_concurrency::PendingReply<'_, usize>, _: &Context, ) { let __reply_value: usize = { @@ -55,7 +55,7 @@ impl Handler for Counter { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { @@ -314,7 +314,7 @@ impl Handler for SlowHelper { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { @@ -392,7 +392,7 @@ impl Handler for SlowStop { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { @@ -594,7 +594,7 @@ impl Handler for RaceActor { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { @@ -606,7 +606,7 @@ impl Handler for RaceActor { async fn handle( &mut self, _: Next, - reply: spawned_concurrency::reply::PendingReply<'_, usize>, + reply: spawned_concurrency::PendingReply<'_, usize>, _: &Context, ) { let __reply_value: usize = { @@ -899,7 +899,7 @@ impl Handler for ExitOnceDuringRestart { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { @@ -1385,7 +1385,7 @@ impl Handler for Named { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { diff --git a/concurrency/tests/supervision_strategies.rs b/concurrency/tests/supervision_strategies.rs index dd92266f..6c4c07a1 100644 --- a/concurrency/tests/supervision_strategies.rs +++ b/concurrency/tests/supervision_strategies.rs @@ -73,7 +73,7 @@ impl Handler for Probe { async fn handle( &mut self, _: Next, - reply: spawned_concurrency::reply::PendingReply<'_, usize>, + reply: spawned_concurrency::PendingReply<'_, usize>, _: &Context, ) { let __reply_value: usize = { @@ -87,7 +87,7 @@ impl Handler for Probe { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { @@ -99,7 +99,7 @@ impl Handler for Probe { async fn handle( &mut self, msg: CrashTogether, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { @@ -797,7 +797,7 @@ impl Handler for MaybeStop { async fn handle( &mut self, _: CrashPlain, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { diff --git a/concurrency/tests/supervision_trace.rs b/concurrency/tests/supervision_trace.rs index 730c8a71..7c1fa490 100644 --- a/concurrency/tests/supervision_trace.rs +++ b/concurrency/tests/supervision_trace.rs @@ -99,7 +99,7 @@ impl Handler for Worker { async fn handle( &mut self, msg: HoldForDrain, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { msg.0.send(()).unwrap(); @@ -120,7 +120,7 @@ impl Handler for Worker { async fn handle( &mut self, _: DropBomb, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { panic!("discarded message must not be handled"); @@ -218,7 +218,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { panic!("injected child failure"); @@ -228,7 +228,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Quit, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { ctx.stop(); @@ -239,7 +239,7 @@ impl Handler for Worker { async fn handle( &mut self, msg: ExitWith, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { ctx.stop_with_reason(msg.0); @@ -624,7 +624,7 @@ impl Handler for ExitOnceDuringRestart { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { panic!("injected child failure"); diff --git a/concurrency/tests/supervisor_policies.rs b/concurrency/tests/supervisor_policies.rs index 20e671ae..c695bcc1 100644 --- a/concurrency/tests/supervisor_policies.rs +++ b/concurrency/tests/supervisor_policies.rs @@ -28,7 +28,7 @@ impl Handler for Probe { async fn handle( &mut self, _: Generation, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { reply.reply(self.0); @@ -38,7 +38,7 @@ impl Handler for Probe { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { diff --git a/concurrency/tests/support/failure_actor.rs b/concurrency/tests/support/failure_actor.rs index 9e080ed1..6a646945 100644 --- a/concurrency/tests/support/failure_actor.rs +++ b/concurrency/tests/support/failure_actor.rs @@ -159,7 +159,7 @@ impl spawned_concurrency::tasks::Handler for AsyncWorker { async fn handle( &mut self, command: Command, - reply: spawned_concurrency::reply::PendingReply<'_, Reply>, + reply: spawned_concurrency::PendingReply<'_, Reply>, _: &spawned_concurrency::tasks::Context, ) { let __reply_value: Reply = { @@ -196,7 +196,7 @@ impl spawned_concurrency::tasks::Handler for AsyncWorker { async fn handle( &mut self, down: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &spawned_concurrency::tasks::Context, ) { if let Some(sender) = &self.0.down { @@ -223,7 +223,7 @@ impl spawned_concurrency::threads::Handler for NativeWorker { fn handle( &mut self, command: Command, - reply: spawned_concurrency::reply::PendingReply<'_, Reply>, + reply: spawned_concurrency::PendingReply<'_, Reply>, _: &spawned_concurrency::threads::Context, ) { let __reply_value: Reply = { @@ -259,7 +259,7 @@ impl spawned_concurrency::threads::Handler for NativeWorker { fn handle( &mut self, down: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &spawned_concurrency::threads::Context, ) { if let Some(sender) = &self.0.down { diff --git a/concurrency/tests/support/store_actor.rs b/concurrency/tests/support/store_actor.rs index 520a1d89..7c8a27b1 100644 --- a/concurrency/tests/support/store_actor.rs +++ b/concurrency/tests/support/store_actor.rs @@ -115,7 +115,7 @@ impl spawned_concurrency::tasks::Handler for AsyncStore { async fn handle( &mut self, op: StoreOp, - reply: spawned_concurrency::reply::PendingReply<'_, Option>, + reply: spawned_concurrency::PendingReply<'_, Option>, _: &spawned_concurrency::tasks::Context, ) { // Yield inside the turn so concurrent requests really queue up. @@ -134,7 +134,7 @@ impl spawned_concurrency::threads::Handler for NativeStore { fn handle( &mut self, op: StoreOp, - reply: spawned_concurrency::reply::PendingReply<'_, Option>, + reply: spawned_concurrency::PendingReply<'_, Option>, _: &spawned_concurrency::threads::Context, ) { reply.reply(self.0.apply(op)); diff --git a/concurrency/tests/tree_trace.rs b/concurrency/tests/tree_trace.rs index 2283d85a..d71b75ca 100644 --- a/concurrency/tests/tree_trace.rs +++ b/concurrency/tests/tree_trace.rs @@ -23,7 +23,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { panic!("nested trace failure"); diff --git a/concurrency/tests/ui-pass/actor_tasks_lifecycle.rs b/concurrency/tests/ui-pass/actor_tasks_lifecycle.rs index 26f3fad1..3870b3e9 100644 --- a/concurrency/tests/ui-pass/actor_tasks_lifecycle.rs +++ b/concurrency/tests/ui-pass/actor_tasks_lifecycle.rs @@ -25,11 +25,11 @@ mod asynchronous { async fn halt(&mut self, _ctx: &Context) {} #[exit_received] async fn trapped(&mut self, _exit: Exit, _ctx: &Context) {} - #[handler] + #[request_handler] async fn ping(&mut self, msg: Ping, _ctx: &Context) -> u32 { msg.0 + 1 } - #[handler] + #[request_handler] fn pong( &mut self, msg: Pong, diff --git a/concurrency/tests/ui-pass/protocol_edge_cases.rs b/concurrency/tests/ui-pass/protocol_edge_cases.rs index 80493d82..b3f3a734 100644 --- a/concurrency/tests/ui-pass/protocol_edge_cases.rs +++ b/concurrency/tests/ui-pass/protocol_edge_cases.rs @@ -56,7 +56,7 @@ impl Store { fn on_callback(&mut self, msg: http_store::Callback, _: &Context) -> usize { (msg.f)("abc") } - #[handler] + #[request_handler] fn on_total(&mut self, _: http_store::Total, _: &Context) -> u64 { u64::from(self.total) } diff --git a/concurrency/tests/ui/actor_exit_with_handler.rs b/concurrency/tests/ui/actor_exit_with_handler.rs index b8cc0ebd..ec8a0d4e 100644 --- a/concurrency/tests/ui/actor_exit_with_handler.rs +++ b/concurrency/tests/ui/actor_exit_with_handler.rs @@ -12,7 +12,7 @@ use spawned_concurrency::Exit; #[actor] impl Counter { #[exit_received] - #[handler] + #[send_handler] fn on_exit(&mut self, _exit: Exit, _ctx: &Context) {} } fn main() {} diff --git a/concurrency/tests/ui/actor_handler_generic.rs b/concurrency/tests/ui/actor_handler_generic.rs index 5754546c..95359b4c 100644 --- a/concurrency/tests/ui/actor_handler_generic.rs +++ b/concurrency/tests/ui/actor_handler_generic.rs @@ -10,7 +10,7 @@ pub struct Counter(u64); use spawned_concurrency::threads::{Actor, Context, Handler}; #[actor] impl Counter { - #[handler] + #[send_handler] fn any>(&mut self, _msg: M, _ctx: &Context) {} } fn main() {} diff --git a/concurrency/tests/ui/actor_handler_removed.rs b/concurrency/tests/ui/actor_handler_removed.rs new file mode 100644 index 00000000..5fd32f5d --- /dev/null +++ b/concurrency/tests/ui/actor_handler_removed.rs @@ -0,0 +1,17 @@ +// `#[handler]` was removed in 0.8: it hid whether a method answers. +use spawned_concurrency::actor; +use spawned_concurrency::message::Message; +use spawned_concurrency::tasks::{Actor, Context, Handler}; +pub struct Ping; +impl Message for Ping { + type Result = u32; +} +pub struct Counter; +#[actor] +impl Counter { + #[handler] + async fn ping(&mut self, _msg: Ping, _ctx: &Context) -> u32 { + 1 + } +} +fn main() {} diff --git a/concurrency/tests/ui/actor_handler_removed.stderr b/concurrency/tests/ui/actor_handler_removed.stderr new file mode 100644 index 00000000..534dc6a9 --- /dev/null +++ b/concurrency/tests/ui/actor_handler_removed.stderr @@ -0,0 +1,13 @@ +error: #[handler] was removed in 0.8: use #[send_handler] (returns `()`, no reply) or #[request_handler] (answers with the return value) + --> tests/ui/actor_handler_removed.rs:12:5 + | +12 | #[handler] + | ^^^^^^^^^^ + +warning: unused imports: `Actor`, `Context`, and `Handler` + --> tests/ui/actor_handler_removed.rs:4:34 + | +4 | use spawned_concurrency::tasks::{Actor, Context, Handler}; + | ^^^^^ ^^^^^^^ ^^^^^^^ + | + = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default diff --git a/concurrency/tests/ui/actor_send_handler_returns_value.rs b/concurrency/tests/ui/actor_send_handler_returns_value.rs new file mode 100644 index 00000000..ebdef4ee --- /dev/null +++ b/concurrency/tests/ui/actor_send_handler_returns_value.rs @@ -0,0 +1,17 @@ +// A `#[send_handler]` returning a value would silently answer a request. +use spawned_concurrency::actor; +use spawned_concurrency::message::Message; +use spawned_concurrency::tasks::{Actor, Context, Handler}; +pub struct Ping; +impl Message for Ping { + type Result = u32; +} +pub struct Counter; +#[actor] +impl Counter { + #[send_handler] + async fn ping(&mut self, _msg: Ping, _ctx: &Context) -> u32 { + 1 + } +} +fn main() {} diff --git a/concurrency/tests/ui/actor_send_handler_returns_value.stderr b/concurrency/tests/ui/actor_send_handler_returns_value.stderr new file mode 100644 index 00000000..ccc8e259 --- /dev/null +++ b/concurrency/tests/ui/actor_send_handler_returns_value.stderr @@ -0,0 +1,13 @@ +error: #[send_handler] methods return `()`; use #[request_handler] to answer a request with a value + --> tests/ui/actor_send_handler_returns_value.rs:13:67 + | +13 | async fn ping(&mut self, _msg: Ping, _ctx: &Context) -> u32 { + | ^^^ + +warning: unused imports: `Actor`, `Context`, and `Handler` + --> tests/ui/actor_send_handler_returns_value.rs:4:34 + | +4 | use spawned_concurrency::tasks::{Actor, Context, Handler}; + | ^^^^^ ^^^^^^^ ^^^^^^^ + | + = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default diff --git a/concurrency/tests/ui/actor_two_handler_attributes.stderr b/concurrency/tests/ui/actor_two_handler_attributes.stderr index 3bc34613..e0013012 100644 --- a/concurrency/tests/ui/actor_two_handler_attributes.stderr +++ b/concurrency/tests/ui/actor_two_handler_attributes.stderr @@ -1,4 +1,4 @@ -error: a method can carry only one of #[handler], #[send_handler], #[request_handler] and #[deferred_handler] +error: a method can carry only one of #[send_handler], #[request_handler] and #[deferred_handler] --> tests/ui/actor_two_handler_attributes.rs:14:5 | 14 | #[request_handler] diff --git a/docs/GUIDE.md b/docs/GUIDE.md index b99e7377..bd788fb4 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -412,7 +412,7 @@ When you need to send a specific message type to an actor without knowing its co pub type Recipient = Arc>; ``` -`Receiver` is the object-safe trait that provides `send()` and `send_request()` for a single message type (on a `Recipient`, `send_request` takes a `Deadline`: `Deadline::after(d)` or `d.into()`). +`Receiver` is the object-safe trait that provides `send()` and `send_request()` for a single message type (on a `Recipient`, `send_request` takes a `Deadline`: `Deadline::after(d)` or `d.into()`). It is sealed: `ActorRef` and `Context` implement it, and test doubles come from `spawned_test`. ### Getting a Recipient diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 38f9ddbf..7b3d0bdb 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -48,6 +48,13 @@ every change; this guide covers what you need to edit when upgrading. | `SupervisorConfig { strategy: s, max_restarts: n, ..Default::default() }` | `SupervisorConfig::new().strategy(s).max_restarts(n)` (or assign fields of `SupervisorConfig::default()`) | | `Backoff { initial: a, jitter: false, seed: Some(7), ..Default::default() }` | `Backoff::new().initial(a).jitter(false).seed(7)` | | `metrics::Config { timing_sample_every: n }` | `metrics::Config::new().timing_sample_every(n)` | +| `#[handler]` on an `#[actor]` method | `#[request_handler]` (answers with the return value) or `#[send_handler]` | +| `#[send_handler]` method returning a value | `#[request_handler]` (a send handler must return `()`; it used to answer a `request` with the value) | +| `impl Receiver for MyType` | not possible: `Receiver` is sealed (use `ActorRef`/`Context`, or `spawned_test`'s probe) | +| `spawned_concurrency::reply::{PendingReply, ReplyTo, ..}` | `spawned_concurrency::{PendingReply, ReplyTo, ..}` (the `reply` module is hidden; jobs types are under `tasks::`/`threads::`) | +| `actor.__in_turn(f)`, `reply.__discharge_stopped()`, `reply::reply_pair` *(0.8 pre-release, hidden)* | `spawned_test::{tasks, threads}::in_actor`, `spawned_test::reply_pair()`, `spawned_test::detached_reply()` | +| `spawned_concurrency::lifecycle_trace` without the `lifecycle-trace` feature | only with the feature (test-only, no stability promise) | +| `spawned_rt::time::{to_ticks, from_ticks, now_ticks, MAX_TICKS}` | still public for spawned-concurrency, `#[doc(hidden)]`, no stability promise | ### Exit signals, startup and runtime drops as in OTP (ranks 5e, 5f, 5g, targets 0.8, in progress) @@ -769,9 +776,9 @@ The `#[actor]` macro supports three handler attributes: |-----------|---------| | `#[request_handler]` | Messages that expect a reply (`Response`) | | `#[send_handler]` | Fire-and-forget messages (no return or `-> ()`) | -| `#[handler]` | Generic — works for either kind | +| `#[handler]` | Generic — works for either kind (removed in 0.8) | -All three generate the same `impl Handler` code. The distinction is semantic: use `#[request_handler]` and `#[send_handler]` to document intent, or `#[handler]` when you don't want to commit to a specific kind. +In 0.5 to 0.7 all three generate the same `impl Handler` code. Since 0.8, `#[handler]` is removed and a `#[send_handler]` method must return `()` (see the 0.7 to 0.8 table). Handler signature: `fn name(&mut self, msg: MessageType, ctx: &Context) -> ReturnType` diff --git a/examples/capabilities/src/bin/byte_budget_pipeline.rs b/examples/capabilities/src/bin/byte_budget_pipeline.rs index a8aa6b1f..d53c6cf2 100644 --- a/examples/capabilities/src/bin/byte_budget_pipeline.rs +++ b/examples/capabilities/src/bin/byte_budget_pipeline.rs @@ -21,7 +21,7 @@ impl Handler for Sink { async fn handle( &mut self, packet: Packet, - reply: spawned_concurrency::reply::PendingReply<'_, usize>, + reply: spawned_concurrency::PendingReply<'_, usize>, _: &Context, ) { let __reply_value: usize = { @@ -43,7 +43,7 @@ impl Handler for Sink { async fn handle( &mut self, _: Total, - reply: spawned_concurrency::reply::PendingReply<'_, usize>, + reply: spawned_concurrency::PendingReply<'_, usize>, _: &Context, ) { reply.reply(self.0); @@ -55,7 +55,7 @@ impl Handler for Forwarder { async fn handle( &mut self, packet: Packet, - reply: spawned_concurrency::reply::PendingReply<'_, usize>, + reply: spawned_concurrency::PendingReply<'_, usize>, _: &Context, ) { reply.reply(self.0.request(packet).await.unwrap()); diff --git a/examples/capabilities/src/bin/coordinated_shutdown.rs b/examples/capabilities/src/bin/coordinated_shutdown.rs index fd2f1878..39369b48 100644 --- a/examples/capabilities/src/bin/coordinated_shutdown.rs +++ b/examples/capabilities/src/bin/coordinated_shutdown.rs @@ -44,7 +44,7 @@ impl Handler for Peers { async fn handle( &mut self, msg: Peer, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.peers.push(msg.0); @@ -55,7 +55,7 @@ impl Handler for Peers { async fn handle( &mut self, _: Snapshot, - reply: spawned_concurrency::reply::PendingReply<'_, Vec>, + reply: spawned_concurrency::PendingReply<'_, Vec>, _: &Context, ) { reply.reply(self.peers.clone()); @@ -79,7 +79,7 @@ impl Handler for Importer { async fn handle( &mut self, msg: Import, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.peers.send(Peer(msg.0)).unwrap(); @@ -90,7 +90,7 @@ impl Handler for Importer { async fn handle( &mut self, _: Fence, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); diff --git a/examples/capabilities/src/bin/dedicated_thread.rs b/examples/capabilities/src/bin/dedicated_thread.rs index f505f486..80c9990a 100644 --- a/examples/capabilities/src/bin/dedicated_thread.rs +++ b/examples/capabilities/src/bin/dedicated_thread.rs @@ -15,7 +15,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Check, - reply: spawned_concurrency::reply::PendingReply<'_, (usize, u64)>, + reply: spawned_concurrency::PendingReply<'_, (usize, u64)>, _: &Context, ) { let __reply_value: (usize, u64) = { diff --git a/examples/capabilities/src/bin/execution_groups.rs b/examples/capabilities/src/bin/execution_groups.rs index ab170220..34aa0305 100644 --- a/examples/capabilities/src/bin/execution_groups.rs +++ b/examples/capabilities/src/bin/execution_groups.rs @@ -15,7 +15,7 @@ impl Handler for Worker { async fn handle( &mut self, msg: Double, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { reply.reply(msg.0 * 2); @@ -27,7 +27,7 @@ impl Handler for Router { async fn handle( &mut self, msg: Double, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { reply.reply(self.0.request(msg).await.unwrap()); diff --git a/examples/capabilities/src/bin/execution_profiles.rs b/examples/capabilities/src/bin/execution_profiles.rs index 6ee9b96a..267671d8 100644 --- a/examples/capabilities/src/bin/execution_profiles.rs +++ b/examples/capabilities/src/bin/execution_profiles.rs @@ -15,7 +15,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Location, - reply: spawned_concurrency::reply::PendingReply<'_, ::Result>, + reply: spawned_concurrency::PendingReply<'_, ::Result>, _: &Context, ) { reply.reply(( diff --git a/examples/capabilities/src/bin/framework_metrics.rs b/examples/capabilities/src/bin/framework_metrics.rs index 076fddef..2e1d663b 100644 --- a/examples/capabilities/src/bin/framework_metrics.rs +++ b/examples/capabilities/src/bin/framework_metrics.rs @@ -14,7 +14,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); diff --git a/examples/capabilities/src/bin/graceful_drain.rs b/examples/capabilities/src/bin/graceful_drain.rs index e13fe4bf..139f93cc 100644 --- a/examples/capabilities/src/bin/graceful_drain.rs +++ b/examples/capabilities/src/bin/graceful_drain.rs @@ -13,7 +13,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Add, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0 += 1; @@ -28,7 +28,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Fence, - reply: spawned_concurrency::reply::PendingReply<'_, usize>, + reply: spawned_concurrency::PendingReply<'_, usize>, _: &Context, ) { reply.reply(self.0); diff --git a/examples/capabilities/src/bin/native_lifecycle.rs b/examples/capabilities/src/bin/native_lifecycle.rs index f57992bb..b371eae8 100644 --- a/examples/capabilities/src/bin/native_lifecycle.rs +++ b/examples/capabilities/src/bin/native_lifecycle.rs @@ -22,7 +22,7 @@ impl Handler for Observer { fn handle( &mut self, event: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.counts.1 += 1; @@ -38,7 +38,7 @@ impl Handler for Observer { fn handle( &mut self, _: Counts, - reply: spawned_concurrency::reply::PendingReply<'_, (usize, usize)>, + reply: spawned_concurrency::PendingReply<'_, (usize, usize)>, _: &Context, ) { reply.reply(self.counts); diff --git a/examples/capabilities/src/bin/protocol_timeouts.rs b/examples/capabilities/src/bin/protocol_timeouts.rs index aa2f6952..da788a38 100644 --- a/examples/capabilities/src/bin/protocol_timeouts.rs +++ b/examples/capabilities/src/bin/protocol_timeouts.rs @@ -26,7 +26,7 @@ mod asynchronous { async fn handle( &mut self, _: lookup_protocol::Value, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { @@ -40,7 +40,7 @@ mod asynchronous { async fn handle( &mut self, _: lookup_protocol::Quick, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { @@ -82,7 +82,7 @@ mod synchronous { fn handle( &mut self, _: lookup_protocol::Value, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { @@ -96,7 +96,7 @@ mod synchronous { fn handle( &mut self, _: lookup_protocol::Quick, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { diff --git a/examples/capabilities/src/bin/registrations_and_fakes.rs b/examples/capabilities/src/bin/registrations_and_fakes.rs index 861c1997..b0ae7fd6 100644 --- a/examples/capabilities/src/bin/registrations_and_fakes.rs +++ b/examples/capabilities/src/bin/registrations_and_fakes.rs @@ -40,7 +40,7 @@ impl Handler for Service { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { diff --git a/examples/capabilities/src/bin/runtime_isolation.rs b/examples/capabilities/src/bin/runtime_isolation.rs index 40876284..3dad79d3 100644 --- a/examples/capabilities/src/bin/runtime_isolation.rs +++ b/examples/capabilities/src/bin/runtime_isolation.rs @@ -14,7 +14,7 @@ impl Handler for Critical { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, String>, + reply: spawned_concurrency::PendingReply<'_, String>, _: &Context, ) { reply.reply(std::thread::current().name().unwrap().to_owned()); @@ -26,7 +26,7 @@ impl Handler for Router { async fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, String>, + reply: spawned_concurrency::PendingReply<'_, String>, _: &Context, ) { let __reply_value: String = { diff --git a/examples/capabilities/src/bin/shutdown_diagnostics.rs b/examples/capabilities/src/bin/shutdown_diagnostics.rs index 2dabf4ca..b8470c3e 100644 --- a/examples/capabilities/src/bin/shutdown_diagnostics.rs +++ b/examples/capabilities/src/bin/shutdown_diagnostics.rs @@ -19,7 +19,7 @@ impl Handler for Worker { async fn handle( &mut self, msg: Hold, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { let _ = msg.0.send(()); @@ -35,7 +35,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Unhealthy, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { // Cooperative abnormal exit, without a panic or forced interruption. diff --git a/examples/capabilities/src/bin/streams_and_recipients.rs b/examples/capabilities/src/bin/streams_and_recipients.rs index 8e79b280..919dde46 100644 --- a/examples/capabilities/src/bin/streams_and_recipients.rs +++ b/examples/capabilities/src/bin/streams_and_recipients.rs @@ -22,7 +22,7 @@ mod asynchronous { async fn handle( &mut self, msg: Item, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0 += N * msg.0; @@ -33,7 +33,7 @@ mod asynchronous { async fn handle( &mut self, _: Total, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { reply.reply(self.0); @@ -87,7 +87,7 @@ mod synchronous { fn handle( &mut self, msg: Item, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0 += N * msg.0; @@ -98,7 +98,7 @@ mod synchronous { fn handle( &mut self, _: Total, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { reply.reply(self.0); diff --git a/examples/capabilities/src/bin/supervision_recovery.rs b/examples/capabilities/src/bin/supervision_recovery.rs index f80578b9..288ec59b 100644 --- a/examples/capabilities/src/bin/supervision_recovery.rs +++ b/examples/capabilities/src/bin/supervision_recovery.rs @@ -20,7 +20,7 @@ impl Handler for Counter { async fn handle( &mut self, _: Next, - reply: spawned_concurrency::reply::PendingReply<'_, u64>, + reply: spawned_concurrency::PendingReply<'_, u64>, _: &Context, ) { let __reply_value: u64 = { @@ -34,7 +34,7 @@ impl Handler for Counter { async fn handle( &mut self, _: Crash, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { diff --git a/examples/capabilities/src/bin/supervision_tree.rs b/examples/capabilities/src/bin/supervision_tree.rs index 2bcc7ee1..82859127 100644 --- a/examples/capabilities/src/bin/supervision_tree.rs +++ b/examples/capabilities/src/bin/supervision_tree.rs @@ -22,7 +22,7 @@ impl Handler for Worker { async fn handle( &mut self, _: Fail, - _reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { diff --git a/examples/capabilities/src/bin/timeout_outcomes.rs b/examples/capabilities/src/bin/timeout_outcomes.rs index a6d08d56..fcf745a9 100644 --- a/examples/capabilities/src/bin/timeout_outcomes.rs +++ b/examples/capabilities/src/bin/timeout_outcomes.rs @@ -20,7 +20,7 @@ impl Handler for Counter { async fn handle( &mut self, msg: Apply, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { { @@ -39,7 +39,7 @@ impl Handler for Counter { async fn handle( &mut self, _: Read, - reply: spawned_concurrency::reply::PendingReply<'_, usize>, + reply: spawned_concurrency::PendingReply<'_, usize>, _: &Context, ) { reply.reply(self.0); diff --git a/examples/exit_reason/src/main.rs b/examples/exit_reason/src/main.rs index 64e2d1dc..865c55b7 100644 --- a/examples/exit_reason/src/main.rs +++ b/examples/exit_reason/src/main.rs @@ -97,7 +97,7 @@ fn main() { async fn handle( &mut self, _msg: Noop, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _ctx: &Context, ) { reply.reply(()); @@ -180,7 +180,7 @@ fn main() { async fn handle( &mut self, msg: StartMonitor, - reply: spawned_concurrency::reply::PendingReply<'_, MonitorRef>, + reply: spawned_concurrency::PendingReply<'_, MonitorRef>, ctx: &Context, ) { reply.reply(ctx.monitor(&msg.0)); @@ -190,7 +190,7 @@ fn main() { async fn handle( &mut self, msg: StopMonitor, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { ctx.demonitor(msg.0); @@ -201,7 +201,7 @@ fn main() { async fn handle( &mut self, msg: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _ctx: &Context, ) { tracing::info!( @@ -286,7 +286,7 @@ fn main() { async fn handle( &mut self, msg: LinkChild, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, ctx: &Context, ) { ctx.link(&msg.0); diff --git a/examples/thread_only/src/main.rs b/examples/thread_only/src/main.rs index 54bcf3bb..f8dc0748 100644 --- a/examples/thread_only/src/main.rs +++ b/examples/thread_only/src/main.rs @@ -35,7 +35,7 @@ impl Handler for Observer { fn handle( &mut self, _: Down, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0.fetch_add(1, Ordering::SeqCst); @@ -46,7 +46,7 @@ impl Handler for Observer { fn handle( &mut self, _: Tick, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0.fetch_add(1, Ordering::SeqCst); diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 7bfda59b..154cde5d 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1136,7 +1136,6 @@ fn is_actor_method_attribute(attr: &Attribute) -> bool { /// |-----------|---------| /// | `#[request_handler]` | Messages that expect a reply (`Response`) | /// | `#[send_handler]` | Fire-and-forget messages | -/// | `#[handler]` | Generic handler (works for either kind) | /// | `#[deferred_handler]` | Requests answered through a `PendingReply` (now, or later via `defer()`) | /// /// Handler signature: `fn name(&mut self, msg: MessageType, ctx: &Context) -> ReturnType` @@ -1461,11 +1460,20 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { if let Some(idx) = handler_idx { let deferred = method.attrs[idx].path().is_ident("deferred_handler"); + let send = method.attrs[idx].path().is_ident("send_handler"); + if method.attrs[idx].path().is_ident("handler") { + return syn::Error::new_spanned( + &method.attrs[idx], + "#[handler] was removed in 0.8: use #[send_handler] (returns `()`, no reply) or #[request_handler] (answers with the return value)", + ) + .to_compile_error() + .into(); + } method.attrs.remove(idx); if let Some(extra) = method.attrs.iter().find(|a| is_handler_attribute(a)) { return syn::Error::new_spanned( extra, - "a method can carry only one of #[handler], #[send_handler], #[request_handler] and #[deferred_handler]", + "a method can carry only one of #[send_handler], #[request_handler] and #[deferred_handler]", ) .to_compile_error() .into(); @@ -1533,6 +1541,21 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { } } } + // A send handler that returned a value would silently answer a + // `request` with it. (`impl Future` returns are checked by the + // `Result = ()` bound of the message instead.) + if send { + if let ReturnType::Type(_, ty) = &method.sig.output { + if !is_unit_type(ty) && !matches!(**ty, Type::ImplTrait(_)) { + return syn::Error::new_spanned( + ty, + "#[send_handler] methods return `()`; use #[request_handler] to answer a request with a value", + ) + .to_compile_error() + .into(); + } + } + } // `&self` is accepted too; a missing or by-value receiver is not. match method.sig.inputs.first() { diff --git a/rt/src/time.rs b/rt/src/time.rs index f3290eb3..390d2c4d 100644 --- a/rt/src/time.rs +++ b/rt/src/time.rs @@ -32,9 +32,11 @@ pub fn entropy() -> u64 { } } +#[doc(hidden)] // clock plumbing for spawned-concurrency; no stability promise /// Largest tick value an instant maps to; `u64::MAX` is reserved for "never". pub const MAX_TICKS: u64 = u64::MAX - 1; +#[doc(hidden)] // clock plumbing for spawned-concurrency; no stability promise /// Nanosecond clock ticks of `instant`, for packing deadlines into atomics. /// /// Ticks count from a process-wide epoch (the first call), or, under @@ -45,6 +47,7 @@ pub fn to_ticks(instant: Instant) -> u64 { nanos.min(MAX_TICKS as u128) as u64 } +#[doc(hidden)] // clock plumbing for spawned-concurrency; no stability promise /// Inverse of [`to_ticks`]. pub fn from_ticks(ticks: u64) -> Instant { let elapsed = std::time::Duration::from_nanos(ticks.min(MAX_TICKS)); @@ -59,6 +62,7 @@ pub fn from_ticks(ticks: u64) -> Instant { } } +#[doc(hidden)] // clock plumbing for spawned-concurrency; no stability promise /// Ticks of [`Instant::now`]. pub fn now_ticks() -> u64 { to_ticks(Instant::now()) diff --git a/testing/src/inbox.rs b/testing/src/inbox.rs index 99be74f5..b8054a30 100644 --- a/testing/src/inbox.rs +++ b/testing/src/inbox.rs @@ -137,7 +137,7 @@ impl Default for AsyncSignal { impl Inbox { pub(crate) fn store_deferred( &self, - token: spawned_concurrency::reply::ReplyTo, + token: spawned_concurrency::ReplyTo, ) { self.deferred .lock() @@ -146,9 +146,7 @@ impl Inbox { .or_default() .push(Box::new(token)); } - pub(crate) fn take_deferred( - &self, - ) -> Vec> { + pub(crate) fn take_deferred(&self) -> Vec> { let tokens = self .deferred .lock() diff --git a/testing/src/lib.rs b/testing/src/lib.rs index e0b98c97..1749120d 100644 --- a/testing/src/lib.rs +++ b/testing/src/lib.rs @@ -67,10 +67,10 @@ pub use inbox::{Outcome, Received}; /// driving every turn the reply depends on: a deferred reply answered by a /// later message is never answered while the test waits for it first. pub fn reply_pair() -> ( - spawned_concurrency::reply::PendingReply<'static, R>, - spawned_concurrency::reply::ReplyReceiver, + spawned_concurrency::PendingReply<'static, R>, + spawned_concurrency::ReplyReceiver, ) { - spawned_concurrency::reply::reply_pair(spawned_concurrency::reply::Deadline::Never) + spawned_concurrency::__private::reply_pair(spawned_concurrency::Deadline::Never) } /// The `PendingReply` a handler receives for a sent message (no caller), @@ -80,5 +80,5 @@ pub fn reply_pair() -> ( /// actor.handle(Tick, spawned_test::detached_reply(), &ctx).await; /// ``` pub fn detached_reply() -> spawned_concurrency::PendingReply<'static, R> { - spawned_concurrency::PendingReply::detached() + spawned_concurrency::__private::detached_reply() } diff --git a/testing/src/probe.rs b/testing/src/probe.rs index cb7bbf21..abffbb44 100644 --- a/testing/src/probe.rs +++ b/testing/src/probe.rs @@ -34,6 +34,9 @@ impl Drop for Replying<'_> { } } +// `Receiver` is sealed; the probe is the one implementor outside spawned. +impl spawned_concurrency::__private::Sealed for ProbeCore {} + impl ProbeCore { pub(crate) fn new(reply: Option>) -> Self { Self { diff --git a/testing/src/shared.rs b/testing/src/shared.rs index 2a6657ef..ba712441 100644 --- a/testing/src/shared.rs +++ b/testing/src/shared.rs @@ -129,9 +129,7 @@ macro_rules! mock_ref_common { /// Take the reply tokens of the `M` requests the mock deferred /// (`Mock::defer`), oldest first: reply later, drop them /// (`ReplyDropped`), or keep them unanswered. - pub fn deferred( - &self, - ) -> Vec> { + pub fn deferred(&self) -> Vec> { self.inbox.take_deferred::() } @@ -175,7 +173,7 @@ macro_rules! probe_common { msg: M, deadline: Deadline, ) -> Result, ActorError> { - let (reply, rx) = spawned_concurrency::reply::reply_pair(deadline); + let (reply, rx) = spawned_concurrency::__private::reply_pair(deadline); match self.request(msg)? { Some((value, record)) => { reply.reply(value); @@ -184,7 +182,7 @@ macro_rules! probe_common { } // An unexpected request is discharged as `ActorStopped` // (`NoProc`: no actor stands behind a probe). - None => reply.__discharge_stopped(), + None => spawned_concurrency::__private::discharge_stopped(reply), } Ok(rx) } diff --git a/testing/src/tasks.rs b/testing/src/tasks.rs index 1db211a0..854bdcee 100644 --- a/testing/src/tasks.rs +++ b/testing/src/tasks.rs @@ -15,13 +15,13 @@ use crate::rules::{Fence, Plan, Rules}; use crate::shared::SETTLE_TIMEOUT; use spawned_concurrency::error::ActorError; use spawned_concurrency::message::Message; -use spawned_concurrency::reply::{Deadline, PendingReply, ReplyReceiver}; use spawned_concurrency::supervision::{ChildState, SupervisedChild, Supervisor, SupervisorReport}; use spawned_concurrency::system::{ActorSystem, ExecutionGroup, ShutdownReport, SystemError}; use spawned_concurrency::tasks::{ Actor, ActorRef, ActorStart, Context, Handler, Receiver, Recipient, }; use spawned_concurrency::{ChildHandle, ExitReason}; +use spawned_concurrency::{Deadline, PendingReply, ReplyReceiver}; use spawned_rt::tasks::{sleep, timeout}; use std::future::Future; use std::ops::Deref; @@ -271,7 +271,7 @@ where { let at = Location::caller(); async move { - let reply = match actor.__in_turn(f) { + let reply = match spawned_concurrency::__private::in_turn(actor, f) { Ok(rx) => rx.await, Err(error) => Err(error), }; diff --git a/testing/src/threads.rs b/testing/src/threads.rs index fa850edd..9bd31245 100644 --- a/testing/src/threads.rs +++ b/testing/src/threads.rs @@ -6,11 +6,11 @@ use crate::rules::{Fence, Plan, Rules}; use crate::shared::SETTLE_TIMEOUT; use spawned_concurrency::error::ActorError; use spawned_concurrency::message::Message; -use spawned_concurrency::reply::{Deadline, PendingReply, ReplyReceiver}; use spawned_concurrency::threads::{ Actor, ActorRef, ActorStart, Context, Handler, Receiver, Recipient, }; use spawned_concurrency::{ChildHandle, ExitReason}; +use spawned_concurrency::{Deadline, PendingReply, ReplyReceiver}; use std::ops::Deref; use std::panic::Location; use std::sync::Arc; @@ -182,7 +182,7 @@ where R: Send + 'static, F: FnOnce(&mut A, &Context) -> R + Send + 'static, { - let reply = actor.__in_turn(f).and_then(|rx| rx.recv()); + let reply = spawned_concurrency::__private::in_turn_threads(actor, f).and_then(|rx| rx.recv()); reply.unwrap_or_else(|error| { panic!( "in_actor on {:?} failed: {error}; exit reason {:?}", diff --git a/testing/tests/tasks_kit.rs b/testing/tests/tasks_kit.rs index ff1829ab..9e356cd0 100644 --- a/testing/tests/tasks_kit.rs +++ b/testing/tests/tasks_kit.rs @@ -235,7 +235,7 @@ impl Handler for Counter { async fn handle( &mut self, _: Bump, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0.fetch_add(1, Ordering::SeqCst); @@ -246,7 +246,7 @@ impl Handler for Counter { async fn handle( &mut self, _: Get, - reply: spawned_concurrency::reply::PendingReply<'_, usize>, + reply: spawned_concurrency::PendingReply<'_, usize>, _: &Context, ) { reply.reply(self.0.load(Ordering::SeqCst)); @@ -352,7 +352,7 @@ impl Handler for Stuck { async fn handle( &mut self, _: Hang, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.0.store(true, Ordering::SeqCst); diff --git a/testing/tests/threads_kit.rs b/testing/tests/threads_kit.rs index 02f1256b..f03814b8 100644 --- a/testing/tests/threads_kit.rs +++ b/testing/tests/threads_kit.rs @@ -153,7 +153,7 @@ impl Handler for Idle { fn handle( &mut self, _: Ping, - reply: spawned_concurrency::reply::PendingReply<'_, ()>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { reply.reply(()); From 7a5a5f2fbd2ad001598849a7aa7a9b100ada2e2e Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:42:48 -0300 Subject: [PATCH 160/208] fix: clippy, rustdoc and doctest follow-ups to the sealed Receiver and builders A needless borrow in deferred_api's in_actor helper, a Receiver doc link that does not resolve without the async feature, and the metrics module doctest still building Config with a struct literal. --- concurrency/src/lib.rs | 2 +- concurrency/src/metrics.rs | 2 +- concurrency/tests/deferred_api.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/concurrency/src/lib.rs b/concurrency/src/lib.rs index 54927cb9..835abc81 100644 --- a/concurrency/src/lib.rs +++ b/concurrency/src/lib.rs @@ -147,7 +147,7 @@ pub mod __private { use crate::reply::{Deadline, PendingReply, ReplyReceiver}; use crate::ActorError; - /// Seals [`Receiver`](crate::tasks::Receiver): implemented by + /// Seals `Receiver` (tasks and threads): implemented by /// `ActorRef`, `Context` and spawned-test's probe only. pub trait Sealed {} diff --git a/concurrency/src/metrics.rs b/concurrency/src/metrics.rs index 8a0f53c5..5c15904e 100644 --- a/concurrency/src/metrics.rs +++ b/concurrency/src/metrics.rs @@ -6,7 +6,7 @@ //! //! ``` //! use spawned_concurrency::metrics::{self, Config}; -//! metrics::configure(Config { timing_sample_every: 16 }).unwrap(); +//! metrics::configure(Config::new().timing_sample_every(16)).unwrap(); //! let snapshot = metrics::snapshot(); //! assert_eq!(snapshot.actors_active, 0); //! let text = metrics::render_prometheus(); diff --git a/concurrency/tests/deferred_api.rs b/concurrency/tests/deferred_api.rs index 08113ce1..a09fe834 100644 --- a/concurrency/tests/deferred_api.rs +++ b/concurrency/tests/deferred_api.rs @@ -48,7 +48,7 @@ mod tasks_mode { R: Send + 'static, F: FnOnce(&mut Owner, &Context) -> R + Send + 'static, { - spawned_concurrency::__private::in_turn(&actor, f) + spawned_concurrency::__private::in_turn(actor, f) .unwrap() .await .unwrap() From 35b9fcbd706750d4be55836f6319508923bcec14 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:42:48 -0300 Subject: [PATCH 161/208] refactor!: one way to get a Context or stop with Shutdown; Response no longer fails silently (0.8 API pass) Context::from_ref is crate-private (ActorRef::context is the one public way) and ChildHandle::shutdown is removed (stop_with_reason(Shutdown)). Response in tasks mode: unwrap, expect, map and the new into_result on a response that was not awaited wait for the reply off an async runtime and panic inside one (blocking an executor thread can deadlock the actor that must answer); is_ok and is_err panic on it instead of answering false. Unit tests cover the three cases. Docs: an Experimental in 0.8 list in the GUIDE, the stop/exit/kill table on ChildHandle, start_linked's unmanaged placement, the deferred unified start API in ROADMAP, MIGRATION rows and CHANGELOG. --- CHANGELOG.md | 12 ++ ROADMAP.md | 2 +- concurrency/examples/panic_abort.rs | 3 +- concurrency/src/child_handle.rs | 18 ++- concurrency/src/response.rs | 180 +++++++++++++++++------ concurrency/src/tasks/actor.rs | 17 ++- concurrency/src/tasks/stream_tests.rs | 12 +- concurrency/src/tasks/timer_tests.rs | 8 +- concurrency/src/threads/actor.rs | 20 ++- concurrency/src/threads/stream_tests.rs | 4 +- concurrency/src/threads/timer_tests.rs | 16 +- concurrency/tests/cancellation_safety.rs | 4 +- concurrency/tests/cross_backend.rs | 2 +- concurrency/tests/foreign_panics.rs | 4 +- concurrency/tests/panic_safety.rs | 2 +- concurrency/tests/startup_admission.rs | 8 +- concurrency/tests/supervision.rs | 4 +- concurrency/tests/supervision_trace.rs | 18 ++- concurrency/tests/supervisor_policies.rs | 4 +- docs/GUIDE.md | 57 +++++-- docs/MIGRATION.md | 4 + docs/design/semantics.md | 4 +- examples/thread_only/src/main.rs | 8 +- fuzz/src/driver.rs | 2 +- testing/tests/tasks_kit.rs | 4 +- testing/tests/threads_kit.rs | 8 +- 26 files changed, 313 insertions(+), 112 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18ec2b92..86e9a116 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,18 @@ - Macros: `#[handler]` is removed (use `#[request_handler]` or `#[send_handler]`), and a `#[send_handler]` method returning a value is a compile error (it used to answer a `request` with it). + - Duplicates removed: `Context::from_ref` (use `ActorRef::context`) and + `ChildHandle::shutdown` (use `stop_with_reason(ExitReason::Shutdown)`). + - `Response` in tasks mode: `unwrap`, `expect`, `map` and the new + `into_result` on a response that was not awaited wait for the reply + when called off an async runtime and panic inside one (they always + panicked); `is_ok`/`is_err` panic on it instead of answering `false`. + - Docs: an "Experimental in 0.8" list in the GUIDE (job API, + `PendingReplies`, `exit_pending`, registry, helpers, placement + experiments, `spawned-rt` re-exports), the stop/exit/kill table on + `ChildHandle`, and `start_linked`'s unmanaged placement. The + `ActorError` classifiers (`target_gone`, `is_timeout`, `is_transient`, + `exit_reason`) are kept, defined per variant and tested for every one. - Deferred replies and jobs, fixes from the 0.8 pre-release review: a job deadline still fires while the result waits for scheduled emits, and the job keeps its `MAX_OUTSTANDING` credit until its terminal message; a job diff --git a/ROADMAP.md b/ROADMAP.md index 0b924a7e..2740a88c 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -66,7 +66,7 @@ Decided 2026-11-11, in this order: **soundness, introspection, determinism, test 1.0 is a promise, not a feature set: the API is stable, every guarantee is checked, and it has been proven in production. Planned order (each release follows the [Priorities](#priorities)): - **0.8:** deferred replies and pipe-to-self (rank 11), OTP exit signals, startup and runtime-drop semantics (5e, 5f, 5g), and an API cleanup (smaller job API, `#[non_exhaustive]` and builders, OTP names such as `send_request`). -- **0.9:** unified start API (`start`, `start_link`, `start_monitor` with `StartOptions`, children inheriting their parent's placement), dynamic supervisor (12a), supervisor alignment with OTP (budget exhaustion exits `Shutdown`, static `terminate_child`), introspection v1 (`process_info`, `get_state`, tree and system snapshots, opt-in stall watchdog), and the determinism fixes found by the audit. +- **0.9:** unified start API (`start`, `start_link`, `start_monitor` with `StartOptions`, children inheriting their parent's placement; a design item deferred from the 0.8 API review, which only documented `start_linked`'s unmanaged placement), dynamic supervisor (12a), supervisor alignment with OTP (budget exhaustion exits `Shutdown`, static `terminate_child`), introspection v1 (`process_info`, `get_state`, tree and system snapshots, opt-in stall watchdog), and the determinism fixes found by the audit. - **0.10:** the admission package designed together: bounded mailboxes (17), byte credits (18), deadlines (19); bounded draining on shutdown (13); the OTP control lane (14); per-actor memory limits (18a). - **1.0:** stabilization. diff --git a/concurrency/examples/panic_abort.rs b/concurrency/examples/panic_abort.rs index 3b862d13..efa00df8 100644 --- a/concurrency/examples/panic_abort.rs +++ b/concurrency/examples/panic_abort.rs @@ -125,7 +125,8 @@ mod tasks { .await .expect("no restart after Kill") .expect("child unavailable after Kill"); - new.child_handle().shutdown(); + new.child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); assert_eq!(new.wait_exit().await, ExitReason::Shutdown); let stayed = tokio::time::timeout(LIMIT, child.wait_for_generation(first + 1)).await; assert!( diff --git a/concurrency/src/child_handle.rs b/concurrency/src/child_handle.rs index 97308b1c..1c29caad 100644 --- a/concurrency/src/child_handle.rs +++ b/concurrency/src/child_handle.rs @@ -65,6 +65,18 @@ pub(crate) type CancelFn = Arc; /// Unlike `ActorRef`, a `ChildHandle` cannot send messages directly — it /// only provides supervision-related operations (stop, exit, kill, wait, /// check liveness, link/monitor). +/// +/// Ending an actor, as in OTP: +/// +/// | Call | Ends with | `stopped()` | Trapping actor | OTP | +/// |---|---|---|---|---| +/// | `stop()` | `Normal` | runs | ignores trapping | `gen_server:stop/1` | +/// | `stop_with_reason(r)` | `r` (also `Kill`) | runs | ignores trapping | `gen_server:stop/3` | +/// | `exit(r)` | `r`, or nothing for `Normal` | skipped | gets `Exit { from: EXTERNAL, reason: r }` | `exit(Pid, R)` | +/// | `kill()` = `exit(Kill)` | `Killed` | skipped (unless begun) | cannot trap it | `exit(Pid, kill)` | +/// +/// So `stop_with_reason(Kill)` ends `Kill` after `stopped()` and links see a +/// trappable `Kill`, while `kill()` ends `Killed`, untrappable. #[derive(Clone)] pub struct ChildHandle { target: Arc, @@ -163,12 +175,6 @@ impl ChildHandle { }); } - /// Administrative stop with reason `Shutdown`, as [`stop`](Self::stop). - /// The first requested reason wins over later stops. - pub fn shutdown(&self) { - self.stop_with_reason(ExitReason::Shutdown); - } - /// Send an exit signal from outside any actor (OTP `exit_signal/2`, the /// sender is [`ActorId::EXTERNAL`]): /// diff --git a/concurrency/src/response.rs b/concurrency/src/response.rs index 87f31a6d..1960b25c 100644 --- a/concurrency/src/response.rs +++ b/concurrency/src/response.rs @@ -38,7 +38,12 @@ impl RequestOptions { // --------------------------------------------------------------------------- enum ResponseState { - /// The deadline is already in the receiver's slot, fixed at send. + /// Sent, not yet polled: the deadline is already in the receiver's slot, + /// fixed at send. Awaiting arms the runtime timer; off a runtime, + /// `unwrap`, `expect`, `map` and `into_result` block on it instead. + #[cfg(feature = "async")] + Sent(ReplyReceiver), + /// Being awaited. #[cfg(feature = "async")] Receiver(Pin>>), Ready(Result), @@ -48,10 +53,19 @@ enum ResponseState { /// Concrete wrapper for protocol request-response methods that works in both /// `tasks` (async) and `threads` (blocking) modes. /// -/// - **Tasks mode**: wraps a [`ReplyReceiver`](crate::ReplyReceiver); +/// - **Tasks mode**: wraps the request's [receiver](crate::ReplyReceiver); /// `.await` returns `Result`. The request's deadline was fixed /// when it was sent. -/// - **Threads mode**: wraps a pre-computed result; use `.unwrap()` / `.expect()` directly +/// - **Threads mode**: wraps a pre-computed result; use `.unwrap()`, +/// `.expect()` or `.into_result()` directly. +/// +/// A tasks-mode response that has not been awaited is *pending*. `unwrap`, +/// `expect`, `map` and `into_result` on a pending response block the calling +/// thread until the reply when called outside an async runtime (from plain +/// synchronous code), and panic inside one (blocking an executor thread can +/// deadlock the actor that must answer): there, `.await` it. `is_ok` and +/// `is_err` cannot wait (they borrow), so on a pending response they panic +/// rather than answer `false`. /// /// Protocol methods return `Response`: /// ```ignore @@ -72,6 +86,17 @@ pub struct Response( // is; `Future::poll` uses `self.get_mut()`. impl Unpin for Response {} +/// Whether the calling thread runs an async runtime, where a pending +/// response must be awaited, not waited for. +#[cfg(feature = "async")] +fn on_async_runtime() -> bool { + #[cfg(not(spawned_sim))] + return spawned_rt::tasks::Handle::try_current().is_ok(); + // The simulator runs every task on one thread: blocking would deadlock. + #[cfg(spawned_sim)] + return true; +} + impl Response { /// Create a `Response` from a pre-computed result. /// @@ -85,64 +110,80 @@ impl Response { ) } - /// Extract the value, panicking on error. - /// - /// For ready (pre-computed) responses. Panics on pending responses — - /// use `.await.unwrap()` in async contexts instead. - pub fn unwrap(self) -> T { - match self.0 { - ResponseState::Ready(result) => result.unwrap(), + /// The result: at once for a ready response; for a pending one, block + /// until the reply, the request's deadline or the actor's death (outside + /// an async runtime only; inside one this panics: `.await` it). + pub fn into_result(self) -> Result { + self.resolve("into_result") + } + + fn resolve(mut self, op: &str) -> Result { + match std::mem::replace(&mut self.0, ResponseState::Done) { + ResponseState::Ready(result) => result, + #[cfg(feature = "async")] + ResponseState::Sent(rx) => { + if on_async_runtime() { + panic!( + "called {op}() on a pending Response inside an async runtime; \ + .await it instead" + ); + } + let result = rx.recv(); + #[cfg(feature = "metrics")] + if let Some(observation) = self.1.take() { + observation.finish(&result); + } + result + } #[cfg(feature = "async")] ResponseState::Receiver(_) => { - panic!("called unwrap() on a pending Response; use .await in async contexts") + panic!("called {op}() on a Response that is being awaited; .await it") } - ResponseState::Done => panic!("Response already consumed"), + ResponseState::Done => panic!("called {op}() on a Response already consumed"), } } - /// Extract the value, panicking with a custom message on error. + /// Extract the value, panicking on error. A pending response is waited + /// for (see [`Response`]). + pub fn unwrap(self) -> T { + self.resolve("unwrap").unwrap() + } + + /// Extract the value, panicking with a custom message on error. A + /// pending response is waited for (see [`Response`]). pub fn expect(self, msg: &str) -> T { - match self.0 { - ResponseState::Ready(result) => result.expect(msg), - #[cfg(feature = "async")] - ResponseState::Receiver(_) => { - panic!("{msg}: called expect() on a pending Response; use .await in async contexts") - } - ResponseState::Done => panic!("{msg}: Response already consumed"), - } + self.resolve("expect").expect(msg) } - /// Returns `true` if the response contains `Ok`. - /// Only meaningful for ready responses. - /// Returns `false` for pending (`Receiver`) or consumed (`Done`) states. + /// Returns `true` if the response is ready and `Ok`. Panics on a + /// pending response (it cannot wait through a borrow; use + /// [`into_result`](Self::into_result) or `.await`). pub fn is_ok(&self) -> bool { - matches!(&self.0, ResponseState::Ready(Ok(_))) + self.ready_ref("is_ok").is_ok() } - /// Returns `true` if the response contains `Err`. - /// Only meaningful for ready responses. - /// Returns `false` for pending (`Receiver`) or consumed (`Done`) states. + /// Returns `true` if the response is ready and `Err`. Panics on a + /// pending response (see [`is_ok`](Self::is_ok)). pub fn is_err(&self) -> bool { - matches!(&self.0, ResponseState::Ready(Err(_))) + self.ready_ref("is_err").is_err() } - /// Maps the inner value if the response is ready and `Ok`. - /// - /// Panics on pending or consumed responses. - pub fn map U>(self, f: F) -> Response { - match self.0 { - ResponseState::Ready(result) => Response( - ResponseState::Ready(result.map(f)), - #[cfg(feature = "metrics")] - None, - ), + fn ready_ref(&self, op: &str) -> &Result { + match &self.0 { + ResponseState::Ready(result) => result, #[cfg(feature = "async")] - ResponseState::Receiver(_) => { - panic!("called map() on a pending Response; use .await in async contexts") + ResponseState::Sent(_) | ResponseState::Receiver(_) => { + panic!("called {op}() on a pending Response; .await it, or use into_result()") } - ResponseState::Done => panic!("Response already consumed"), + ResponseState::Done => panic!("called {op}() on a Response already consumed"), } } + + /// Maps the inner value. A pending response is waited for (see + /// [`Response`]). + pub fn map U>(self, f: F) -> Response { + Response::ready(self.resolve("map").map(f)) + } } #[cfg(feature = "async")] @@ -152,7 +193,7 @@ impl From, ActorError>> for Response { let observation = crate::instrumentation::RequestObservation::new(); match result { Ok(rx) => Self( - ResponseState::Receiver(Box::pin(rx.into_timed_future())), + ResponseState::Sent(rx), #[cfg(feature = "metrics")] Some(observation), ), @@ -178,6 +219,12 @@ impl Future for Response { _cx: &mut std::task::Context<'_>, ) -> std::task::Poll { let this = self.get_mut(); + #[cfg(feature = "async")] + if matches!(this.0, ResponseState::Sent(_)) { + if let ResponseState::Sent(rx) = std::mem::replace(&mut this.0, ResponseState::Done) { + this.0 = ResponseState::Receiver(Box::pin(rx.into_timed_future())); + } + } let result = match &mut this.0 { #[cfg(feature = "async")] ResponseState::Receiver(rx) => match rx.as_mut().poll(_cx) { @@ -191,6 +238,8 @@ impl Future for Response { ResponseState::Ready(result) => std::task::Poll::Ready(result), _ => unreachable!(), }, + #[cfg(feature = "async")] + ResponseState::Sent(_) => unreachable!("converted above"), ResponseState::Done => panic!("Response polled after completion"), }; #[cfg(feature = "metrics")] @@ -256,6 +305,51 @@ mod tests { }); } + /// Review footgun (tasks): `actor.query().unwrap()` used to panic on a + /// pending response and `is_ok()` answered `false`. Off a runtime it now + /// waits for the reply; inside one it panics, and `is_ok` never answers + /// `false` for a reply it has not seen. + #[test] + #[cfg(all(feature = "async", not(spawned_sim)))] + fn pending_response_waits_off_a_runtime() { + let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); + let answer = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(20)); + reply.reply(7); + }); + let resp: Response = Response::from(Ok(rx)); + assert_eq!(resp.unwrap(), 7); + answer.join().unwrap(); + + let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); + drop(reply); + let resp: Response = Response::from(Ok(rx)); + assert!(matches!(resp.into_result(), Err(ActorError::ReplyDropped))); + + let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); + reply.reply(2); + assert_eq!(Response::from(Ok(rx)).map(|x| x * 5).unwrap(), 10); + } + + #[test] + #[cfg(all(feature = "async", not(spawned_sim)))] + #[should_panic(expected = "inside an async runtime")] + fn pending_response_unwrap_panics_inside_a_runtime() { + let rt = spawned_rt::tasks::Runtime::new().unwrap(); + rt.block_on(async { + let (_reply, rx) = crate::reply::reply_pair::(Deadline::Never); + Response::from(Ok(rx)).unwrap(); + }); + } + + #[test] + #[cfg(feature = "async")] + #[should_panic(expected = "pending Response")] + fn is_ok_on_a_pending_response_panics_instead_of_answering_false() { + let (_reply, rx) = crate::reply::reply_pair::(Deadline::Never); + let _ = Response::from(Ok(rx)).is_ok(); + } + #[test] fn map_transforms_value() { let r: Response = Response::ready(Ok(2)); diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 7476906d..bc697a83 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -614,9 +614,8 @@ impl Debug for Context { } impl Context { - /// Create a `Context` from an `ActorRef`. Useful for setting up timers - /// or stream listeners from outside the actor. - pub fn from_ref(actor_ref: &ActorRef) -> Self { + /// The `Context` of `actor_ref`'s actor (public as `ActorRef::context`). + pub(crate) fn from_ref(actor_ref: &ActorRef) -> Self { Self { shared: actor_ref.shared.clone(), } @@ -1341,7 +1340,8 @@ impl ActorRef { Arc::new(self.clone()) } - /// Get a `Context` from this ref, for timer setup or stream listeners. + /// Get a `Context` from this ref, for timer setup or stream listeners + /// from outside the actor. pub fn context(&self) -> Context { Context::from_ref(self) } @@ -2190,6 +2190,11 @@ pub trait ActorStart: Actor { /// reaches `exit_received`), a non-trapping child as any link notice. A /// parent already past its death point gives the child a `NoProc` parent /// exit. + /// + /// The child is always an unmanaged `Backend::Async` actor, even when + /// the parent is managed: it is outside the parent's execution group and + /// its ordered shutdown (`spawn_helper` on it returns `UnmanagedActor`). + /// The unified start API (planned for 0.9) will inherit the placement. fn start_linked(self, parent_ctx: &Context

) -> ActorRef { ActorRef::spawn(self, Backend::Async, Some(parent_ctx.child_handle())) } @@ -4357,7 +4362,9 @@ mod api_coverage { // The Exit is handled by the default `exit_received`, which ignores it. assert!(matches!(trapper.request(Ask(3)).await, Ok(3))); assert!(trapper.exit_reason().is_none()); - trapper.child_handle().shutdown(); + trapper + .child_handle() + .stop_with_reason(crate::ExitReason::Shutdown); assert_eq!(trapper.wait_exit().await, ExitReason::Shutdown); }); } diff --git a/concurrency/src/tasks/stream_tests.rs b/concurrency/src/tasks/stream_tests.rs index 51217963..3a08af6c 100644 --- a/concurrency/src/tasks/stream_tests.rs +++ b/concurrency/src/tasks/stream_tests.rs @@ -87,7 +87,7 @@ pub fn test_sum_numbers_from_stream() { let summatory = Summatory::new(0).start(); let stream = stream::iter(vec![1u16, 2, 3, 4, 5].into_iter().map(Ok::)); - let ctx = Context::from_ref(&summatory); + let ctx = summatory.context(); spawn_listener( ctx, stream.filter_map(|result| async move { result.ok().map(StreamMsg::Add) }), @@ -116,7 +116,7 @@ pub fn test_sum_numbers_from_channel() { } }); - let ctx = Context::from_ref(&summatory); + let ctx = summatory.context(); spawn_listener( ctx, ReceiverStream::new(rx) @@ -146,7 +146,7 @@ pub fn test_sum_numbers_from_broadcast_channel() { } }); - let ctx = Context::from_ref(&summatory); + let ctx = summatory.context(); spawn_listener( ctx, BroadcastStream::new(rx) @@ -181,7 +181,7 @@ pub fn test_stream_cancellation() { } }); - let ctx = Context::from_ref(&summatory); + let ctx = summatory.context(); let listener_handle = spawn_listener( ctx.clone(), ReceiverStream::new(rx) @@ -215,7 +215,7 @@ pub fn test_halting_on_stream_error() { } }); - let ctx = Context::from_ref(&summatory); + let ctx = summatory.context(); spawn_listener(ctx, msg_stream); // The error message stops the actor. @@ -239,7 +239,7 @@ pub fn test_skipping_on_stream_error() { } }); - let ctx = Context::from_ref(&summatory); + let ctx = summatory.context(); spawn_listener(ctx, msg_stream); let val = wait_for_async( diff --git a/concurrency/src/tasks/timer_tests.rs b/concurrency/src/tasks/timer_tests.rs index f3d50582..ff580fe7 100644 --- a/concurrency/src/tasks/timer_tests.rs +++ b/concurrency/src/tasks/timer_tests.rs @@ -182,7 +182,7 @@ pub fn test_send_after_and_cancellation() { runtime.block_on(async move { let repeater = Delayed::new(0).start(); - let ctx = Context::from_ref(&repeater); + let ctx = repeater.context(); let _ = send_after(Duration::from_millis(100), ctx, Inc); let count = wait_for_async( @@ -193,7 +193,7 @@ pub fn test_send_after_and_cancellation() { .unwrap(); assert_eq!(1, count); - let ctx = Context::from_ref(&repeater); + let ctx = repeater.context(); let timer = send_after(Duration::from_millis(100), ctx, Inc); timer.cancellation_token.cancel(); @@ -211,7 +211,7 @@ pub fn test_send_after_gen_server_teardown() { runtime.block_on(async move { let repeater = Delayed::new(0).start(); - let ctx = Context::from_ref(&repeater); + let ctx = repeater.context(); let _ = send_after(Duration::from_millis(100), ctx, Inc); let count = wait_for_async( @@ -222,7 +222,7 @@ pub fn test_send_after_gen_server_teardown() { .unwrap(); assert_eq!(1, count); - let ctx = Context::from_ref(&repeater); + let ctx = repeater.context(); let _ = send_after(Duration::from_millis(100), ctx, Inc); let count2 = repeater.request(StopDelayed).await.unwrap(); diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 15272c72..d80aecdf 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -476,9 +476,8 @@ impl Debug for Context { } impl Context { - /// Create a `Context` from an `ActorRef`. Useful for setting up timers - /// or stream listeners from outside the actor. - pub fn from_ref(actor_ref: &ActorRef) -> Self { + /// The `Context` of `actor_ref`'s actor (public as `ActorRef::context`). + pub(crate) fn from_ref(actor_ref: &ActorRef) -> Self { Self { id: actor_ref.id, sender: actor_ref.sender.clone(), @@ -1120,7 +1119,8 @@ impl ActorRef { Arc::new(self.clone()) } - /// Get a `Context` from this ref, for timer setup or stream listeners. + /// Get a `Context` from this ref, for timer setup or stream listeners + /// from outside the actor. pub fn context(&self) -> Context { Context::from_ref(self) } @@ -2922,7 +2922,9 @@ mod api_coverage { // The Exit is handled by the default `exit_received`, which ignores it. assert!(matches!(trapper.request(Ask(3)), Ok(3))); assert!(trapper.exit_reason().is_none()); - trapper.child_handle().shutdown(); + trapper + .child_handle() + .stop_with_reason(crate::ExitReason::Shutdown); assert_eq!(trapper.wait_exit(), ExitReason::Shutdown); } @@ -2941,7 +2943,9 @@ mod api_coverage { assert!(matches!(peer.wait_exit(), ExitReason::Panic(_))); assert!(matches!(trapper.request(Ask(1)), Ok(1))); assert!(trapper.exit_reason().is_none()); - trapper.child_handle().shutdown(); + trapper + .child_handle() + .stop_with_reason(crate::ExitReason::Shutdown); trapper.join(); } @@ -2977,7 +2981,9 @@ mod api_coverage { worker.join().unwrap(); } for actor in [a, b] { - actor.child_handle().shutdown(); + actor + .child_handle() + .stop_with_reason(crate::ExitReason::Shutdown); actor.join(); } } diff --git a/concurrency/src/threads/stream_tests.rs b/concurrency/src/threads/stream_tests.rs index cd83a74d..c6f4ab1c 100644 --- a/concurrency/src/threads/stream_tests.rs +++ b/concurrency/src/threads/stream_tests.rs @@ -60,7 +60,7 @@ fn listener_consumes_iterator() { } .start(); - let ctx = Context::from_ref(&actor); + let ctx = actor.context(); let items = vec![ Push { value: 1 }, Push { value: 2 }, @@ -88,7 +88,7 @@ fn listener_stops_on_cancellation() { } .start(); - let ctx = Context::from_ref(&actor); + let ctx = actor.context(); // An iterator that blocks between items, giving us time to cancel let iter = (1..=100).map(move |i| { diff --git a/concurrency/src/threads/timer_tests.rs b/concurrency/src/threads/timer_tests.rs index 08f938b1..ddca532a 100644 --- a/concurrency/src/threads/timer_tests.rs +++ b/concurrency/src/threads/timer_tests.rs @@ -177,7 +177,7 @@ impl Handler for Delayed { pub fn test_send_after_and_cancellation() { let actor = Delayed::new(0).start(); - let ctx = Context::from_ref(&actor); + let ctx = actor.context(); let _ = send_after(Duration::from_millis(100), ctx, Inc); let count = wait_for( @@ -187,7 +187,7 @@ pub fn test_send_after_and_cancellation() { .unwrap(); assert_eq!(1, count); - let ctx = Context::from_ref(&actor); + let ctx = actor.context(); let timer = send_after(Duration::from_millis(100), ctx, Inc); timer.cancellation_token.cancel(); @@ -202,7 +202,7 @@ pub fn test_send_after_and_cancellation() { pub fn test_send_after_actor_shutdown() { let actor = Delayed::new(0).start(); - let ctx = Context::from_ref(&actor); + let ctx = actor.context(); let _ = send_after(Duration::from_millis(100), ctx, Inc); let count = wait_for( @@ -212,7 +212,7 @@ pub fn test_send_after_actor_shutdown() { .unwrap(); assert_eq!(1, count); - let ctx = Context::from_ref(&actor); + let ctx = actor.context(); let _ = send_after(Duration::from_millis(100), ctx, Inc); let count2 = actor.request(StopDelayed).unwrap(); @@ -283,8 +283,8 @@ pub fn threads_send_after_uses_the_monotonic_clock_over_long_delays() { let delay = Duration::from_millis(300); let long = Duration::from_millis(1_500); let armed = std::time::Instant::now(); - let _short = send_after(delay, Context::from_ref(&actor), Fired); - let cancelled = send_after(long, Context::from_ref(&actor), Fired); + let _short = send_after(delay, actor.context(), Fired); + let cancelled = send_after(long, actor.context(), Fired); assert!( wait_for(|| handled.lock().unwrap().len(), |n| *n >= 1) >= 1, "timer never fired" @@ -296,7 +296,7 @@ pub fn threads_send_after_uses_the_monotonic_clock_over_long_delays() { fired.duration_since(armed) ); cancelled.cancellation_token.cancel(); - let _uncancelled = send_after(long, Context::from_ref(&actor), Fired); + let _uncancelled = send_after(long, actor.context(), Fired); let armed_long = std::time::Instant::now(); assert!( wait_for(|| handled.lock().unwrap().len(), |n| *n >= 2) >= 2, @@ -328,7 +328,7 @@ pub fn threads_send_interval_is_fixed_delay_while_the_actor_stalls() { stall: Some(period * 7 / 2), } .start(); - let timer = send_interval(period, Context::from_ref(&actor), Stamp(sends.clone())); + let timer = send_interval(period, actor.context(), Stamp(sends.clone())); assert!( wait_for(|| handled.lock().unwrap().len(), |n| *n >= 10) >= 10, "timer never fired" diff --git a/concurrency/tests/cancellation_safety.rs b/concurrency/tests/cancellation_safety.rs index 94c7ee68..028dd525 100644 --- a/concurrency/tests/cancellation_safety.rs +++ b/concurrency/tests/cancellation_safety.rs @@ -455,7 +455,9 @@ async fn monitor_and_link_setup_requests_are_cancel_safe() { fx.counters.watched.load(Ordering::SeqCst) }; assert_eq!(registered, usize::from(n > 0), "{op}: N={n}"); - target.child_handle().shutdown(); + target + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); assert_eq!( spawned_rt::tasks::timeout(LIMIT, target.wait_exit()) .await diff --git a/concurrency/tests/cross_backend.rs b/concurrency/tests/cross_backend.rs index 38616a94..76608b5c 100644 --- a/concurrency/tests/cross_backend.rs +++ b/concurrency/tests/cross_backend.rs @@ -806,7 +806,7 @@ fn end(env: &Env, actor: &Ref, ending: Ending) -> ExitReason { actor.request(Cmd::Boom), Err(ActorError::ActorStopped(ExitReason::Panic(_))) )), - Ending::Shutdown => child.shutdown(), + Ending::Shutdown => child.stop_with_reason(spawned_concurrency::ExitReason::Shutdown), } env.wait_exit(&child) } diff --git a/concurrency/tests/foreign_panics.rs b/concurrency/tests/foreign_panics.rs index 978cdf00..e414d1fc 100644 --- a/concurrency/tests/foreign_panics.rs +++ b/concurrency/tests/foreign_panics.rs @@ -596,7 +596,9 @@ fn panic_hook_child() { let stopped = Arc::new(AtomicBool::new(false)); let actor = env.start(Victim::new(&stopped, false)); assert_eq!(actor.ping(&env), 7); - actor.child().shutdown(); + actor + .child() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); assert_eq!(env.wait_exit(&actor.child()), ExitReason::Shutdown); env.finish(); } diff --git a/concurrency/tests/panic_safety.rs b/concurrency/tests/panic_safety.rs index 0edab0b2..123783b9 100644 --- a/concurrency/tests/panic_safety.rs +++ b/concurrency/tests/panic_safety.rs @@ -518,7 +518,7 @@ fn exercise(h: &dyn Harness, fault: Fault, gate: &Gate, events: &Events) -> Exit let trigger = h.spawn_trigger(); h.victim_setup(Setup::Trap); h.victim_setup(Setup::Link(trigger.clone())); - trigger.shutdown(); + trigger.stop_with_reason(spawned_concurrency::ExitReason::Shutdown); } Fault::DownHandler => { let trigger = h.spawn_trigger(); diff --git a/concurrency/tests/startup_admission.rs b/concurrency/tests/startup_admission.rs index 158775e3..a234ab13 100644 --- a/concurrency/tests/startup_admission.rs +++ b/concurrency/tests/startup_admission.rs @@ -98,7 +98,9 @@ async fn managed_actor_stopped_before_first_poll_runs_no_hooks_but_drops_state() .unwrap() }) .collect(); - actor.child_handle().shutdown(); + actor + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); release.send(()).unwrap(); spawned_rt::tasks::timeout(LIMIT, blocker.join()) .await @@ -135,7 +137,9 @@ async fn managed_actor_stopped_before_first_poll_runs_no_hooks_but_drops_state() async fn legacy_actor_stopped_before_first_poll_keeps_both_hooks() { let counts = Arc::new(Counts::default()); let actor = Probe(counts.clone()).start(); // this runtime has not yielded - actor.child_handle().shutdown(); + actor + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); assert_eq!( spawned_rt::tasks::timeout(LIMIT, actor.wait_exit()) .await diff --git a/concurrency/tests/supervision.rs b/concurrency/tests/supervision.rs index ce95d759..0f1f47f0 100644 --- a/concurrency/tests/supervision.rs +++ b/concurrency/tests/supervision.rs @@ -148,7 +148,9 @@ async fn restart_modes_distinguish_normal_shutdown_and_panic() { let generation = child.snapshot().generation; match &exit { ExitReason::Normal => old.child_handle().stop(), - ExitReason::Shutdown => old.child_handle().shutdown(), + ExitReason::Shutdown => old + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown), _ => assert!(old.request(Crash).await.is_err()), } // The child really exited with the reason under test. diff --git a/concurrency/tests/supervision_trace.rs b/concurrency/tests/supervision_trace.rs index 7c1fa490..c04697c3 100644 --- a/concurrency/tests/supervision_trace.rs +++ b/concurrency/tests/supervision_trace.rs @@ -191,7 +191,9 @@ async fn trace_queued_destructor_failure_after_shutdown_restarts_transient_child .unwrap() .unwrap(); actor.send(DropBomb).unwrap(); - actor.child_handle().shutdown(); + actor + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); release.send(()).unwrap(); // v0.8: the reason stays Shutdown, but the cleanup failure restarts a // Transient child (the supervisor is not closing). @@ -386,12 +388,18 @@ async fn trace_external_shutdown_of_permanent_child_restarts() { let a = tree.add(worker("a", &group).restart(modes[0])).unwrap(); let b = tree.add(worker("b", &group).restart(modes[1])).unwrap(); let supervisor = tree.start().await.unwrap(); - a.current().unwrap().child_handle().shutdown(); + a.current() + .unwrap() + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); spawned_rt::tasks::timeout(LIMIT, a.wait_for_generation(1)) .await .unwrap() .unwrap(); - b.current().unwrap().child_handle().shutdown(); + b.current() + .unwrap() + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); wait_state(&b, ChildState::Stopped).await; assert!(supervisor.shutdown(LIMIT).await.complete); write_trace("external_shutdown", supervisor.trace_id(), &modes, 5); @@ -844,7 +852,9 @@ async fn trace_randomized_lifecycles() { 1 => { let _ = actor.send(Quit); } - _ => actor.child_handle().shutdown(), + _ => actor + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown), } } spawned_rt::tasks::sleep(Duration::from_millis(rng.next(4))).await; diff --git a/concurrency/tests/supervisor_policies.rs b/concurrency/tests/supervisor_policies.rs index c695bcc1..7c45235e 100644 --- a/concurrency/tests/supervisor_policies.rs +++ b/concurrency/tests/supervisor_policies.rs @@ -300,7 +300,9 @@ async fn scenario( false } Action::Shutdown => { - actor.child_handle().shutdown(); + actor + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); assert_eq!(actor.wait_exit().await, ExitReason::Shutdown); false } diff --git a/docs/GUIDE.md b/docs/GUIDE.md index bd788fb4..baf80bc9 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -28,6 +28,7 @@ or model that checks each one, are collected in the - [Execution groups and supervision](#execution-groups-and-supervision) (v0.7) - [Metrics](#metrics) - [Testing your actors](#testing-your-actors) +- [Experimental in 0.8](#experimental-in-08) - [Further reading](#further-reading) --- @@ -75,7 +76,7 @@ In all cases, `join()` will eventually return and subsequent `send()`/`request() Call `ctx.stop()` from inside a handler or lifecycle hook. This cancels the actor's internal token, causing the message loop to exit after the current handler finishes. -From outside, use the actor's `ChildHandle` (`actor_ref.child_handle()`): `stop()` exits `Normal`, `shutdown()` exits `Shutdown`, and `stop_with_reason(reason)` picks the reason. All are cooperative: the running handler finishes and `stopped()` runs. They ignore `trap_exit`. `ActorRef` itself has no stop method; an explicit shutdown message in your protocol also works. +From outside, use the actor's `ChildHandle` (`actor_ref.child_handle()`): `stop()` exits `Normal` and `stop_with_reason(reason)` picks the reason (`stop_with_reason(ExitReason::Shutdown)` for a shutdown). All are cooperative: the running handler finishes and `stopped()` runs. They ignore `trap_exit`. `ActorRef` itself has no stop method; an explicit shutdown message in your protocol also works. Stopping discards messages still queued: once the stop is observed no further message is handled, and queued requests fail with `ActorStopped(reason)`, the actor's final exit reason. @@ -140,7 +141,7 @@ State that must be released on every exit, including kills and runtime drops, be | `ctx.execution_group()` | The `ExecutionGroup` a managed actor runs on (v0.7) | | `ctx.mailbox_depth()` | Queued messages (with the `metrics` feature) | -`Context::from_ref(&actor_ref)` creates a context from an `ActorRef`, useful for setting up timers or stream listeners from outside the actor. +`actor_ref.context()` returns the actor's `Context`, useful for setting up timers or stream listeners from outside the actor. ### Self-scheduling example @@ -482,19 +483,28 @@ The registry uses `Any`-based downcasting, so `whereis` returns `None` if the st `Response` is the return type for protocol request methods. It works in both execution modes: -- **Tasks mode** — wraps a oneshot receiver. Use `.await` to get `Result`. -- **Threads mode** — wraps a pre-computed result. Use `.unwrap()` or `.expect()` directly. +- **Tasks mode** — wraps the request's `ReplyReceiver`. Use `.await` to get `Result`. +- **Threads mode** — wraps a pre-computed result. Use `.unwrap()`, `.expect()` or `.into_result()` directly. -### Methods (sync, for threads mode) +### Methods (sync) | Method | Description | |--------|-------------| +| `.into_result()` | The `Result` | | `.unwrap()` | Extract the value, panic on error | | `.expect(msg)` | Extract the value, panic with custom message on error | -| `.is_ok()` | Returns `true` if the response contains `Ok` | -| `.is_err()` | Returns `true` if the response contains `Err` | +| `.is_ok()` | Returns `true` if the response is ready and `Ok` | +| `.is_err()` | Returns `true` if the response is ready and `Err` | | `.map(f)` | Transform the inner value if `Ok` | +A tasks-mode response that was not awaited is *pending*. Off an async +runtime (plain synchronous code), `into_result`, `unwrap`, `expect` and `map` +block until the reply, its deadline or the actor's death. Inside a runtime +they panic, since blocking an executor thread can deadlock the actor that +must answer: `.await` it. `is_ok` and `is_err` borrow and cannot wait, so on +a pending response they panic instead of answering `false` (0.7 answered +`false`, and `unwrap` always panicked). + ### Async usage (tasks mode) ```rust @@ -888,7 +898,7 @@ Signals that arrive while `started()` runs take effect when it returns, with the trap flag read then, so `ctx.trap_exit(true)` first thing in `started()` covers them. -`stop()`, `shutdown()` and `stop_with_reason(r)` (on a `ChildHandle` or the +`stop()` and `stop_with_reason(r)` (on a `ChildHandle` or the `Context`) are not exit signals. They are administrative stops, as `gen_server:stop`: untrappable, `stopped()` runs, and the first stop's reason wins over later stops. `stop_with_reason(ExitReason::Kill)` is such a stop @@ -1230,7 +1240,7 @@ Enable an application tracing subscriber to collect them; Spawned installs none. System/supervisor shutdown suppresses every restart mode. Stopping an individual Permanent child while its supervisor is open intentionally restarts it. Use -supervisor shutdown to close the tree. `ChildHandle::shutdown()` commits Shutdown +supervisor shutdown to close the tree. `stop_with_reason(Shutdown)` commits Shutdown before cooperative cancellation while preserving a previously committed cause; actual handler or cleanup panics remain failures. Existing link propagation still applies and can affect siblings explicitly linked by the application. @@ -1749,6 +1759,35 @@ assert_linearizable(&Kv, &recorder.take()); --- +## Experimental in 0.8 + +These items are public and tested but **may change before 1.0** (each is +waiting on a planned design or on a first user). Everything else follows the +release's semver promise. + +- The job API (`pipe*`, `reserve_*_job`, `JobOptions`, `JobCtx`), pending the + compute pool (rank 11b) and helper scopes (rank 15), and the + `Actor::MAX_NATIVE_JOBS` default (`min(4, available_parallelism)`). +- `PendingReplies` and `Context::deferred_outstanding`. +- `Context::exit_pending` (no OTP counterpart). +- `ChildSpec::max_outstanding` / `max_native_jobs`. +- The registry functions (`register`, `whereis`, `register_owned`, ..), which + rank 12 moves from process scope to `ActorSystem` scope. +- Helpers: `spawn_helper`, `spawn_blocking_helper`, `ScopedTask`, + `FactoryContext` (rank 15). +- Placement: `Backend::{Blocking, Thread}`, `pinned_group`, the affinity + functions, `thread_group_with_helpers`, `async_group_on`. +- The `diagnostics` feature and the `*-experiments` features; the + `lifecycle-trace` and `failpoints` features are test-only. +- `spawned_test::linearizability`. +- `spawned-rt`'s re-exports of Tokio and std types (beyond `run`, + `block_on`, `ctrl_c`, `time::Instant` and the simulation entry points). + +`start_linked` always starts an unmanaged `Backend::Async` actor, even from +a managed parent: the child is outside the parent's execution group and +ordered shutdown (the unified start API, planned for 0.9, will inherit the +parent's placement). + ## Further reading - [Semantics contract](design/semantics.md): every guarantee in one place (ordering, delivery, request outcomes, exit propagation, restarts, shutdown, timers, panics), each linked to what checks it, and what is not guaranteed. diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 7b3d0bdb..8e1be472 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -55,6 +55,10 @@ every change; this guide covers what you need to edit when upgrading. | `actor.__in_turn(f)`, `reply.__discharge_stopped()`, `reply::reply_pair` *(0.8 pre-release, hidden)* | `spawned_test::{tasks, threads}::in_actor`, `spawned_test::reply_pair()`, `spawned_test::detached_reply()` | | `spawned_concurrency::lifecycle_trace` without the `lifecycle-trace` feature | only with the feature (test-only, no stability promise) | | `spawned_rt::time::{to_ticks, from_ticks, now_ticks, MAX_TICKS}` | still public for spawned-concurrency, `#[doc(hidden)]`, no stability promise | +| `Context::from_ref(&actor_ref)` | `actor_ref.context()` | +| `child_handle.shutdown()` | `child_handle.stop_with_reason(ExitReason::Shutdown)` | +| tasks: `actor.query().unwrap()` without `.await` (panicked) | off a runtime it now waits for the reply; inside one it still panics: `actor.query().await.unwrap()` | +| tasks: `response.is_ok()` / `is_err()` on an un-awaited response (answered `false`) | panics: `.await` it, or `response.into_result().is_ok()` off a runtime | ### Exit signals, startup and runtime drops as in OTP (ranks 5e, 5f, 5g, targets 0.8, in progress) diff --git a/docs/design/semantics.md b/docs/design/semantics.md index 95c7e27d..6890d305 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -235,7 +235,7 @@ Checked by: [`startup_admission`][startup_admission], (both group kinds). (`ready_gate_loom` checks the benchmark scheduler's gate, not this barrier.) **L3. Stop is cooperative.** `ctx.stop()`, `ctx.stop_with_reason(r)`, -`ChildHandle::stop()`, `stop_with_reason(r)` and `shutdown()` are self-exits +`ChildHandle::stop()` and `stop_with_reason(r)` are self-exits or administrative stops: they ignore `trap_exit`, the running handler finishes, `stopped()` runs, then the actor exits. Nothing preempts a running handler or blocking code, a kill included. The first stop's reason wins over @@ -255,7 +255,7 @@ Checked by: `Lifecycle!KilledFromKill`, `Lifecycle!LateSignalWins` (TLC); | Reason | Arises from | |---|---| | `Normal` | `ctx.stop()`, `ChildHandle::stop()`, a parent's `Normal` exit to a trapping child | -| `Shutdown` | `ChildHandle::shutdown()`, `stop_with_reason(Shutdown)`, supervisor or system shutdown, an exit signal or link notice with `Shutdown` (non-trapping) | +| `Shutdown` | `stop_with_reason(Shutdown)`, supervisor or system shutdown, an exit signal or link notice with `Shutdown` (non-trapping) | | `Panic(msg)` | a panic in `started()` (`"panic in started(): .."`), a handler, `exit_received` or `Down` handler (`"panic in handler: .."`), `stopped()` (`"panic in stopped(): .."`); an exit signal or link notice with `Panic` | | `Kill` | `stop_with_reason(Kill)` (a stop, not a kill), a `Kill` link notice (non-trapping) | | `Killed` | `ChildHandle::kill()`, `exit(Kill)`, a supervisor's or system's kill escalation, a `Killed` link notice; the runtime dropping the actor unless a non-`Normal` reason was requested | diff --git a/examples/thread_only/src/main.rs b/examples/thread_only/src/main.rs index f8dc0748..483b9663 100644 --- a/examples/thread_only/src/main.rs +++ b/examples/thread_only/src/main.rs @@ -75,7 +75,9 @@ fn main() { let observer = Observer(events.clone()).start(); observer.context().monitor(&counter.child_handle()); spawned_concurrency::threads::send_after(Duration::from_millis(5), observer.context(), Tick); - counter.child_handle().shutdown(); + counter + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); assert_eq!(counter.wait_exit(), ExitReason::Shutdown); assert_eq!( spawned_rt::threads::block_on(counter.child_handle().wait_exit_async()), @@ -96,7 +98,9 @@ fn main() { let first = Counter(0).start(); let second = Counter(0).start(); first.context().link(&second.child_handle()); - second.child_handle().shutdown(); + second + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); assert_eq!(first.wait_exit(), ExitReason::Shutdown); assert_eq!(second.wait_exit(), ExitReason::Shutdown); #[cfg(feature = "metrics")] diff --git a/fuzz/src/driver.rs b/fuzz/src/driver.rs index ee70ec20..bc230b95 100644 --- a/fuzz/src/driver.rs +++ b/fuzz/src/driver.rs @@ -650,7 +650,7 @@ impl Driver { let handle = aref.child_handle(); match r { Reason::Normal => handle.stop(), - Reason::Shutdown => handle.shutdown(), + Reason::Shutdown => handle.stop_with_reason(ExitReason::Shutdown), r => handle.stop_with_reason(reason(r)), } } diff --git a/testing/tests/tasks_kit.rs b/testing/tests/tasks_kit.rs index 9e356cd0..0ece8f97 100644 --- a/testing/tests/tasks_kit.rs +++ b/testing/tests/tasks_kit.rs @@ -621,7 +621,9 @@ async fn kill_stops_a_trapping_target_and_other_signals_are_trapped() { faults::exit(&target.child_handle(), ExitReason::Shutdown); assert_eq!(settle(&target, Get, WAIT).await, 0); assert_eq!(target.child_handle().exit_reason(), None); - target.child_handle().shutdown(); + target + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); assert_eq!(join_within(&target, WAIT).await, ExitReason::Shutdown); } diff --git a/testing/tests/threads_kit.rs b/testing/tests/threads_kit.rs index f03814b8..e0952a20 100644 --- a/testing/tests/threads_kit.rs +++ b/testing/tests/threads_kit.rs @@ -290,7 +290,9 @@ fn native_nothing_is_logged_after_close_returns() { #[test] fn native_panic_injection_into_an_exited_target_is_a_no_op() { let actor = Idle.start(); - actor.child_handle().shutdown(); + actor + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); assert_eq!(join_within(&actor, WAIT), ExitReason::Shutdown); // A signal to an actor past its death point is dropped. faults::exit(&actor.child_handle(), ExitReason::Panic("late".into())); @@ -426,7 +428,9 @@ fn kill_stops_a_trapping_target_and_other_signals_are_trapped() { faults::exit(&target.child_handle(), ExitReason::Shutdown); settle(&target, Ping, WAIT); assert_eq!(target.child_handle().exit_reason(), None); - target.child_handle().shutdown(); + target + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); assert_eq!(join_within(&target, WAIT), ExitReason::Shutdown); } From 358e8e55bf70e96453194a7ba3b806f5301118e9 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:50:21 -0300 Subject: [PATCH 162/208] ci(api): accept 0.8 as a breaking release against v0.7.3 verification/api/release.json names the baseline (v0.7.3) and the release type: major for cargo-semver-checks, the 0.x counterpart of 0.7 -> 0.8 (every break is a row of MIGRATION's 0.7 to 0.8 table). check-api.py semver reads it: for a breaking release the four profiles only check that both rustdoc APIs build (the tool skips its breaking lints); the item-level no-default audit stays for the v0.6.0 patch line. Switch back to patch against v0.8.0 once it is tagged. Checked on fede with the CI-pinned cargo-semver-checks 0.51.0: all four profiles pass. --- docs/TESTING.md | 2 +- scripts/check-api.py | 27 +++++++++++++++++++++++---- verification/api/release.json | 5 +++++ 3 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 verification/api/release.json diff --git a/docs/TESTING.md b/docs/TESTING.md index a53798af..5995170e 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -155,7 +155,7 @@ The fuller rationale is archived in the ```sh python3 scripts/check-api.py features -python3 scripts/check-api.py semver --baseline v0.6.0 +python3 scripts/check-api.py semver # baseline and release type from verification/api/release.json ``` Install cargo-hack 0.6.46 for the first command. CI pins cargo-semver-checks diff --git a/scripts/check-api.py b/scripts/check-api.py index 0ffa5793..5fca389a 100644 --- a/scripts/check-api.py +++ b/scripts/check-api.py @@ -1,5 +1,14 @@ #!/usr/bin/env python3 -"""Check feature builds, migrated APIs and declared v0.7 feature-split breaks.""" +"""Check feature builds, migrated APIs and the release's declared API breaks. + +`verification/api/release.json` names the baseline release and the release +type of the work in progress. A `patch` release must keep the baseline's API +(v0.7's feature-split breaks against v0.6.0 are declared item by item in +`v0.6.0-no-default.json`); a breaking pre-1.0 release (`minor`, as 0.8 is +against 0.7.3, declared `major` to the tool) only checks that both APIs build +in every profile: the tool skips its breaking-change lints for a major +release. +""" import argparse import json import os @@ -45,10 +54,15 @@ def declared_break(command, expected, root): def semver(root, baseline): + release = json.loads((root / "verification/api/release.json").read_text()) + baseline = baseline or release["baseline"] + release_type = release["release_type"] if baseline == release["baseline"] else "patch" + if release_type != "patch": + print(f"Breaking release against {baseline} ({release_type}): {release['reason']}", flush=True) declarations = json.loads((root / "verification/api/v0.6.0-no-default.json").read_text()) - if baseline != declarations["baseline"]: + if release_type == "patch" and baseline != declarations["baseline"]: raise ValueError("the declared feature-split audit requires baseline " + declarations["baseline"]) - base = ["cargo", "semver-checks", "--color", "never", "--release-type", "patch", "--baseline-rev", baseline] + base = ["cargo", "semver-checks", "--color", "never", "--release-type", release_type, "--baseline-rev", baseline] libraries = ["-p", "spawned-concurrency", "-p", "spawned-rt"] # Old no-default consumers restore the previous API by explicitly enabling # async. Check this migration as well as defaults, with/without metrics. @@ -60,6 +74,10 @@ def semver(root, baseline): ]: print("Checking API profile: " + profile, flush=True) run(*base, *features, *packages, cwd=root) + if release_type != "patch": + # Breaks are allowed and listed in MIGRATION; the item-level audit + # applies to the v0.6.0 -> v0.7 patch line only. + return for package, expected in declarations["packages"].items(): print("Auditing raw no-default API: " + package, flush=True) command = base + ["--only-explicit-features", "-p", package] @@ -104,7 +122,8 @@ def features(root, cache, offline): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("check", choices=("features", "semver")) - parser.add_argument("--baseline", default="v0.6.0") + # Default: the baseline of verification/api/release.json. + parser.add_argument("--baseline", default=None) parser.add_argument("--offline", action="store_true") args = parser.parse_args() if args.offline: diff --git a/verification/api/release.json b/verification/api/release.json new file mode 100644 index 00000000..e4533acc --- /dev/null +++ b/verification/api/release.json @@ -0,0 +1,5 @@ +{ + "baseline": "v0.7.3", + "release_type": "major", + "reason": "0.8 is a breaking pre-1.0 release (0.7 -> 0.8, the 0.x counterpart of a major release): Handler::handle takes the reply obligation, Receiver gained send_request and is sealed, enums and read-only structs became non_exhaustive, config structs gained builders, and the job, request and deadline APIs were consolidated. Every break is a row of docs/MIGRATION.md's 0.7 to 0.8 call-site table. The tool is told `major`: it then skips its breaking-change lints and only checks that the current and baseline rustdoc APIs build in every profile. Return to release_type patch (baseline v0.8.0) once 0.8.0 is tagged." +} From ea3a15acf3714123e7c92039161c255734c6c0d3 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 14:53:39 -0300 Subject: [PATCH 163/208] test(ui): a pass case for the 0.8 request, deadline, notice and config API send_request with a Duration, an Instant and Deadline::Never, a Recipient taking a Deadline, deadline getters returning Deadline, shorten_deadline, ActorRef::context, stop_with_reason(Shutdown), Down and Exit matched with .., Exit::new and the SupervisorConfig/Backoff builders. --- concurrency/tests/ui-pass/api_08.rs | 60 +++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 concurrency/tests/ui-pass/api_08.rs diff --git a/concurrency/tests/ui-pass/api_08.rs b/concurrency/tests/ui-pass/api_08.rs new file mode 100644 index 00000000..f7faeff6 --- /dev/null +++ b/concurrency/tests/ui-pass/api_08.rs @@ -0,0 +1,60 @@ +// The 0.8 request, deadline, notice and configuration API compiles as +// documented in MIGRATION's 0.7 to 0.8 call-site table. +use spawned_concurrency::message::Message; +use spawned_concurrency::supervision::{Backoff, SupervisorConfig, SupervisorStrategy}; +use spawned_concurrency::tasks::{Actor, ActorRef, Context, Handler, PendingReply, Recipient}; +use spawned_concurrency::{ + ActorError, ActorId, Deadline, Down, Exit, ExitReason, ReplyTo, Response, + DEFAULT_REQUEST_TIMEOUT, +}; +use std::time::Duration; + +struct Get; +impl Message for Get { + type Result = u32; +} +struct Counter; +impl Actor for Counter {} +impl Handler for Counter { + async fn handle(&mut self, _: Get, reply: PendingReply<'_, u32>, _: &Context) { + // A deadline forwards as is: `other.send_request(msg, reply.deadline())`. + let _: Deadline = reply.deadline(); + reply.reply(1); + } +} + +fn requests(actor: &ActorRef, recipient: &Recipient) -> Result<(), ActorError> { + let _rx = actor.send_request(Get, Duration::from_secs(1))?; + let _rx = actor.send_request(Get, Deadline::Never)?; + let _rx = actor.send_request(Get, spawned_concurrency::Instant::now())?; + let rx = recipient.send_request(Get, Deadline::after(DEFAULT_REQUEST_TIMEOUT))?; + let _: Deadline = rx.deadline(); + let _response: Response = Response::from(actor.send_request(Get, DEFAULT_REQUEST_TIMEOUT)); + let _ctx: Context = actor.context(); + actor.child_handle().stop_with_reason(ExitReason::Shutdown); + Ok(()) +} + +fn token(token: &ReplyTo) { + token.shorten_deadline(Duration::from_millis(5)); + let _: Deadline = token.deadline(); +} + +fn notices(down: Down, exit: Exit) { + let Down { actor, reason, .. } = down; + let _: (ActorId, ExitReason) = (actor, reason); + let Exit { from, .. } = exit; + let _: ActorId = from; + let _ = Exit::new(ActorId::EXTERNAL, ExitReason::Normal); +} + +fn config() -> SupervisorConfig { + SupervisorConfig::new() + .strategy(SupervisorStrategy::RestForOne) + .max_restarts(3) + .backoff(Backoff::new().jitter(false).seed(7)) +} + +fn main() { + let _ = (requests, token, notices, config); +} From 5f1eb5cb1c8db51ac7ce33a0d644647c40bfb437 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 15:06:44 -0300 Subject: [PATCH 164/208] verification: Coordinator models the grace-expiry kill; Lifecycle, DeferredReplies and DeferredJobs leave the TLC-only allowlist system.rs changed in #240 (E3): the ordered loop sends each root its parent's Shutdown signal (cleanup timeout capped to the remaining grace), and at grace expiry the remaining roots get the kill instead of a Shutdown stop; the panic fallback still stops with Shutdown. Coordinator.tla gains `expired` and `killed`: Emergency after a non-crash grace expiry kills every signalled, unfinished root. New invariants KillOnlyAtGrace (no kill before the grace expired) and GraceKillsRemaining (after it, the join waits only on finished or killed roots, so a trapping root stuck in stopped() cannot hold it); controls NoKillAtGrace and KillEarly violate them (TLC and Apalache). MCCoordinator: 155 distinct states, no error (liveness included); Apalache CoordinatorInd init/step/actions hold at 8 and 32 targets with helper H_expired; every mutation, old and new, is violated. model-map.toml: Lifecycle, DeferredReplies and DeferredJobs are trace-validated on this branch (TraceLifecycle, TraceReplies, TraceJobs; 75 traces), so their tlc_only entries go. Lifecycle's EarlyFire is declared unlogged: it exists only under Fault = EarlyNotify. --- verification/model-map.toml | 11 ++----- verification/tla/Coordinator.tla | 51 ++++++++++++++++++++++------- verification/tla/CoordinatorInd.tla | 8 ++++- verification/tla/MCCoordinator.cfg | 2 ++ verification/tla/apalache.sh | 4 ++- verification/tla/negative_checks.py | 4 +++ 6 files changed, 57 insertions(+), 23 deletions(-) diff --git a/verification/model-map.toml b/verification/model-map.toml index 315c6ada..ba6c8696 100644 --- a/verification/model-map.toml +++ b/verification/model-map.toml @@ -113,9 +113,6 @@ code = [ "concurrency/src/mailbox_admission.rs", "concurrency/src/lifecycle_trace.rs", ] -[models.Lifecycle.tlc_only] -reason = "models the proposed v0.8 exit-signal contract; the code on main is v0.7, so there are no lifecycle trace events yet" -tracking = "PR #240 adds LifeEvent and TraceLifecycle.tla; remove this entry once it merges" [models.DeferredReplies] # Rank 11: reply slots, deadlines, deferral credits, owner exit and death. @@ -128,9 +125,6 @@ code = [ "concurrency/src/reply/**", "concurrency/src/lifecycle_trace.rs", ] -[models.DeferredReplies.tlc_only] -reason = "rank 11 design model; the implementation is not on main yet" -tracking = "PR #240 adds reply events and TraceReplies.tla; remove this entry once it merges" [models.DeferredJobs] # Rank 11: pipe jobs, keyed replacement, ActorId and cancellation fencing, @@ -146,9 +140,6 @@ code = [ "concurrency/src/threads/jobs.rs", "concurrency/src/lifecycle_trace.rs", ] -[models.DeferredJobs.tlc_only] -reason = "rank 11 design model; the implementation is not on main yet" -tracking = "PR #240 adds JobEvent and TraceJobs.tla; remove this entry once it merges" [models.DynamicSupervisor] # Rank 12a design model (PR #245); no code yet. @@ -221,6 +212,8 @@ emitters = [ specs = ["verification/tla/TraceLifecycle.tla"] [trace.rejected] Held = "a signal held while started() runs (phase 0) is outside Lifecycle.tla; the trace is rejected" +[trace.unlogged] +EarlyFire = "fault device only (Fault = EarlyNotify: links and monitors fire when exit begins); no code path takes it" [trace.translated] Spawn = { into = [], why = "trace_check.py numbers actors from Spawn events" } Send = { into = ["Request", "Enqueue"], why = "trace_check.py splits Send into Request and Enqueue" } diff --git a/verification/tla/Coordinator.tla b/verification/tla/Coordinator.tla index 9b659045..34224f03 100644 --- a/verification/tla/Coordinator.tla +++ b/verification/tla/Coordinator.tla @@ -6,6 +6,12 @@ EXTENDS Naturals, FiniteSets (* `targets.iter().rev()`), so "lower index stops first" is "later *) (* admitted stops first". Targets abstract roots whose startup committed; *) (* an unready supervisor startup rolls itself back and is not modelled. *) +(* *) +(* v0.8 (E3): the ordered loop sends each root its parent's Shutdown exit *) +(* signal; when the grace expires, the remaining roots get the kill *) +(* (`killed`), so a trapping root stuck in stopped() cannot hold the join. *) +(* The panic fallback (emergency_shutdown) still stops with Shutdown. A *) +(* supervisor root applies its own per-child kill (Supervision.tla). *) (* `@type:` comments are Apalache annotations (CoordinatorInd.tla); TLC *) (* ignores them. *) CONSTANTS @@ -30,36 +36,47 @@ VARIABLES \* @type: Bool; reported, \* @type: Bool; - released -vars == <> + released, + \* @type: Bool; + expired, + \* @type: Set(Int); + killed +vars == <> Init == /\ phase = "Running" /\ open = TRUE /\ signalled = {} /\ finished = {} /\ index = 1 /\ crashed = FALSE /\ reported = FALSE /\ released = FALSE + /\ expired = FALSE /\ killed = {} Begin == /\ phase = "Running" /\ phase' = "Ordering" /\ open' = FALSE - /\ UNCHANGED <> + /\ UNCHANGED <> StopNext == /\ phase = "Ordering" /\ index \in Targets /\ signalled' = signalled \cup {index} /\ phase' = "Waiting" - /\ UNCHANGED <> + /\ killed' = IF Fault = "KillEarly" THEN killed \cup {index} ELSE killed + /\ UNCHANGED <> Advance == /\ phase = "Waiting" /\ index \in finished /\ index' = index + 1 /\ phase' = "Ordering" - /\ UNCHANGED <> + /\ UNCHANGED <> Ordered == /\ phase = "Ordering" /\ index > TargetCount /\ phase' = "Joining" - /\ UNCHANGED <> + /\ UNCHANGED <> GraceExpired == /\ phase \in {"Ordering", "Waiting"} /\ phase' = "Emergency" - /\ UNCHANGED <> + /\ expired' = TRUE + /\ UNCHANGED <> Crash == /\ phase # "Done" /\ ~crashed /\ crashed' = TRUE /\ reported' = (Fault # "ForgetFailure") /\ phase' = "Emergency" /\ open' = FALSE - /\ UNCHANGED <> + /\ UNCHANGED <> +\* After the grace expired (and no panic took the fallback), every root not +\* yet finished gets the kill; otherwise the parent's Shutdown. Emergency == /\ phase = "Emergency" /\ phase' = "Joining" /\ signalled' = IF Fault = "ForgetSignal" THEN signalled ELSE Targets - /\ UNCHANGED <> + /\ killed' = IF expired /\ ~crashed /\ Fault # "NoKillAtGrace" + THEN killed \cup (signalled' \ finished) ELSE killed + /\ UNCHANGED <> Finish(i) == /\ i \in signalled \ finished /\ finished' = finished \cup {i} - /\ UNCHANGED <> + /\ UNCHANGED <> Join == /\ phase = "Joining" /\ (finished = Targets \/ Fault = "ReleaseEarly") /\ phase' = "Done" /\ released' = TRUE - /\ UNCHANGED <> + /\ UNCHANGED <> \* Negative control for the #209 bug: once admission closed, a supervisor \* startup guard dropped after readiness rolled the (ready) tree back, \* stopping it outside the coordinator's order. The fix only rolls back @@ -67,7 +84,7 @@ Join == /\ phase = "Joining" /\ (finished = Targets \/ Fault = "ReleaseEarly") \* this action is disabled and the model is unchanged. RollbackReady(i) == /\ Fault = "RollbackReady" /\ ~open /\ i \in Targets \ signalled /\ signalled' = signalled \cup {i} - /\ UNCHANGED <> + /\ UNCHANGED <> Next == Begin \/ StopNext \/ Advance \/ Ordered \/ GraceExpired \/ Crash \/ Emergency \/ (\E i \in Targets: Finish(i)) \/ Join \/ (\E i \in Targets: RollbackReady(i)) Safety == /\ phase \in {"Running", "Ordering", "Waiting", "Emergency", "Joining", "Done"} @@ -75,6 +92,7 @@ Safety == /\ phase \in {"Running", "Ordering", "Waiting", "Emergency", "Joining" /\ (phase # "Running" => ~open) /\ (released => finished = Targets /\ phase = "Done") /\ (crashed => reported) + /\ killed \subseteq signalled \* Ordered shutdown: while the coordinator is in its ordered loop, exactly \* the targets before the cursor (and the cursor itself once StopNext ran) \* have been signalled, and every target before the cursor has finished. @@ -89,6 +107,15 @@ OrderedShutdown == \* every target was signalled, whether by the ordered loop or the \* emergency fallback: no root is left running unsignalled behind a join. JoinAfterAllSignalled == phase \in {"Joining", "Done"} => signalled = Targets +\* v0.8: a root is killed only once the grace expired (before it, roots get +\* the parent's Shutdown and may run stopped()). +KillOnlyAtGrace == killed # {} => expired +\* v0.8: after the grace expired without a panic, the join waits only on +\* killed roots: every root has finished or was killed (so a trapping root +\* stuck in stopped() cannot hold runtime ownership). +GraceKillsRemaining == + (phase \in {"Joining", "Done"} /\ expired /\ ~crashed) => + \A i \in Targets : i \in finished \/ i \in killed FairSpec == Init /\ [][Next]_vars /\ WF_vars(Begin) /\ WF_vars(StopNext) /\ WF_vars(Advance) /\ WF_vars(Ordered) /\ WF_vars(Emergency) /\ WF_vars(Join) diff --git a/verification/tla/CoordinatorInd.tla b/verification/tla/CoordinatorInd.tla index 8071ebbf..f5f99310 100644 --- a/verification/tla/CoordinatorInd.tla +++ b/verification/tla/CoordinatorInd.tla @@ -20,6 +20,7 @@ TypeInd == /\ finished \in SUBSET Targets /\ index \in 1..TargetCount + 1 /\ crashed \in BOOLEAN /\ reported \in BOOLEAN /\ released \in BOOLEAN + /\ expired \in BOOLEAN /\ killed \in SUBSET Targets ----------------------------------------------------------------------------- (* Helper lemma (the only one needed: without it Apalache returns a CTI *) @@ -34,14 +35,19 @@ TypeInd == \* to Running. Begin then opens the ordered loop at target 1. H_cursor == phase = "Running" => index = 1 +\* The grace expires only in the ordered loop, which it ends: GraceExpired +\* moves to Emergency, from which the loop is never re-entered. +H_expired == expired => phase \in {"Emergency", "Joining", "Done"} + ----------------------------------------------------------------------------- (* The model's state invariants (Coordinator.tla). Safety contains its own *) (* typing conjuncts; OrderedShutdown and JoinAfterAllSignalled are the *) (* ordered-shutdown properties. *) SafetyInd == Safety /\ OrderedShutdown /\ JoinAfterAllSignalled + /\ KillOnlyAtGrace /\ GraceKillsRemaining -Helpers == H_cursor +Helpers == H_cursor /\ H_expired IndInv == TypeInd /\ Helpers /\ SafetyInd diff --git a/verification/tla/MCCoordinator.cfg b/verification/tla/MCCoordinator.cfg index e3a7bfdb..5097b025 100644 --- a/verification/tla/MCCoordinator.cfg +++ b/verification/tla/MCCoordinator.cfg @@ -2,6 +2,8 @@ SPECIFICATION FairSpec INVARIANT Safety INVARIANT OrderedShutdown INVARIANT JoinAfterAllSignalled +INVARIANT KillOnlyAtGrace +INVARIANT GraceKillsRemaining PROPERTY CleanupCompletes CONSTANTS TargetCount = 3 diff --git a/verification/tla/apalache.sh b/verification/tla/apalache.sh index fd2a97f0..37c54f17 100755 --- a/verification/tla/apalache.sh +++ b/verification/tla/apalache.sh @@ -195,7 +195,9 @@ for model in $models; do # without signalling the remaining targets (JoinAfterAllSignalled); # RollbackReady: the #209 bug, a ready root stopped by its startup # rollback outside the coordinator's order (OrderedShutdown). - for fault in ReleaseEarly ForgetFailure ForgetSignal RollbackReady; do + # NoKillAtGrace / KillEarly: v0.8 grace-expiry kill missing, or sent + # before the grace expired (GraceKillsRemaining, KillOnlyAtGrace). + for fault in ReleaseEarly ForgetFailure ForgetSignal RollbackReady NoKillAtGrace KillEarly; do mutate "$module" "$fault" "$module.tla" \ "s/OVERRIDE_Fault == \"None\"/OVERRIDE_Fault == \"$fault\"/" IndInv 4 done diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index bbecf473..84c56454 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -531,6 +531,10 @@ def decide_at_once(events): model_fault("MCCoordinator", "ForgetSignal", args.tlc, temporal=True, drop=("JoinAfterAllSignalled",)) model_fault("MCCoordinator", "RollbackReady", args.tlc, invariant="OrderedShutdown") + # v0.8 grace expiry: no kill (a stuck trapping root holds the join), or a + # kill in the ordered loop (stopped() skipped before the grace expired). + model_fault("MCCoordinator", "NoKillAtGrace", args.tlc, invariant="GraceKillsRemaining") + model_fault("MCCoordinator", "KillEarly", args.tlc, invariant="KillOnlyAtGrace") model_fault("MCNested", "Join", args.tlc) # A Temporary child restarted after a panic (MCSupervision has no # Temporary child, so only MCSupervisionTemporary can catch this). From 167c9ed425185fa7ab69c83951f1a1130d794e6f Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 15:07:33 -0300 Subject: [PATCH 165/208] test(exit): wait for the watcher's Down after a kill (single-cpu race) --- concurrency/tests/otp_exit_signals.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/concurrency/tests/otp_exit_signals.rs b/concurrency/tests/otp_exit_signals.rs index 6709e90f..4f9a16a3 100644 --- a/concurrency/tests/otp_exit_signals.rs +++ b/concurrency/tests/otp_exit_signals.rs @@ -236,7 +236,15 @@ mod tasks_mode { trapper_seen.exits(), vec![Exit::new(t.id(), ExitReason::Killed)] ); - settle(&watcher).await; + // The Down is queued by the monitor's waiter after publication + // (GUIDE: "shortly after"), so wait for it rather than one round + // trip, which a single CPU can finish first. + let began = std::time::Instant::now(); + while watcher_seen.downs().is_empty() { + assert!(began.elapsed() < LIMIT, "no Down after the kill"); + settle(&watcher).await; + tokio::time::sleep(Duration::from_millis(2)).await; + } assert_eq!(watcher_seen.downs().len(), 1); assert_eq!(watcher_seen.downs()[0].reason, ExitReason::Killed); trapper.child_handle().stop(); From 256fecd4e27e91bcfa5a3a9a3bd6dccc46a3d326 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 15:18:30 -0300 Subject: [PATCH 166/208] verification: trace delayed emits and held results against DeferredJobs.tla The job core now records ScheduleEmit (`Schedule`, atomic with the credit-holding registration), FireScheduled (`Fire`: enqueued or dropped, and whether the last fire took a held result), a held result (`Finish` with x = "held") and its release (`Release`, notified or quiet). TraceJobs.tla maps them; a cancel's own dropped fires, which the model's Cancel already accounts for, are stutter steps. Model: releasing a held result was atomic in DeferredJobs.tla (the last FireScheduled enqueued the terminal too), but the code takes the held result under the job's lock and enqueues its terminal after releasing it, so a job deadline or a cancel can take or suppress the notification in between. The release is now its own step, ReleaseHeld (the job keeps its credit until then; HeldCounted counts it; the exit may or may not reach a releasing job, which the list compaction decides). MCDeferredJobs 229,070, MCDeferredJobsEmits 39,640 (was 24,842), MCDeferredJobsHeld 3,230,596 (was 1,946,047) distinct states, no error. Restart is declared unlogged in the model map: the code has no cross-instance mailbox (a job's messages die with their instance's mailbox), so a trace is one instance. New traces (job_trace.rs): held_release (tasks and threads), due_now, held_deadline, cancel_scheduled; 12/12 accepted. New controls, each rejected: release before the last fire, terminal while an emit is scheduled, release by a fire that did not take it, delayed emit after the deadline took the result, fire without schedule, drop while the job still accepts. --- concurrency/src/lifecycle_trace.rs | 13 +- concurrency/src/reply/jobs.rs | 26 +++- concurrency/tests/job_trace.rs | 198 ++++++++++++++++++++++++++++ verification/model-map.toml | 2 + verification/tla/DeferredJobs.tla | 85 ++++++++---- verification/tla/TraceJobs.tla | 32 ++++- verification/tla/negative_checks.py | 35 +++++ verification/tla/trace_check.py | 7 +- 8 files changed, 354 insertions(+), 44 deletions(-) diff --git a/concurrency/src/lifecycle_trace.rs b/concurrency/src/lifecycle_trace.rs index ee8b4f5b..7895bceb 100644 --- a/concurrency/src/lifecycle_trace.rs +++ b/concurrency/src/lifecycle_trace.rs @@ -114,8 +114,11 @@ impl TreeEvent { /// One pipe-to-self job action of `verification/tla/DeferredJobs.tla`, /// recorded per owner instance. `job` is a process-wide job number /// (normalized by the checker); `x` is the action's parameter: the message -/// kind (`emit`, `done`, `deadline`) and fence verdict for `Dispatch`, and -/// whether the terminal message was enqueued for `Finish`. +/// kind (`emit`, `done`, `deadline`) and fence verdict for `Dispatch`, +/// whether the terminal message was enqueued for `Finish` (`notified`, +/// `quiet`, or `held` when scheduled emits keep the result) and `Release`, +/// and whether a delayed emit was enqueued for `Fire` (`emit`, `dropped`; +/// `release` when it was the last and took a held result). #[derive(Debug, Clone, PartialEq, Eq)] pub struct JobEvent { pub action: &'static str, @@ -124,6 +127,7 @@ pub struct JobEvent { pub keyed: bool, pub deadline: bool, pub fenced: bool, + pub release: bool, } impl JobEvent { @@ -135,13 +139,14 @@ impl JobEvent { keyed: false, deadline: false, fenced: false, + release: false, } } /// JSON object understood by `verification/tla/trace_check.py`. pub fn to_json(&self) -> String { format!( - r#"{{"a":"{}","j":{},"x":"{}","keyed":{},"dl":{},"fenced":{}}}"#, - self.action, self.job, self.x, self.keyed, self.deadline, self.fenced + r#"{{"a":"{}","j":{},"x":"{}","keyed":{},"dl":{},"fenced":{},"rel":{}}}"#, + self.action, self.job, self.x, self.keyed, self.deadline, self.fenced, self.release ) } } diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index bbf883b5..c67f71bb 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -351,9 +351,12 @@ impl JobState { crate::instrumentation::hooks::job_notified(end); } if !deadline { + // Only a held result's terminal notifies without the + // deadline (the work's own result goes through `finish`): + // the model's ReleaseHeld. self.record(|job| trace::JobEvent { x: if taken { "notified" } else { "quiet" }, - ..trace::JobEvent::new("Finish", job) + ..trace::JobEvent::new("Release", job) }); } else if taken { self.record(|job| trace::JobEvent::new("JobDeadline", job)); @@ -393,7 +396,10 @@ impl JobState { x: "quiet", ..trace::JobEvent::new("Finish", job) }), - Finish::Held => {} + Finish::Held => self.record(|job| trace::JobEvent { + x: "held", + ..trace::JobEvent::new("Finish", job) + }), } (finish, unsent) }); @@ -422,7 +428,13 @@ impl JobState { failures: Arc, track: impl FnOnce(u64, std::sync::Weak), ) -> Result<(), JobGone> { - self.core.schedule()?; + trace::atomic(|| { + let scheduled = self.core.schedule(); + if scheduled.is_ok() { + self.record(|job| trace::JobEvent::new("Schedule", job)); + } + scheduled + })?; #[cfg(test)] tests::after_schedule(); let emit = Arc::new(ScheduledEmit { @@ -907,8 +919,14 @@ impl ScheduledEmit { let (result, held, unused) = job.core.fire(deliver); if result.is_ok() { crate::instrumentation::hooks::job_emitted(); - job.record(|id| trace::JobEvent::new("Emit", id)); } + // The model's FireScheduled: enqueued or dropped, and whether + // this last fire took a held result (released below). + job.record(|id| trace::JobEvent { + x: if result.is_ok() { "emit" } else { "dropped" }, + release: held.is_some(), + ..trace::JobEvent::new("Fire", id) + }); (result, held, unused) }); let delivered = result.is_ok(); diff --git a/concurrency/tests/job_trace.rs b/concurrency/tests/job_trace.rs index 0404bd22..971a2554 100644 --- a/concurrency/tests/job_trace.rs +++ b/concurrency/tests/job_trace.rs @@ -89,6 +89,15 @@ enum Scenario { Gated, Deadline, Blocking, + /// emit_at in the future, work done first: the result is held and the + /// last fire releases it. + HeldRelease, + /// emit_at already due: fired at the call. + DueNow, + /// The job deadline takes a held result; the later fire is dropped. + HeldDeadline, + /// A cancel drops the scheduled emit. + CancelScheduled, } struct Run(Scenario); @@ -166,6 +175,59 @@ impl Handler for Owner { ) .unwrap(); } + Scenario::HeldRelease => { + let opts = JobOptions::new().emit_credits(std::num::NonZeroU16::new(1).unwrap()); + ctx.pipe( + opts, + |job: JobCtx| async move { + let at = spawned_concurrency::Instant::now() + Duration::from_millis(60); + job.emit_at(at, Emitted(9)).await.unwrap(); + 7u32 + }, + done("held"), + ) + .unwrap(); + } + Scenario::DueNow => { + ctx.pipe( + JobOptions::new(), + |job: JobCtx| async move { + let at = spawned_concurrency::Instant::now(); + job.emit_at(at, Emitted(4)).await.unwrap(); + 4u32 + }, + done("due"), + ) + .unwrap(); + } + Scenario::HeldDeadline => { + let opts = JobOptions::new().deadline(Duration::from_millis(30)); + ctx.pipe( + opts, + |job: JobCtx| async move { + let at = spawned_concurrency::Instant::now() + Duration::from_millis(150); + job.emit_at(at, Emitted(8)).await.unwrap(); + 8u32 + }, + done("held_dl"), + ) + .unwrap(); + } + Scenario::CancelScheduled => { + let handle = ctx + .pipe( + JobOptions::new(), + |job: JobCtx| async move { + let at = spawned_concurrency::Instant::now() + Duration::from_secs(60); + job.emit_at(at, Emitted(6)).await.unwrap(); + job.cancelled().await; + 6u32 + }, + done("cancel_sched"), + ) + .unwrap(); + self.handles.push(handle); + } } reply.reply(()); } @@ -332,3 +394,139 @@ async fn blocking_job_emits_then_completes() { finish("blocking", actor, 2).await; assert_eq!(log.snapshot(), ["emit:0", "emit:1", "blocking:ok:5"]); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn delayed_emit_releases_the_held_result() { + let (actor, log) = start(); + actor.request(Run(Scenario::HeldRelease)).await.unwrap(); + wait_for(&log, "held:ok:7").await; + let actions = finish("held_release", actor, 1).await; + assert_eq!(log.snapshot(), ["emit:9", "held:ok:7"]); + for a in ["Schedule", "Finish", "Fire", "Release"] { + assert!(actions.iter().any(|x| x == a), "{a} missing: {actions:?}"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn due_delayed_emit_fires_at_the_call() { + let (actor, log) = start(); + actor.request(Run(Scenario::DueNow)).await.unwrap(); + wait_for(&log, "due:ok:4").await; + let actions = finish("due_now", actor, 16).await; + assert_eq!(log.snapshot(), ["emit:4", "due:ok:4"]); + assert_eq!( + actions, + ["Start", "Schedule", "Fire", "Finish", "Dispatch", "Dispatch", "Exit"] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn job_deadline_takes_a_held_result() { + let (actor, log) = start(); + actor.request(Run(Scenario::HeldDeadline)).await.unwrap(); + wait_for(&log, "held_dl:job deadline exceeded").await; + sleep(Duration::from_millis(250)).await; + let actions = finish("held_deadline", actor, 16).await; + assert_eq!(log.snapshot(), ["held_dl:job deadline exceeded"]); + assert_eq!( + actions, + [ + "Start", + "Schedule", + "Finish", + "JobDeadline", + "Dispatch", + "Fire", + "Exit" + ] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cancel_drops_a_scheduled_emit() { + let (actor, log) = start(); + actor.request(Run(Scenario::CancelScheduled)).await.unwrap(); + // The emit is registered before the job awaits its cancellation. + sleep(Duration::from_millis(30)).await; + actor.request(CancelStored).await.unwrap(); + sleep(Duration::from_millis(30)).await; + let actions = finish("cancel_scheduled", actor, 16).await; + assert!(log.snapshot().is_empty(), "{:?}", log.snapshot()); + assert_eq!( + actions, + ["Start", "Schedule", "Cancel", "Fire", "Finish", "Exit"] + ); +} + +/// Threads mode: `emit_at_blocking` from a `pipe_blocking` job, result held +/// until the delayed emit fires. +mod threads_mode { + use super::*; + use spawned_concurrency::threads::{ + Actor as TActor, ActorStart as _, Context as TContext, Handler as THandler, + JobCtx as TJobCtx, + }; + + struct TOwner(Arc); + impl TActor for TOwner {} + impl THandler for TOwner { + fn handle(&mut self, msg: Done, _: PendingReply<'_, ()>, _: &TContext) { + self.0.push(msg.0); + } + } + impl THandler for TOwner { + fn handle(&mut self, msg: Emitted, _: PendingReply<'_, ()>, _: &TContext) { + self.0.push(format!("emit:{}", msg.0)); + } + } + struct Go; + impl Message for Go { + type Result = (); + } + impl THandler for TOwner { + fn handle(&mut self, _: Go, reply: PendingReply<'_, ()>, ctx: &TContext) { + let opts = JobOptions::new().emit_credits(std::num::NonZeroU16::new(1).unwrap()); + ctx.pipe_blocking( + opts, + |job: TJobCtx| { + let at = spawned_concurrency::Instant::now() + Duration::from_millis(60); + job.emit_at_blocking(at, Emitted(3)).unwrap(); + 3u32 + }, + done("theld"), + ) + .unwrap(); + reply.reply(()); + } + } + + #[test] + fn delayed_emit_releases_the_held_result_threads() { + let log = Arc::new(Log::default()); + let actor = TOwner(log.clone()).start(); + let id: u64 = actor.id().to_string().parse().unwrap(); + lifecycle_trace::watch_deferred(id); + actor.request(Go).unwrap(); + for _ in 0..400 { + if log.snapshot().iter().any(|e| e == "theld:ok:3") { + break; + } + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(log.snapshot(), ["emit:3", "theld:ok:3"]); + actor.context().stop(); + actor.join(); + let events = lifecycle_trace::take_jobs(id); + let actions: Vec<&str> = events.iter().map(|e| e.action).collect(); + for a in ["Schedule", "Finish", "Fire", "Release", "Exit"] { + assert!(actions.contains(&a), "{a} missing: {actions:?}"); + } + let json = format!( + "{{\"name\":\"jobs_held_release_threads\",\"schema\":\"jobs-v1\",\"credits\":1,\"keys\":1,\"events\":[\n{}\n]}}\n", + events.iter().map(|e| e.to_json()).collect::>().join(",\n") + ); + let dir = trace_dir(); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("jobs_held_release_threads.json"), json).unwrap(); + } +} diff --git a/verification/model-map.toml b/verification/model-map.toml index ba6c8696..c8f4c170 100644 --- a/verification/model-map.toml +++ b/verification/model-map.toml @@ -224,6 +224,8 @@ emitters = [ { call = '\bJobEvent::new\(', arg = 0 }, ] specs = ["verification/tla/TraceJobs.tla"] +[trace.unlogged] +Restart = "a new owner instance reading the old one's slot-addressed mailbox: the code has no such mailbox (a job's messages go to its owner instance's own mailbox, discarded at that instance's death), so the model's cross-instance delivery is a conservative over-approximation (NoLateResult holds a fortiori); a trace is one instance" [[trace]] struct = "ReplyEvent" diff --git a/verification/tla/DeferredJobs.tla b/verification/tla/DeferredJobs.tla index 56cd4aab..731a1a0e 100644 --- a/verification/tla/DeferredJobs.tla +++ b/verification/tla/DeferredJobs.tla @@ -54,6 +54,12 @@ (* otherwise, until execution ends: a held result keeps it (HeldCounted). *) (* The job deadline can take a held result's notification (it stays armed *) (* until the terminal message). *) +(* *) +(* Releasing a held result is two steps, as in the code: the last fire *) +(* takes the held result under the job's lock (FireScheduled; `releasing`) *) +(* and the terminal message is enqueued after the lock is released *) +(* (ReleaseHeld), unless a deadline or a cancel took or suppressed the *) +(* notification in between. The job keeps its credit until ReleaseHeld. *) (***************************************************************************) EXTENDS Naturals, FiniteSets, Sequences CONSTANTS J, Keyed, JobBound, MaxGen, MaxEmit, Credits, Fault @@ -79,14 +85,15 @@ VARIABLES seen, \* history of dispatched messages sched, \* scheduled (emit_at) emits not yet fired, each with a credit held, \* the work ended with emits scheduled: its result is held - credit \* the job holds its MAX_OUTSTANDING credit + credit, \* the job holds its MAX_OUTSTANDING credit + releasing \* the last fire took the held result; its terminal is next vars == <> + started, mbox, termSent, seen, sched, held, credit, releasing>> \* Variables of delayed emits and held results, unchanged by the steps that \* do not concern them. -heldVars == <> +heldVars == <> Init == /\ inst = 1 /\ alive = TRUE @@ -98,6 +105,7 @@ Init == /\ termSent = [j \in Jobs |-> 0] /\ seen = {} /\ sched = [j \in Jobs |-> 0] /\ held = [j \in Jobs |-> FALSE] /\ credit = [j \in Jobs |-> FALSE] + /\ releasing = [j \in Jobs |-> FALSE] Running == {j \in Jobs : exec[j] = "running"} \* Jobs holding a MAX_OUTSTANDING credit (admission checks the bound). @@ -115,8 +123,9 @@ CancelCore(S) == /\ notif' = [j \in Jobs |-> IF j \in S /\ notif[j] = "pending" THEN "suppressed" ELSE notif[j]] /\ sched' = [j \in Jobs |-> IF j \in S THEN 0 ELSE sched[j]] /\ held' = [j \in Jobs |-> IF j \in S THEN FALSE ELSE held[j]] +\* A releasing job keeps its credit until ReleaseHeld. CreditAfterCancel(S) == - [j \in Jobs |-> IF j \in S /\ exec[j] = "finished" THEN FALSE ELSE credit[j]] + [j \in Jobs |-> IF j \in S /\ exec[j] = "finished" /\ ~releasing[j] THEN FALSE ELSE credit[j]] CancelSet(S) == CancelCore(S) /\ credit' = CreditAfterCancel(S) \* Admission and the exit's cancel_all are serialized (one jobs lock and @@ -133,7 +142,7 @@ Start(j) == ELSE /\ started' = IF j \in Keyed THEN Append(started, j) ELSE started /\ credit' = [credit EXCEPT ![j] = TRUE] /\ UNCHANGED <> - /\ UNCHANGED <> + /\ UNCHANGED <> Accepting(j) == notif[j] = "pending" /\ ~cancelled[j] @@ -157,27 +166,39 @@ ScheduleEmit(j) == /\ emitted' = [emitted EXCEPT ![j] = @ + 1] /\ sched' = [sched EXCEPT ![j] = @ + 1] /\ UNCHANGED <> + termSent, seen, held, credit, releasing>> \* A scheduled emit fires: enqueued while the notification is pending (else -\* dropped, its credit returning). The last one releases a held result: its -\* terminal message, then the job's credit. +\* dropped, its credit returning). The last one takes a held result for +\* release (ReleaseHeld enqueues its terminal message). FireScheduled(j) == /\ sched[j] > 0 /\ sched' = [sched EXCEPT ![j] = @ - 1] /\ LET fires == Accepting(j) release == held[j] /\ sched[j] = 1 - emitMsg == IF fires THEN <<[j |-> j, kind |-> "emit"]>> ELSE <<>> - doneMsg == IF release THEN <<[j |-> j, kind |-> "done"]>> ELSE <<>> - IN /\ mbox' = mbox \o emitMsg \o doneMsg + IN /\ mbox' = IF fires THEN Append(mbox, [j |-> j, kind |-> "emit"]) ELSE mbox /\ inflight' = IF fires THEN [inflight EXCEPT ![j] = @ + 1] ELSE inflight /\ IF release THEN /\ held' = [held EXCEPT ![j] = FALSE] - /\ notif' = [notif EXCEPT ![j] = "notified"] - /\ termSent' = [termSent EXCEPT ![j] = @ + 1] - /\ credit' = [credit EXCEPT ![j] = FALSE] - ELSE UNCHANGED <> - /\ UNCHANGED <> + /\ releasing' = [releasing EXCEPT ![j] = TRUE] + ELSE UNCHANGED <> + /\ UNCHANGED <> + +\* The released result's terminal message, after the job's lock was +\* released: enqueued unless a deadline or a cancel came in between. The +\* job's credit returns either way. +ReleaseHeld(j) == + /\ releasing[j] + /\ releasing' = [releasing EXCEPT ![j] = FALSE] + /\ credit' = [credit EXCEPT ![j] = FALSE] + /\ IF Accepting(j) + THEN /\ notif' = [notif EXCEPT ![j] = "notified"] + /\ mbox' = Append(mbox, [j |-> j, kind |-> "done"]) + /\ termSent' = [termSent EXCEPT ![j] = @ + 1] + ELSE UNCHANGED <> + /\ UNCHANGED <> \* Physical completion; the notification is a one-shot. With emits still \* scheduled the result is held (keeping the credit); otherwise the terminal @@ -199,14 +220,15 @@ Finish(j) == /\ UNCHANGED <> ELSE /\ credit' = [credit EXCEPT ![j] = FALSE] /\ UNCHANGED <> - /\ UNCHANGED <> + /\ UNCHANGED <> \* Job deadline (only for jobs started with one): the notification is \* DeadlineExceeded now. Running work keeps running and keeps its credit; a \* held result is dropped and its credit returns. JobDeadline(j) == /\ hasDl[j] /\ notif[j] = "pending" - /\ exec[j] = "running" \/ (held[j] /\ Fault # "HeldDeadlineLost") + /\ exec[j] = "running" \/ (held[j] /\ Fault # "HeldDeadlineLost") \/ releasing[j] /\ notif' = [notif EXCEPT ![j] = "notified"] /\ mbox' = Append(mbox, [j |-> j, kind |-> "deadline"]) /\ termSent' = [termSent EXCEPT ![j] = @ + 1] @@ -214,22 +236,28 @@ JobDeadline(j) == /\ credit' = IF held[j] \/ Fault = "ReleaseOnTimeout" THEN [credit EXCEPT ![j] = FALSE] ELSE credit /\ UNCHANGED <> + sched, releasing>> Cancel(j) == /\ alive /\ owner[j] = inst /\ exec[j] # "none" /\ ~cancelled[j] /\ CancelSet({j}) - /\ UNCHANGED <> + /\ UNCHANGED <> \* Owner exit cancels the live job set (running, or with emits scheduled or \* a result held). Queued messages stay addressed to the slot (the -\* conservative case); they are fenced at dispatch. +\* conservative case); they are fenced at dispatch. A releasing job may or +\* may not still be in the job list the exit cancels (the list is compacted +\* only at size thresholds): either way. Exit == /\ alive /\ alive' = FALSE - /\ CancelSet({j \in Jobs : owner[j] = inst - /\ (exec[j] = "running" - \/ ((sched[j] > 0 \/ held[j]) /\ Fault # "ExitKeepsHeld"))}) - /\ UNCHANGED <> + /\ \E R \in SUBSET {j \in Jobs : owner[j] = inst /\ releasing[j]} : + CancelSet({j \in Jobs : owner[j] = inst + /\ (exec[j] = "running" + \/ ((sched[j] > 0 \/ held[j]) /\ Fault # "ExitKeepsHeld"))} + \cup R) + /\ UNCHANGED <> Restart == /\ ~alive /\ inst < MaxGen /\ inst' = inst + 1 /\ alive' = TRUE @@ -259,10 +287,11 @@ Dispatch == Next == \/ Exit \/ Restart \/ Dispatch \/ \E j \in Jobs : Start(j) \/ Emit(j) \/ Finish(j) \/ JobDeadline(j) \/ Cancel(j) - \/ ScheduleEmit(j) \/ FireScheduled(j) + \/ ScheduleEmit(j) \/ FireScheduled(j) \/ ReleaseHeld(j) Spec == Init /\ [][Next]_vars -FairSpec == Spec /\ (\A j \in Jobs : WF_vars(JobDeadline(j)) /\ WF_vars(FireScheduled(j))) +FairSpec == Spec /\ (\A j \in Jobs : WF_vars(JobDeadline(j)) /\ WF_vars(FireScheduled(j)) + /\ WF_vars(ReleaseHeld(j))) /\ WF_vars(Dispatch) ------------------------------------------------------------------------------ @@ -284,7 +313,7 @@ AtMostOneTerminal == \A j \in Jobs : termSent[j] <= 1 /\ Cardinality(TermSeen(j) PermitsBounded == Cardinality(Running) <= JobBound \* Running work and held results count against the bound: a held result \* keeps the job's credit until its terminal message. -HeldCounted == Cardinality({j \in Jobs : exec[j] = "running" \/ held[j]}) <= JobBound +HeldCounted == Cardinality({j \in Jobs : exec[j] = "running" \/ held[j] \/ releasing[j]}) <= JobBound \* The job deadline stays armed while a result is held. DeadlineArmedWhileHeld == \A j \in Jobs : (held[j] /\ hasDl[j] /\ notif[j] = "pending") => ENABLED JobDeadline(j) diff --git a/verification/tla/TraceJobs.tla b/verification/tla/TraceJobs.tla index 1f15d136..04932c03 100644 --- a/verification/tla/TraceJobs.tla +++ b/verification/tla/TraceJobs.tla @@ -14,7 +14,16 @@ (* Emit(j) an emit enqueued *) (* Finish(j) execution ended; x = "notified" if it enqueued the *) (* terminal message, "quiet" if the notification was *) -(* already taken or suppressed *) +(* already taken or suppressed, "held" if scheduled *) +(* emits keep the result *) +(* Schedule(j) emit_at registered a delayed emit (ScheduleEmit) *) +(* Fire(j) a delayed emit fired (FireScheduled): x = "emit" if *) +(* enqueued, "dropped" if not; rel: the last one took a *) +(* held result. A cancel drops the scheduled emits after *) +(* the model's Cancel already did: those "dropped" fires *) +(* are stutter steps *) +(* Release(j) the held result's terminal, after the job's lock *) +(* (ReleaseHeld): x = "notified" or "quiet" *) (* JobDeadline(j) the job deadline took the notification *) (* Cancel(j) handle cancel (or the opt-in caller give-up) *) (* Exit the owner's exit cancelled the job set *) @@ -22,10 +31,12 @@ (* *) (* One trace is one owner instance (MaxGen = 1) and at most one key. The *) (* model's permit bound is not checked (JobBound = J): rejected *) -(* admissions are not model actions. Delayed emits (emit_at) are not *) -(* recorded; the recorded scenarios use none (MCDeferredJobsHeld checks *) -(* them). The trace is accepted iff some behavior consumes the whole log *) -(* (TLC reports TraceIncomplete violated). *) +(* admissions are not model actions. The model's Restart (a new owner *) +(* instance reading the same slot-addressed mailbox) has no code *) +(* counterpart: a job's messages go to its owner instance's own mailbox, *) +(* which is discarded at that instance's death, so a trace is one *) +(* instance. The trace is accepted iff some behavior consumes the whole *) +(* log (TLC reports TraceIncomplete violated). *) (***************************************************************************) EXTENDS DeferredJobs, TLC @@ -43,6 +54,17 @@ Step(e) == [] e.a = "Emit" -> e.j \in Jobs /\ Emit(e.j) [] e.a = "Finish" -> e.j \in Jobs /\ Finish(e.j) /\ (termSent'[e.j] > termSent[e.j]) = (e.x = "notified") + /\ held'[e.j] = (e.x = "held") + [] e.a = "Schedule" -> e.j \in Jobs /\ ScheduleEmit(e.j) + [] e.a = "Fire" -> e.j \in Jobs /\ + \/ /\ FireScheduled(e.j) + /\ (inflight'[e.j] > inflight[e.j]) = (e.x = "emit") + /\ releasing'[e.j] = e.rel + \/ /\ e.x = "dropped" /\ ~e.rel + /\ cancelled[e.j] /\ sched[e.j] = 0 + /\ UNCHANGED vars + [] e.a = "Release" -> e.j \in Jobs /\ ReleaseHeld(e.j) + /\ (termSent'[e.j] > termSent[e.j]) = (e.x = "notified") [] e.a = "JobDeadline" -> e.j \in Jobs /\ JobDeadline(e.j) [] e.a = "Cancel" -> e.j \in Jobs /\ Cancel(e.j) [] e.a = "Exit" -> Exit diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 84c56454..80dd3fab 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -457,6 +457,41 @@ def dispatch_after_exit(events): require_rejected(by_name["jobs_plain"], dispatch_after_exit, "dispatch after the owner's exit", tlc) + # Delayed emits (emit_at) and held results. + def release_before_fire(events): + release = next(i for i, e in enumerate(events) if e["a"] == "Release") + fire = next(i for i, e in enumerate(events) if e["a"] == "Fire") + events.insert(fire, events.pop(release)) + require_rejected(by_name["jobs_held_release"], release_before_fire, + "held result released before its last delayed emit fired", tlc) + + def terminal_while_scheduled(events): + next(e for e in events if e["a"] == "Finish")["x"] = "notified" + require_rejected(by_name["jobs_held_release"], terminal_while_scheduled, + "terminal message enqueued while a delayed emit is scheduled", tlc) + + def release_without_last_fire(events): + next(e for e in events if e["a"] == "Fire")["rel"] = False + require_rejected(by_name["jobs_held_release"], release_without_last_fire, + "held result released by a fire that did not take it", tlc) + + def emit_after_deadline(events): + next(e for e in events if e["a"] == "Fire")["x"] = "emit" + require_rejected(by_name["jobs_held_deadline"], emit_after_deadline, + "delayed emit enqueued after the job deadline took the result", tlc) + + def fire_unscheduled(events): + events.remove(next(e for e in events if e["a"] == "Schedule")) + require_rejected(by_name["jobs_due_now"], fire_unscheduled, + "delayed emit fired without being scheduled", tlc) + + def drop_before_cancel(events): + cancel = next(i for i, e in enumerate(events) if e["a"] == "Cancel") + fire = next(i for i, e in enumerate(events) if e["a"] == "Fire") + events.insert(cancel, events.pop(fire)) + require_rejected(by_name["jobs_cancel_scheduled"], drop_before_cancel, + "delayed emit dropped while the job still accepts it", tlc) + def main(): parser = argparse.ArgumentParser(description=__doc__) diff --git a/verification/tla/trace_check.py b/verification/tla/trace_check.py index 5c36e389..7b1d6600 100644 --- a/verification/tla/trace_check.py +++ b/verification/tla/trace_check.py @@ -282,18 +282,19 @@ def generate_jobs(trace, workdir): if not jobs: raise ValueError("a jobs trace needs at least one job") keyed = sorted({jobs[int(e["j"])] for e in trace["events"] if e["a"] == "Start" and e["keyed"]}) + # Emits and delayed emits (Schedule) both count towards MaxEmit. emits = {} for e in trace["events"]: - if e["a"] == "Emit": + if e["a"] in {"Emit", "Schedule"}: emits[int(e["j"])] = emits.get(int(e["j"]), 0) + 1 def flag(value): return "TRUE" if value else "FALSE" records = ",\n ".join( - "[a |-> {}, j |-> {}, x |-> {}, keyed |-> {}, dl |-> {}, fenced |-> {}]".format( + "[a |-> {}, j |-> {}, x |-> {}, keyed |-> {}, dl |-> {}, fenced |-> {}, rel |-> {}]".format( tla_string(e["a"]), jobs.get(int(e["j"]), 0), tla_string(e["x"]), - flag(e["keyed"]), flag(e["dl"]), flag(e["fenced"])) + flag(e["keyed"]), flag(e["dl"]), flag(e["fenced"]), flag(e.get("rel", False))) for e in trace["events"]) with open(os.path.join(workdir, "TraceRun.tla"), "w") as out: out.write("---- MODULE TraceRun ----\nEXTENDS TraceJobs\n" From 3e8c73fbd54ed833849669d0e66c20102136cd8b Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 15:23:19 -0300 Subject: [PATCH 167/208] verification: trace actor-to-actor messages as Lifecycle's Send Lifecycle.tla's Send (a plain message from one actor to another, which ExitAfterMessages depends on) had no trace counterpart: the recorder did not know the sending actor, so every send was a request from Ext and MaxSends was 0. With the lifecycle-trace feature only, the recorder now tracks the recorded actor whose turn or hook a thread runs (a thread-local set around each poll in tasks mode, each call in threads mode). A plain message (not a request, not a Down) from such an actor to another is recorded with its sender, its envelope reports the model's mailbox kind "msg", and trace_check.py maps it to Send(s, t) with MaxSends from the trace. Requests from actors, self-sends and refused sends stay requests from Ext (documented in TraceLifecycle.tla). Without the feature the envelope has no extra field and the default send path is unchanged (a constant 0 to an inlined no-op). New traces: lifecycle_message_then_exit (tasks: a2 sends a1 a message, then stops Shutdown; the trapping a1 handles the message before the Exit) and lifecycle_threads_actor_message; 16/16 lifecycle traces accepted. New controls, each rejected: a send after the sender's death point, the death notice overtaking the earlier message, an actor's message dispatched as an outside request. --- concurrency/src/lifecycle_trace.rs | 78 ++++++++++++++++++- concurrency/src/reply/mod.rs | 5 ++ concurrency/src/tasks/actor.rs | 49 +++++++++--- concurrency/src/threads/actor.rs | 32 +++++--- concurrency/tests/lifecycle_trace.rs | 107 +++++++++++++++++++++++++++ verification/model-map.toml | 2 +- verification/tla/TraceLifecycle.tla | 13 +++- verification/tla/negative_checks.py | 23 ++++++ verification/tla/trace_check.py | 10 ++- 9 files changed, 290 insertions(+), 29 deletions(-) diff --git a/concurrency/src/lifecycle_trace.rs b/concurrency/src/lifecycle_trace.rs index 7895bceb..a848c868 100644 --- a/concurrency/src/lifecycle_trace.rs +++ b/concurrency/src/lifecycle_trace.rs @@ -447,16 +447,71 @@ mod imp { } /// A send or request to `owner` was admitted or refused (`NoProc`). A - /// `Down` is recorded by its monitor as a notice delivery instead. - pub(crate) fn life_send(owner: u64, admitted: bool) { + /// `Down` is recorded by its monitor as a notice delivery instead. `from` + /// is the recorded actor that sent it ([`life_sender`]); an admitted + /// send of one is the model's actor-to-actor `Send`, anything else a + /// request from `Ext`. + pub(crate) fn life_send(owner: u64, admitted: bool, from: u64) { if life_kind::() == "down" { return; } life_emit(owner, || { - super::LifeEvent::new("Send", owner).x(if admitted { "queued" } else { "noproc" }) + super::LifeEvent::new("Send", owner) + .s(if admitted { from } else { 0 }) + .x(if admitted { "queued" } else { "noproc" }) }); } + + /// The recorded actor whose turn (or hook) is sending a plain message + /// (not a request, not a `Down`) to another actor `owner`; 0 otherwise. + pub(crate) fn life_sender(owner: u64, is_send: bool) -> u64 { + if !is_send || life_kind::() == "down" { + return 0; + } + let current = CURRENT.with(Cell::get); + if current != 0 && current != owner && life_traced(current) { + current + } else { + 0 + } + } + + /// Run `f` as part of actor `id`'s turn or hook (threads mode). + pub(crate) fn with_actor(id: u64, f: impl FnOnce() -> R) -> R { + struct Restore(u64); + impl Drop for Restore { + fn drop(&mut self) { + CURRENT.with(|c| c.set(self.0)); + } + } + let _restore = Restore(CURRENT.with(|c| c.replace(id))); + f() + } + + type Turn<'a> = std::pin::Pin + Send + 'a>>; + + /// Poll a turn or hook future as part of actor `id` (tasks mode). + pub(crate) fn in_actor(id: u64, turn: Turn<'_>) -> Turn<'_> { + Box::pin(InActor { id, turn }) + } + struct InActor<'a> { + id: u64, + turn: Turn<'a>, + } + impl std::future::Future for InActor<'_> { + type Output = (); + fn poll( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll<()> { + let id = self.id; + with_actor(id, || self.turn.as_mut().poll(cx)) + } + } + thread_local! { + /// The recorded actor whose turn this thread is running. + static CURRENT: Cell = const { Cell::new(0) }; static DEPTH: Cell = const { Cell::new(0) }; static SUPPRESS_CLOSE: Cell = const { Cell::new(false) }; } @@ -757,7 +812,22 @@ mod imp { pub(crate) fn life_spawn(_: u64, _: u64) {} #[allow(dead_code)] #[inline(always)] - pub(crate) fn life_send(_: u64, _: bool) {} + pub(crate) fn life_send(_: u64, _: bool, _: u64) {} + #[allow(dead_code, clippy::extra_unused_type_parameters)] + #[inline(always)] + pub(crate) fn life_sender(_: u64, _: bool) -> u64 { + 0 + } + #[allow(dead_code)] + #[inline(always)] + pub(crate) fn with_actor(_: u64, f: impl FnOnce() -> R) -> R { + f() + } + #[allow(dead_code)] + #[inline(always)] + pub(crate) fn in_actor(_: u64, turn: T) -> T { + turn + } } #[cfg(feature = "lifecycle-trace")] diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index a0436a80..87d517ab 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -196,6 +196,11 @@ impl SlotHandle { pub(crate) fn detached() -> Self { Self(None) } + /// Whether no caller waits (a send). + #[cfg_attr(not(feature = "lifecycle-trace"), allow(dead_code))] + pub(crate) fn is_detached(&self) -> bool { + self.0.is_none() + } pub(crate) fn take(&mut self) -> Option>> { self.0.take() } diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index bc697a83..990d8ba0 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -214,6 +214,10 @@ struct MessageEnvelope { msg: M, #[cfg(feature = "metrics")] observation: Option, + /// The recorded actor that sent it (0: outside code or a request); its + /// mailbox item kind is then the model's `msg`. + #[cfg(feature = "lifecycle-trace")] + from: u64, } impl Envelope for MessageEnvelope @@ -224,7 +228,11 @@ where /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. #[cfg(feature = "lifecycle-trace")] fn trace_kind(&self) -> &'static str { - crate::lifecycle_trace::life_kind::() + if self.from != 0 { + "msg" + } else { + crate::lifecycle_trace::life_kind::() + } } #[cfg(any(feature = "diagnostics", spawned_sim))] fn message_type(&self) -> &'static str { @@ -307,13 +315,16 @@ where A: Actor + Handler, M: Message, { + let from = crate::lifecycle_trace::life_sender::(replies.owner().raw(), slot.is_detached()); let envelope: Box + Send> = Box::new(MessageEnvelope { msg, slot, #[cfg(feature = "metrics")] observation: Some(_mailbox.enqueued()), + #[cfg(feature = "lifecycle-trace")] + from, }); - admit::(sender, replies, envelope) + admit::(sender, replies, envelope, from) } /// A monitor's `Down`, enqueued by the framework (the waiter at the target's @@ -339,7 +350,7 @@ fn enqueue_down>( #[cfg(feature = "metrics")] observation: Some(_mailbox.enqueued()), }); - admit::(sender, replies, envelope) + admit::(sender, replies, envelope, 0) } /// Admission of a built envelope (`M` names its kind for the trace). @@ -347,6 +358,7 @@ fn admit( sender: &mpsc::Sender + Send>>, replies: &Registry, envelope: Box + Send>, + _from: u64, ) -> Result<(), ActorError> { // Admission and its trace event are one step of the lifecycle trace. let owner = replies.owner().raw(); @@ -354,14 +366,14 @@ fn admit( if replies.refuses_admission() { #[cfg(feature = "metrics")] envelope.reject(); - crate::lifecycle_trace::life_send::(owner, false); + crate::lifecycle_trace::life_send::(owner, false, _from); return Err(ActorError::NoProc); } match sender.send(envelope) { Ok(()) => { #[cfg(feature = "metrics")] crate::instrumentation::accepted(); - crate::lifecycle_trace::life_send::(owner, true); + crate::lifecycle_trace::life_send::(owner, true, _from); Ok(()) } Err(_error) => { @@ -369,7 +381,7 @@ fn admit( { _error.0.reject(); } - crate::lifecycle_trace::life_send::(owner, false); + crate::lifecycle_trace::life_send::(owner, false, _from); Err(ActorError::NoProc) } } @@ -1930,7 +1942,13 @@ async fn run_actor_lifecycle( } let start_result = AssertUnwindSafe(async { fail_point!("actor.before_started", async); - DeathFirst::new(Box::pin(actor.started(&ctx)), &mut *rx, &mut *cleanup, None).await + DeathFirst::new( + crate::lifecycle_trace::in_actor(ctx.shared.id.raw(), Box::pin(actor.started(&ctx))), + &mut *rx, + &mut *cleanup, + None, + ) + .await }) .catch_unwind() .await; @@ -2026,7 +2044,10 @@ async fn run_actor_lifecycle( // same call). let caller = envelope.caller(); let result = AssertUnwindSafe(async { - let turn = envelope.handle(actor, &ctx); + let turn = crate::lifecycle_trace::in_actor( + ctx.shared.id.raw(), + envelope.handle(actor, &ctx), + ); DeathFirst::new(turn, &mut *rx, &mut *cleanup, caller).await }) .catch_unwind() @@ -2103,7 +2124,13 @@ async fn run_actor_lifecycle( } let stop_result = AssertUnwindSafe(async { fail_point!("actor.before_stopped", async); - DeathFirst::new(Box::pin(actor.stopped(&ctx)), &mut *rx, &mut *cleanup, None).await + DeathFirst::new( + crate::lifecycle_trace::in_actor(ctx.shared.id.raw(), Box::pin(actor.stopped(&ctx))), + &mut *rx, + &mut *cleanup, + None, + ) + .await }) .catch_unwind() .await; @@ -4101,6 +4128,8 @@ mod turn_regressions { slot: SlotHandle::detached(), #[cfg(feature = "metrics")] observation: None, + #[cfg(feature = "lifecycle-trace")] + from: 0, }) as Box + Send>) .unwrap(); } @@ -4175,6 +4204,8 @@ mod idle_stop_regression { slot, #[cfg(feature = "metrics")] observation: None, + #[cfg(feature = "lifecycle-trace")] + from: 0, }) as Box + Send>) .unwrap(); token.cancel(); diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index d80aecdf..da2c6429 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -147,6 +147,10 @@ struct MessageEnvelope { msg: M, #[cfg(feature = "metrics")] observation: Option, + /// The recorded actor that sent it (0: outside code or a request); its + /// mailbox item kind is then the model's `msg`. + #[cfg(feature = "lifecycle-trace")] + from: u64, } impl Envelope for MessageEnvelope @@ -157,7 +161,11 @@ where /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. #[cfg(feature = "lifecycle-trace")] fn trace_kind(&self) -> &'static str { - crate::lifecycle_trace::life_kind::() + if self.from != 0 { + "msg" + } else { + crate::lifecycle_trace::life_kind::() + } } #[cfg(feature = "metrics")] fn observation(&mut self) -> Option { @@ -218,13 +226,16 @@ where A: Actor + Handler, M: Message, { + let from = crate::lifecycle_trace::life_sender::(replies.owner().raw(), slot.is_detached()); let envelope: Box + Send> = Box::new(MessageEnvelope { msg, slot, #[cfg(feature = "metrics")] observation: Some(_mailbox.enqueued()), + #[cfg(feature = "lifecycle-trace")] + from, }); - admit::(sender, replies, envelope) + admit::(sender, replies, envelope, from) } /// A monitor's `Down`, enqueued by the framework (the waiter at the target's @@ -250,7 +261,7 @@ fn enqueue_down>( #[cfg(feature = "metrics")] observation: Some(_mailbox.enqueued()), }); - admit::(sender, replies, envelope) + admit::(sender, replies, envelope, 0) } /// Admission of a built envelope (`M` names its kind for the trace). @@ -258,6 +269,7 @@ fn admit( sender: &mpsc::Sender>, replies: &Registry, envelope: Box + Send>, + _from: u64, ) -> Result<(), ActorError> { // Admission and its trace event are one step of the lifecycle trace. let owner = replies.owner().raw(); @@ -265,14 +277,14 @@ fn admit( if replies.refuses_admission() { #[cfg(feature = "metrics")] envelope.reject(); - crate::lifecycle_trace::life_send::(owner, false); + crate::lifecycle_trace::life_send::(owner, false, _from); return Err(ActorError::NoProc); } match sender.send(MailboxItem::Message(envelope)) { Ok(()) => { #[cfg(feature = "metrics")] crate::instrumentation::accepted(); - crate::lifecycle_trace::life_send::(owner, true); + crate::lifecycle_trace::life_send::(owner, true, _from); Ok(()) } Err(_error) => { @@ -282,7 +294,7 @@ fn admit( envelope.reject(); } } - crate::lifecycle_trace::life_send::(owner, false); + crate::lifecycle_trace::life_send::(owner, false, _from); Err(ActorError::NoProc) } } @@ -1365,7 +1377,7 @@ fn run_actor_receiving( let _monitor_cleanup = MonitorCleanup(ctx.monitors.clone()); let start_result = catch_unwind(AssertUnwindSafe(|| { fail_point!("actor.before_started"); - actor.started(&ctx); + crate::lifecycle_trace::with_actor(trace_id, || actor.started(&ctx)); })); if let Err(panic) = start_result { let msg = panic_message(&*panic); @@ -1425,7 +1437,7 @@ fn run_actor_receiving( .map(|observation| observation.begin()); ctx.replies.begin_turn(); let result = catch_unwind(AssertUnwindSafe(|| { - envelope.handle(actor, &ctx); + crate::lifecycle_trace::with_actor(trace_id, || envelope.handle(actor, &ctx)); })); ctx.replies.end_turn(result.is_err()); #[cfg(feature = "metrics")] @@ -1483,7 +1495,7 @@ fn run_actor_receiving( if run_hook { let stop_result = catch_unwind(AssertUnwindSafe(|| { fail_point!("actor.before_stopped"); - actor.stopped(&ctx); + crate::lifecycle_trace::with_actor(trace_id, || actor.stopped(&ctx)); })); if let Err(panic) = stop_result { let msg = panic_message(&*panic); @@ -2479,6 +2491,8 @@ mod idle_stop_regression { slot, #[cfg(feature = "metrics")] observation: None, + #[cfg(feature = "lifecycle-trace")] + from: 0, }))) .unwrap(); token.cancel(); diff --git a/concurrency/tests/lifecycle_trace.rs b/concurrency/tests/lifecycle_trace.rs index b032b722..51264dc4 100644 --- a/concurrency/tests/lifecycle_trace.rs +++ b/concurrency/tests/lifecycle_trace.rs @@ -191,6 +191,11 @@ request!(TryLink(ChildHandle) -> bool, |_this, msg, ctx| ctx.try_link(&msg.0).is request!(Unlink(ChildHandle) -> (), |_this, msg, ctx| ctx.unlink(&msg.0)); request!(Monitor(ChildHandle) -> MonitorRef, |_this, msg, ctx| ctx.monitor(&msg.0)); request!(Demonitor(MonitorRef) -> (), |_this, msg, ctx| ctx.demonitor(msg.0)); +request!(Note() -> (), |this, _msg, _ctx| this.note("note")); +request!(NoteThenExit(ActorRef) -> (), |_this, msg, ctx| { + msg.0.send(Note()).unwrap(); + ctx.stop_with_reason(ExitReason::Shutdown); +}); /// A request whose handler panics. struct Boom(); @@ -702,3 +707,105 @@ fn threads_trapped_exit_then_kill() { ], ); } + +/// An actor-to-actor message (the model's `Send`) and the sender's death +/// notice after it (`ExitAfterMessages`): a2, linked to the trapping a1, +/// sends a1 a message in a turn and then stops `Shutdown`; a1 handles the +/// message before the `Exit`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn message_then_death_notice() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::trapping(&log).start(); + let a2 = Node::new(&Log::default()).start(); + a1.request(Sync()).await.unwrap(); + a2.request(Link(a1.child_handle())).await.unwrap(); + a2.request(NoteThenExit(a1.clone())).await.unwrap(); + assert_eq!(exit_of(&a2).await, ExitReason::Shutdown); + wait_for(&log, "exit:Shutdown").await; + assert_eq!(logged(&log), ["note", "exit:Shutdown"]); + a1.child_handle().stop(); + assert_eq!(exit_of(&a1).await, ExitReason::Normal); + let (id1, id2) = (a1.id().to_string(), a2.id().to_string()); + let events = write("message_then_exit"); + has( + &events, + &[ + "a1.Send:queued", + "a2.Stop:Shutdown", + "a2.Die:Shutdown", + "a1.Dispatch", + "a1.Dispatch", + ], + ); + let json = + std::fs::read_to_string(trace_dir().join("lifecycle_message_then_exit.json")).unwrap(); + assert!( + json.contains(&format!(r#""a":"Send","t":{id1},"s":{id2},"x":"queued""#)), + "the message was not recorded as a2's: {json}" + ); +} + +/// Threads mode: an actor-to-actor message is recorded as the sender's. +#[test] +fn threads_actor_message() { + use spawned_concurrency::threads; + let _serial = SERIAL.blocking_lock(); + lifecycle_trace::record_lifecycle(); + struct Native(Log); + impl threads::Actor for Native {} + struct Hello; + impl Message for Hello { + type Result = (); + } + impl threads::Handler for Native { + fn handle(&mut self, _: Hello, _: PendingReply<'_, ()>, _: &threads::Context) { + self.0.lock().unwrap().push("hello".into()); + } + } + struct Forward(threads::ActorRef); + impl Message for Forward { + type Result = (); + } + impl threads::Handler for Native { + fn handle( + &mut self, + msg: Forward, + reply: PendingReply<'_, ()>, + _: &threads::Context, + ) { + msg.0.send(Hello).unwrap(); + reply.reply(()); + } + } + struct Ping; + impl Message for Ping { + type Result = (); + } + impl threads::Handler for Native { + fn handle(&mut self, _: Ping, reply: PendingReply<'_, ()>, _: &threads::Context) { + reply.reply(()); + } + } + use threads::ActorStart as _; + let log = Log::default(); + let a1 = Native(log.clone()).start(); + let a2 = Native(Log::default()).start(); + a2.request(Forward(a1.clone())).unwrap(); + a1.request(Ping).unwrap(); + assert_eq!(logged(&log), ["hello"]); + a1.child_handle().stop(); + a2.child_handle().stop(); + a1.child_handle().wait_exit_blocking(); + a2.child_handle().wait_exit_blocking(); + let (id1, id2) = (a1.id().to_string(), a2.id().to_string()); + let events = write("threads_actor_message"); + has(&events, &["a1.Send:queued", "a1.Dispatch"]); + let json = + std::fs::read_to_string(trace_dir().join("lifecycle_threads_actor_message.json")).unwrap(); + assert!( + json.contains(&format!(r#""a":"Send","t":{id1},"s":{id2},"x":"queued""#)), + "the message was not recorded as a2's: {json}" + ); +} diff --git a/verification/model-map.toml b/verification/model-map.toml index c8f4c170..1e8c8401 100644 --- a/verification/model-map.toml +++ b/verification/model-map.toml @@ -216,7 +216,7 @@ Held = "a signal held while started() runs (phase 0) is outside Lifecycle.tla; t EarlyFire = "fault device only (Fault = EarlyNotify: links and monitors fire when exit begins); no code path takes it" [trace.translated] Spawn = { into = [], why = "trace_check.py numbers actors from Spawn events" } -Send = { into = ["Request", "Enqueue"], why = "trace_check.py splits Send into Request and Enqueue" } +Send = { into = ["Request", "Enqueue", "Send"], why = "trace_check.py splits a Send from outside code into Request and Enqueue, and keeps one from a recorded actor as the model's Send" } [[trace]] struct = "JobEvent" diff --git a/verification/tla/TraceLifecycle.tla b/verification/tla/TraceLifecycle.tla index 17674b30..d7489486 100644 --- a/verification/tla/TraceLifecycle.tla +++ b/verification/tla/TraceLifecycle.tla @@ -12,6 +12,8 @@ (* Trap ctx.trap_exit, when the flag changes (SetTrap) *) (* Request, a send or request admitted ("queued") or refused ("noproc") *) (* Enqueue (one recorded Send; trace_check.py splits it in two) *) +(* Send a plain message from a recorded actor's turn or hook to *) +(* another, admitted (Send; its Dispatch has kind "msg") *) (* Dispatch the run loop takes the mailbox head of kind k, in the same *) (* step as its stop check (Dispatch); p: the turn of a request *) (* panicked (the model's panic branch) *) @@ -32,9 +34,11 @@ (* Identities: the recorded actors are a1, a2, ... in spawn order (their *) (* start_linked parents give LinkParent; the model starts them all, idle, *) (* and a Spawn event is not replayed). Outside code is Ext; a supervisor's *) -(* signal is Sup and its target is in Children. Every send or request is a *) -(* request from Ext: the model's actor-to-actor Send is not recorded (the *) -(* recorder does not know the sending actor), so MaxSends = 0. *) +(* signal is Sup and its target is in Children. A plain message that a *) +(* recorded actor sends to another during a turn or hook (the recorder *) +(* tracks the actor whose turn a thread runs) is the model's Send; every *) +(* other send or request (from outside code, an unrecorded actor, to *) +(* itself, a request, or refused) is a request from Ext. *) (* *) (* Mapping notes: *) (* - Handlers are atomic in the model. Dispatch is recorded when the *) @@ -56,7 +60,7 @@ (* model's reason classes and is a stutter step. *) (* - Not in the mapping (an event the model cannot take rejects the *) (* trace): phase 0 (signals held while started() runs, recorded as *) -(* Held), managed scopes' own stops, actor-to-actor sends, time. *) +(* Held), managed scopes' own stops, actor-to-actor requests, time. *) (* *) (* There are no unlogged steps: every model step is a logged event, so *) (* TLC replays a trace almost deterministically. The trace is accepted iff *) @@ -121,6 +125,7 @@ Step(e) == CASE e.a = "Trap" -> SetTrap(e.t, e.x = "true") [] e.a = "Request" -> Request(e.q, e.t) [] e.a = "Enqueue" -> Enqueue(e.q) /\ req'[e.q].s = e.x + [] e.a = "Send" -> Send(e.s, e.t) [] e.a = "Dispatch" -> DispatchAs(e.t, e.k, e.p) [] e.a = "Panic" -> PanicAs(e.t, e.p) [] e.a = "Stop" -> SelfExit(e.t, e.x) \/ AdminStop(e.t, e.x) diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 80dd3fab..69fed1f8 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -416,6 +416,29 @@ def drop_keeps_normal(events): require_rejected(by_name["lifecycle_runtime_drop"], drop_keeps_normal, "runtime drop ends Normal instead of Killed (v0.7)", tlc) + # Actor-to-actor messages (the model's Send). + def send_after_death(events): + send = index(events, "Send") + die = index(events, "Die", "Shutdown") + events.insert(die, events.pop(send)) + require_rejected(by_name["lifecycle_message_then_exit"], send_after_death, + "message sent by an actor after its death point", tlc) + + def notice_before_message(events): + send = events.pop(index(events, "Send")) + deliver = index(events, "Deliver") + events.insert(deliver + 1, send) + require_rejected(by_name["lifecycle_message_then_exit"], notice_before_message, + "death notice overtakes the dying actor's earlier message", tlc) + + def message_as_request(events): + msg = index(events, "Dispatch") + while events[msg]["k"] != "msg": + msg = index(events, "Dispatch", after=msg + 1) + events[msg]["k"] = "req" + require_rejected(by_name["lifecycle_threads_actor_message"], message_as_request, + "actor-to-actor message dispatched as an outside request", tlc) + def job_trace_controls(traces, tlc): """Edits of recorded pipe-to-self job traces that DeferredJobs.tla must diff --git a/verification/tla/trace_check.py b/verification/tla/trace_check.py index 7b1d6600..07d90fe2 100644 --- a/verification/tla/trace_check.py +++ b/verification/tla/trace_check.py @@ -184,7 +184,7 @@ def actor(raw): folded |= {turn[t], i} turn[t] = None - monitors, requests, children = {}, 0, set() + monitors, requests, children, sends = {}, 0, set(), 0 signals = late = 0 records = [] @@ -199,6 +199,11 @@ def record(a, t="none", s="none", x="", m=0, q=0, k="", o="", p=False): continue t = actor(e["t"]) if a == "Send": + # A plain message from a recorded actor: the model's Send(s, t). + if int(e.get("s", 0)) and e["x"] == "queued": + sends += 1 + record("Send", t=t, s=actor(e["s"])) + continue requests += 1 record("Request", t=t, q=requests) record("Enqueue", t=t, q=requests, x=e["x"]) @@ -229,6 +234,7 @@ def record(a, t="none", s="none", x="", m=0, q=0, k="", o="", p=False): "monitors": len(monitors), "signals": signals, "late": late, + "sends": sends, } return records, constants @@ -259,7 +265,7 @@ def tla_set(values): out.write("INIT TraceInit\nNEXT TraceNext\nCONSTANTS\n" " Actors <- TraceActors\n Children <- TraceChildren\n" " LinkParent <- TraceLinkParent\n Reqs <- TraceReqs\n" - " MonIds <- TraceMonIds\n MaxSends = 0\n" + f" MonIds <- TraceMonIds\n MaxSends = {c['sends']}\n" f" MaxSignals = {c['signals']}\n MaxLate = {c['late']}\n" " UnlinkEnabled = TRUE\n TrapCanClear = TRUE\n" " SelfReasons <- Reasons\n SignalReasons <- Reasons\n" From ff4e3c66fd5386f8d5237d1fd1220b897b696b2b Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 17:50:43 +0000 Subject: [PATCH 168/208] test: long-running soak harness for resource leaks (#257) * test: soak harness (WIP) * test: soak harness binary (workload, sampler, verdict, thresholds) * test(soak): stop actors left by a failed episode; tolerate reply-before-timeout * docs: soak harness in TESTING.md, ROADMAP row and release checklist * test(soak): fit count trends on window floors * docs(soak): describe floor-based trends and limits * test(soak): minimum-growth floor for trend failures * docs(soak): actor counts from the 10-minute validation run --- .github/workflows/release.yaml | 2 + CHANGELOG.md | 9 + ROADMAP.md | 2 +- concurrency/Cargo.toml | 8 + concurrency/benches/soak/main.rs | 576 +++++++++++++++++ concurrency/benches/soak/native.rs | 356 ++++++++++ concurrency/benches/soak/report.rs | 297 +++++++++ concurrency/benches/soak/state.rs | 180 ++++++ concurrency/benches/soak/thresholds.toml | 62 ++ concurrency/benches/soak/workload.rs | 786 +++++++++++++++++++++++ docs/TESTING.md | 114 +++- 11 files changed, 2390 insertions(+), 2 deletions(-) create mode 100644 concurrency/benches/soak/main.rs create mode 100644 concurrency/benches/soak/native.rs create mode 100644 concurrency/benches/soak/report.rs create mode 100644 concurrency/benches/soak/state.rs create mode 100644 concurrency/benches/soak/thresholds.toml create mode 100644 concurrency/benches/soak/workload.rs diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 389e49c0..d1f1ec6e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,3 +1,5 @@ +# Before publishing a release, follow docs/TESTING.md "Release checklist" +# (CI and client-compat green, a 4-hour soak on the release commit). on: release: types: [published] diff --git a/CHANGELOG.md b/CHANGELOG.md index 890d4357..1b1d91f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -400,6 +400,15 @@ it is computed from, so unrelated models no longer invalidate it, and is regenerated (same keys and state counts). `check.sh` gains `--only` and `--exclude`. +- Tests: a long-running soak harness (`concurrency/benches/soak`, run by hand, + not in CI). It churns actors in both modes and every backend, managed + groups, supervisors with every strategy, links, monitors, `trap_exit`, + timers, the registry, timed-out requests, stream listeners, `ActorSystem` + cycles and mailbox bursts from a seed, samples RSS, live heap, OS threads, + file descriptors, Tokio tasks, registry size, live actors and the `metrics` + gauges into a CSV, fails on growth per hour after warm-up or on any count + not returning to its baseline after quiesce, and has a `--inject-leak` + negative control. Run it before each release (`docs/TESTING.md`). ## 0.7.3 — 2026-11-10 diff --git a/ROADMAP.md b/ROADMAP.md index d47f7d82..349ab350 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -388,7 +388,7 @@ cargo-mutants and TLA+ with trace replay. Release gates and remaining gaps: | v0.7: implemented | Supervisor policies under proptest | Generated crash/exit/shutdown scripts for all strategies and modes, using zero backoff and an unexhausted budget; budget/backoff races have separate controlled tests. [Scope and replay](docs/TESTING.md#generated-supervisor-policy-scripts). | | Client rollout: pending | Fault injection in the Ethlambda canary | Kill and restart supervised actors in a running client and check it recovers; the application rollout gate (rank 5) should exercise failures, not just the happy path. | | After v0.7 | Shuttle for multi-actor framework races | Randomised schedules over whole supervisor/registry tests too large for Loom. Listed in TESTING.md, not started. Internal tool, separate from rank 20. | -| After v0.7 | Long-running soak tests | Memory, file descriptors, threads and tasks stay flat over hours of actor churn and restarts. The ethrex 1.5 GB/day mailbox leak is the motivating bug. Nightly, not per PR. | +| Implemented | Long-running soak tests | `concurrency/benches/soak`: hours of seeded churn over every lifecycle path (both modes, every backend, groups, all supervisor strategies, links, monitors, timers, registry, timeouts, streams, system cycles, bursts); RSS, live heap, threads, fds, Tokio tasks, registry, live actors and `metrics` gauges sampled to CSV; fails on per-hour growth after warm-up or a count not returning to baseline after quiesce; `--inject-leak` negative control. The ethrex 1.5 GB/day mailbox leak is the motivating bug. Run by hand on fede before each release, never scheduled ([soak harness](docs/TESTING.md#soak-harness)). | | With ranks 11-13 | Loom on the new primitives | Reply slots, registry compare-and-remove and mailbox publication versus stop. | | Ongoing | Coverage report (`cargo-llvm-cov`) | Find untested branches in supervision and system code; pairs with the surviving mutants. Report only, no threshold. | | Implemented | Porcupine-style linearizability checker for `request_histories` | `spawned_test::linearizability` (public, no new dependencies): memoised Wing-Gong/Lowe search, per-key partitions, pending requests, an `Unknown` budget verdict and failure explanations with an HTML timeline. `request_histories` now checks PropEr-style `parallel_commands` histories of up to about 1,000 operations against a real actor in both modes, with injected-bug negative controls. Next: check histories from soak and canary runs. See [Porcupine](docs/research/systematic-testing.md#history-checkers). | diff --git a/concurrency/Cargo.toml b/concurrency/Cargo.toml index 8f3e04e2..0872cfc1 100644 --- a/concurrency/Cargo.toml +++ b/concurrency/Cargo.toml @@ -103,6 +103,14 @@ name = "runtime_isolation" harness = false required-features = ["backend-experiments"] +# Long-running resource-leak soak, run by hand (docs/TESTING.md, "Soak harness"). +# Not built by `cargo test`; `--all-targets` clippy keeps it compiling. +[[bench]] +name = "soak" +path = "benches/soak/main.rs" +harness = false +required-features = ["async", "metrics"] + [[bench]] name = "metrics_overhead" harness = false diff --git a/concurrency/benches/soak/main.rs b/concurrency/benches/soak/main.rs new file mode 100644 index 00000000..ee088e5e --- /dev/null +++ b/concurrency/benches/soak/main.rs @@ -0,0 +1,576 @@ +//! Long-running soak harness: hours of seeded actor churn, with resource +//! series sampled into a CSV and judged for growth. Linux only (`/proc`). +//! Run by hand, never in CI; see `docs/TESTING.md` ("Soak harness"). +//! +//! ```text +//! CARGO_PROFILE_BENCH_DEBUG=line-tables-only cargo bench -p spawned-concurrency \ +//! --features metrics --bench soak -- --duration 4h --seed 1 --out /path/to/out +//! ``` +//! +//! Exit code 0 is PASS, 1 is FAIL (a trend, a quiesce check or the workload), +//! 2 is a usage error. +mod native; +mod report; +mod state; +mod workload; + +use report::{Phase, Sample, Thresholds, SERIES}; +use spawned_concurrency::{metrics, registry}; +use state::{Leaks, Rng, Stats, LEAKS, LIMIT, LIVE, STARTED}; +use std::{ + alloc::{GlobalAlloc, Layout, System}, + fs::{self, File}, + io::Write, + path::PathBuf, + sync::{ + atomic::{AtomicBool, AtomicI64, AtomicU8, Ordering::Relaxed}, + mpsc, Arc, Mutex, + }, + time::{Duration, Instant}, +}; + +// --------------------------------------------------------------------------- +// Live-heap counter. Harness binary only: the library never installs an +// allocator. It costs one shared atomic per allocation, which slows the +// workload a little and does not change what is measured. +// --------------------------------------------------------------------------- + +static HEAP: AtomicI64 = AtomicI64::new(0); +struct Counting; +// SAFETY: every call forwards to `System` unchanged; the counter only observes sizes. +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let p = unsafe { System.alloc(layout) }; + if !p.is_null() { + HEAP.fetch_add(layout.size() as i64, Relaxed); + } + p + } + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let p = unsafe { System.alloc_zeroed(layout) }; + if !p.is_null() { + HEAP.fetch_add(layout.size() as i64, Relaxed); + } + p + } + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { System.dealloc(ptr, layout) }; + HEAP.fetch_sub(layout.size() as i64, Relaxed); + } + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new: usize) -> *mut u8 { + let p = unsafe { System.realloc(ptr, layout, new) }; + if !p.is_null() { + HEAP.fetch_add(new as i64 - layout.size() as i64, Relaxed); + } + p + } +} +#[global_allocator] +static GLOBAL: Counting = Counting; + +/// Bound for one async scenario; exceeding it is reported as a stuck episode. +const SCENARIO_LIMIT: Duration = Duration::from_secs(60); + +struct Args { + duration: Duration, + seed: u64, + sample: Duration, + warmup: Option, + lanes: usize, + workers: usize, + out: PathBuf, + config: PathBuf, + leaks: Leaks, + raw: String, +} + +fn parse_duration(s: &str) -> Result { + let mut total = 0f64; + let mut number = String::new(); + for c in s.chars() { + if c.is_ascii_digit() || c == '.' { + number.push(c); + continue; + } + let n: f64 = number.parse().map_err(|_| format!("bad duration {s}"))?; + number.clear(); + total += n * match c { + 'h' => 3600.0, + 'm' => 60.0, + 's' => 1.0, + _ => return Err(format!("bad duration unit in {s} (use h, m, s)")), + }; + } + if !number.is_empty() { + total += number + .parse::() + .map_err(|_| format!("bad duration {s}"))?; + } + Ok(Duration::from_secs_f64(total)) +} + +fn parse_args() -> Result { + let mut pairs = Vec::new(); + let mut it = std::env::args().skip(1).filter(|a| a != "--bench"); + while let Some(arg) = it.next() { + let key = arg + .strip_prefix("--") + .ok_or_else(|| format!("unexpected argument {arg}"))?; + let (key, value) = match key.split_once('=') { + Some((k, v)) => (k.to_string(), v.to_string()), + None => { + let v = it + .next() + .ok_or_else(|| format!("missing value for --{key}"))?; + (key.to_string(), v) + } + }; + pairs.push((key, value)); + } + let get = |k: &str| pairs.iter().rev().find(|p| p.0 == k).map(|p| p.1.clone()); + for (k, _) in &pairs { + if ![ + "duration", + "seed", + "sample", + "warmup", + "lanes", + "workers", + "out", + "config", + "inject-leak", + ] + .contains(&k.as_str()) + { + return Err(format!("unknown option --{k}")); + } + } + let duration = parse_duration(&get("duration").unwrap_or("10m".into()))?; + let seed = get("seed") + .unwrap_or("1".into()) + .parse() + .map_err(|_| "bad --seed")?; + let sample = match get("sample") { + Some(s) => parse_duration(&s)?, + None => Duration::from_secs((duration.as_secs() / 360).clamp(1, 10)), + }; + let warmup = get("warmup").map(|s| parse_duration(&s)).transpose()?; + let lanes = get("lanes") + .unwrap_or("4".into()) + .parse() + .map_err(|_| "bad --lanes")?; + let workers = get("workers") + .unwrap_or("4".into()) + .parse() + .map_err(|_| "bad --workers")?; + let unix = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + let out = get("out") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(format!("soak-out/seed{seed}-{unix}"))); + let config = get("config").map(PathBuf::from).unwrap_or_else(|| { + PathBuf::from(concat!( + env!("CARGO_MANIFEST_DIR"), + "/benches/soak/thresholds.toml" + )) + }); + let mut leaks = Leaks::default(); + if let Some(list) = get("inject-leak") { + for kind in list.split(',') { + match kind { + "registry" => leaks.registry = true, + "arc" => leaks.arc = true, + "task" => leaks.task = true, + "all" => { + leaks = Leaks { + registry: true, + arc: true, + task: true, + } + } + _ => return Err(format!("unknown leak {kind} (registry, arc, task, all)")), + } + } + } + if lanes == 0 || workers == 0 || duration.is_zero() || sample.is_zero() { + return Err("--duration, --sample, --lanes and --workers must be positive".into()); + } + let raw = pairs + .iter() + .map(|(k, v)| format!("--{k} {v}")) + .collect::>() + .join(" "); + Ok(Args { + duration, + seed, + sample, + warmup, + lanes, + workers, + out, + config, + leaks, + raw, + }) +} + +fn proc_status(key: &str) -> f64 { + fs::read_to_string("/proc/self/status") + .ok() + .and_then(|s| { + s.lines().find_map(|l| { + l.strip_prefix(key) + .and_then(|v| v.split_whitespace().next()?.parse::().ok()) + }) + }) + .unwrap_or(f64::NAN) +} +fn count_dir(path: &str) -> f64 { + fs::read_dir(path).map_or(f64::NAN, |d| d.count() as f64) +} + +/// Shared sink for samples: kept in memory for the verdict and appended to the CSV. +struct Recorder { + started: Instant, + phase: AtomicU8, + csv: Mutex, + samples: Mutex>, + runtime: tokio::runtime::Handle, + stats: Arc, +} +impl Recorder { + fn set_phase(&self, phase: Phase) { + self.phase.store(phase as u8, Relaxed); + } + fn phase(&self) -> Phase { + match self.phase.load(Relaxed) { + 0 => Phase::Baseline, + 1 => Phase::Warmup, + 2 => Phase::Steady, + _ => Phase::Quiesce, + } + } + fn sample(&self) -> Sample { + let m = metrics::snapshot(); + let values = [ + proc_status("VmRSS:") / 1024.0, + HEAP.load(Relaxed) as f64 / (1024.0 * 1024.0), + count_dir("/proc/self/task"), + count_dir("/proc/self/fd"), + self.runtime.metrics().num_alive_tasks() as f64, + registry::registered().len() as f64, + LIVE.load(Relaxed) as f64, + m.actors_active as f64, + m.mailbox_depth as f64, + m.handlers_active as f64, + m.requests_active as f64, + ]; + debug_assert_eq!(values.len(), SERIES.len()); + let sample = Sample { + elapsed: self.started.elapsed().as_secs_f64(), + phase: self.phase(), + values, + started: STARTED.load(Relaxed), + scenarios: self.stats.total_runs(), + errors: self.stats.errors(), + }; + let mut csv = self.csv.lock().unwrap_or_else(|p| p.into_inner()); + let _ = writeln!(csv, "{}", report::csv_row(&sample)); + let _ = csv.flush(); + drop(csv); + self.samples + .lock() + .unwrap_or_else(|p| p.into_inner()) + .push(sample.clone()); + sample + } +} + +fn lane_seed(seed: u64, lane: u64) -> u64 { + Rng::new(seed ^ lane.wrapping_mul(0xa076_1d64_78bd_642f)).next() +} + +async fn async_lane(env: Arc, mut rng: Rng) { + while !env.stats.stopping() { + let pick = rng.below(state::ASYNC_SCENARIOS as u64) as usize; + let scope = Arc::new(Mutex::new(workload::Scope::default())); + let episode = workload::SCOPE.scope( + scope.clone(), + tokio::time::timeout(SCENARIO_LIMIT, workload::run(pick, &mut rng, &env)), + ); + let outcome = episode.await; + if !matches!(outcome, Ok(Ok(()))) { + scope.lock().unwrap_or_else(|p| p.into_inner()).clean_up(); + } + match outcome { + Ok(Ok(())) => env.stats.ran(pick), + Ok(Err(err)) => env + .stats + .error(format!("{}: {err}", state::SCENARIOS[pick])), + Err(_) => env.stats.error(format!( + "{}: stuck for {SCENARIO_LIMIT:?}", + state::SCENARIOS[pick] + )), + } + } +} + +fn threads_lane(stats: &Stats, mut rng: Rng, once: bool) { + let mut first = 0; + while (once && first < native::COUNT) || (!once && !stats.stopping()) { + let pick = if once { + first += 1; + first - 1 + } else { + rng.below(native::COUNT as u64) as usize + }; + let index = state::ASYNC_SCENARIOS + pick; + let outcome = native::run(pick, &mut rng, stats); + native::clean_up(outcome.is_err()); + match outcome { + Ok(()) => stats.ran(index), + Err(err) => stats.error(format!("{}: {err}", state::SCENARIOS[index])), + } + } +} + +fn main() { + let args = match parse_args() { + Ok(a) => a, + Err(err) => { + eprintln!("soak: {err}"); + eprintln!( + "usage: soak [--duration 4h] [--seed N] [--sample 10s] [--warmup 30m] [--lanes 4] \ + [--workers 4] [--out DIR] [--config thresholds.toml] [--inject-leak registry,arc,task|all]" + ); + std::process::exit(2); + } + }; + if !std::path::Path::new("/proc/self/task").is_dir() { + eprintln!("soak: needs Linux /proc (threads, fds and RSS are read from it)"); + std::process::exit(2); + } + let thresholds = match Thresholds::load(&args.config) { + Ok(t) => t, + Err(err) => { + eprintln!("soak: {err}"); + std::process::exit(2); + } + }; + LEAKS.set(args.leaks).unwrap(); + // Counters and gauges only: no sampled clocks. + let _ = metrics::configure(metrics::Config { + timing_sample_every: 0, + }); + // Injected crashes are expected millions of times; keep other panics loud. + let default_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + let injected = info + .payload() + .downcast_ref::<&str>() + .is_some_and(|s| s.starts_with("soak:")); + if !injected { + default_hook(info); + } + })); + let warmup = args.warmup.unwrap_or_else(|| { + let fraction = thresholds.or("warmup.fraction", 0.25); + let max = thresholds.or("warmup.max_secs", 1800.0); + Duration::from_secs_f64((args.duration.as_secs_f64() * fraction).min(max)) + }); + fs::create_dir_all(&args.out).expect("create --out directory"); + let mut csv = File::create(args.out.join("samples.csv")).expect("create samples.csv"); + writeln!(csv, "{}", report::csv_header()).unwrap(); + + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(args.workers) + .thread_keep_alive(Duration::from_secs(2)) + .thread_name("soak-worker") + .enable_all() + .build() + .expect("runtime"); + let stats = Arc::new(Stats::new()); + let recorder = Arc::new(Recorder { + started: Instant::now(), + phase: AtomicU8::new(Phase::Baseline as u8), + csv: Mutex::new(csv), + samples: Mutex::new(Vec::new()), + runtime: runtime.handle().clone(), + stats: stats.clone(), + }); + println!( + "soak: {} for {:?} (warm-up {:?}, sample {:?}), output {}", + args.raw, + args.duration, + warmup, + args.sample, + args.out.display() + ); + + // Prime every scenario once, so one-time lazy initialisation (runtime + // slabs, registry capacity, thread-locals) lands before the baseline. + let prime_stats = Arc::new(Stats::new()); + runtime.block_on(async { + let env = Arc::new(workload::Env::new(prime_stats.clone()).expect("prime system")); + let mut rng = Rng::new(lane_seed(args.seed, 999)); + for pick in 0..state::ASYNC_SCENARIOS { + if let Err(err) = workload::run(pick, &mut rng, &env).await { + stats.error(format!("prime {}: {err}", state::SCENARIOS[pick])); + } + } + let report = env.system.shutdown(LIMIT).await; + assert!(report.complete, "prime system shutdown incomplete"); + }); + threads_lane(&prime_stats, Rng::new(lane_seed(args.seed, 998)), true); + std::thread::sleep(Duration::from_secs(4)); + + // Sampler: the baseline first, then one row per interval until quiesce. + for _ in 0..3 { + recorder.sample(); + std::thread::sleep(Duration::from_secs(1)); + } + recorder.set_phase(Phase::Warmup); + let sampling = Arc::new(AtomicBool::new(true)); + let sampler = { + let (recorder, sampling, every) = (recorder.clone(), sampling.clone(), args.sample); + std::thread::Builder::new() + .name("soak-sampler".into()) + .spawn(move || { + let mut next = Instant::now(); + while sampling.load(Relaxed) { + next += every; + recorder.sample(); + while sampling.load(Relaxed) && Instant::now() < next { + std::thread::sleep(Duration::from_millis(100)); + } + } + }) + .unwrap() + }; + + let env = Arc::new( + runtime + .block_on(async { workload::Env::new(stats.clone()) }) + .expect("system"), + ); + let lanes: Vec<_> = (0..args.lanes) + .map(|i| { + let env = env.clone(); + runtime.spawn(async_lane(env, Rng::new(lane_seed(args.seed, i as u64)))) + }) + .collect(); + let (native_done, native_finished) = mpsc::channel(); + { + let (stats, seed) = (stats.clone(), lane_seed(args.seed, args.lanes as u64)); + std::thread::Builder::new() + .name("soak-threads-lane".into()) + .spawn(move || { + threads_lane(&stats, Rng::new(seed), false); + let _ = native_done.send(()); + }) + .unwrap(); + } + + let begin = Instant::now(); + let report_every = Duration::from_secs(60).max(args.sample); + let mut next_report = begin + report_every; + while begin.elapsed() < args.duration { + if begin.elapsed() >= warmup { + recorder.set_phase(Phase::Steady); + } + if Instant::now() >= next_report { + next_report += report_every; + println!( + "soak: {:>6.0}s actors started {} scenarios {} errors {} heap {:.1} MiB", + begin.elapsed().as_secs_f64(), + STARTED.load(Relaxed), + stats.total_runs(), + stats.errors(), + HEAP.load(Relaxed) as f64 / 1048576.0 + ); + } + std::thread::sleep(Duration::from_millis(200)); + } + + // Quiesce: stop the lanes, shut the long-lived system down, then wait for + // every count to return to its baseline. + stats.stop.store(true, Relaxed); + let quiesce_start = Instant::now(); + let mut lanes_joined = runtime.block_on(async { + tokio::time::timeout(SCENARIO_LIMIT * 2, futures::future::join_all(lanes)) + .await + .is_ok() + }); + lanes_joined &= native_finished.recv_timeout(SCENARIO_LIMIT * 2).is_ok(); + let system_shutdown = runtime.block_on(async { + let report = env.system.shutdown(LIMIT).await; + if report.complete { + Ok(()) + } else { + Err(format!( + "groups {:?}, actors {:?}", + report.unfinished_groups, report.unfinished_actor_details + )) + } + }); + drop(env); + sampling.store(false, Relaxed); + sampler.join().unwrap(); + recorder.set_phase(Phase::Quiesce); + let timeout = Duration::from_secs_f64(thresholds.or("quiesce.timeout_secs", 60.0)); + let baseline = recorder + .samples + .lock() + .unwrap() + .iter() + .rev() + .find(|s| matches!(s.phase, Phase::Baseline)) + .unwrap() + .clone(); + let settled = loop { + std::thread::sleep(Duration::from_secs(1)); + let now = recorder.sample(); + let ok = SERIES.iter().enumerate().all(|(i, name)| { + thresholds + .get(&format!("quiesce.{name}")) + .is_none_or(|slack| now.values[i] <= baseline.values[i] + slack) + }); + if ok { + break true; + } + if quiesce_start.elapsed() > timeout { + break false; + } + }; + let samples = recorder.samples.lock().unwrap().clone(); + let verdict = report::judge( + &thresholds, + &samples, + &report::RunInfo { + args: &args.raw, + warmup_secs: warmup.as_secs_f64(), + runs: stats.runs(), + capacity: stats.capacity.load(Relaxed), + errors: stats.errors(), + first_errors: stats.first_errors(), + quiesce_secs: quiesce_start.elapsed().as_secs_f64(), + quiesce_settled: settled, + lanes_joined, + system_shutdown, + }, + ); + let leaks = if args.leaks.registry || args.leaks.arc || args.leaks.task { + format!("injected leaks: {:?}\n", args.leaks) + } else { + String::new() + }; + let text = format!("{leaks}{}", verdict.text); + fs::write(args.out.join("summary.txt"), &text).expect("write summary.txt"); + println!("\n{text}"); + println!("samples: {}", args.out.join("samples.csv").display()); + // Injected leaks keep a pending task on the runtime: don't wait for it. + runtime.shutdown_background(); + std::process::exit(if verdict.pass { 0 } else { 1 }); +} diff --git a/concurrency/benches/soak/native.rs b/concurrency/benches/soak/native.rs new file mode 100644 index 00000000..f102b86c --- /dev/null +++ b/concurrency/benches/soak/native.rs @@ -0,0 +1,356 @@ +//! Threads-mode scenarios (no async runtime), run on their own OS-thread lane. +use crate::state::{e, ensure, leaks, Live, Rng, Stats, LIMIT, R, STARTED}; +use spawned_concurrency::{ + message::Message, + registry, + threads::{ + send_after, send_interval, spawn_listener, Actor, ActorRef, ActorStart, Context, Handler, + TimerHandle, + }, + ActorError, ChildHandle, Down, Exit, ExitReason, MonitorRef, +}; +use std::time::{Duration, Instant}; + +pub struct NWorker { + _live: Live, + value: u64, + timers: Vec, +} +impl NWorker { + fn new() -> Self { + Self { + _live: Live::new(), + value: 0, + timers: Vec::new(), + } + } +} +impl Actor for NWorker { + fn started(&mut self, _: &Context) { + STARTED.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } +} +macro_rules! message { + ($name:ident $(($($field:ty),*))? => $result:ty) => { + pub struct $name $(($(pub $field),*))?; + impl Message for $name { + type Result = $result; + } + }; +} +message!(Inc => u64); +message!(Get => u64); +message!(Crash => ()); +message!(Slow(Duration) => ()); +message!(ArmTimers(u64, u64, Duration) => ()); +message!(CancelTimers => ()); +#[derive(Clone)] +pub struct Bump; +impl Message for Bump { + type Result = (); +} +impl Handler for NWorker { + fn handle(&mut self, _: Inc, _: &Context) -> u64 { + self.value += 1; + self.value + } +} +impl Handler for NWorker { + fn handle(&mut self, _: Get, _: &Context) -> u64 { + self.value + } +} +impl Handler for NWorker { + fn handle(&mut self, _: Bump, _: &Context) { + self.value += 1; + } +} +impl Handler for NWorker { + fn handle(&mut self, _: Crash, _: &Context) { + panic!("soak: injected crash"); + } +} +impl Handler for NWorker { + fn handle(&mut self, msg: Slow, _: &Context) { + std::thread::sleep(msg.0); + } +} +impl Handler for NWorker { + fn handle(&mut self, msg: ArmTimers, ctx: &Context) { + let ArmTimers(once, every, period) = msg; + for i in 0..once { + let delay = period * (i as u32 + 1); + self.timers.push(send_after(delay, ctx.clone(), Bump)); + } + for _ in 0..every { + self.timers.push(send_interval(period, ctx.clone(), Bump)); + } + } +} +impl Handler for NWorker { + fn handle(&mut self, _: CancelTimers, _: &Context) { + for timer in self.timers.drain(..) { + timer.cancellation_token.cancel(); + } + } +} + +pub struct NWatcher { + _live: Live, + downs: u64, + exits: u64, +} +impl Actor for NWatcher { + fn started(&mut self, _: &Context) { + STARTED.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + fn exit_received(&mut self, _: Exit, _: &Context) { + self.exits += 1; + } +} +message!(Watch(ChildHandle) => MonitorRef); +message!(Unwatch(MonitorRef) => ()); +message!(LinkTo(ChildHandle) => ()); +message!(Trap(bool) => ()); +message!(Counts => (u64, u64)); +impl Handler for NWatcher { + fn handle(&mut self, msg: Watch, ctx: &Context) -> MonitorRef { + ctx.monitor(&msg.0) + } +} +impl Handler for NWatcher { + fn handle(&mut self, msg: Unwatch, ctx: &Context) { + ctx.demonitor(msg.0); + } +} +impl Handler for NWatcher { + fn handle(&mut self, msg: LinkTo, ctx: &Context) { + ctx.link(&msg.0); + } +} +impl Handler for NWatcher { + fn handle(&mut self, msg: Trap, ctx: &Context) { + ctx.trap_exit(msg.0); + } +} +impl Handler for NWatcher { + fn handle(&mut self, _: Counts, _: &Context) -> (u64, u64) { + (self.downs, self.exits) + } +} +impl Handler for NWatcher { + fn handle(&mut self, _: Down, _: &Context) { + self.downs += 1; + } +} + +thread_local! { + /// Actors started by the current episode; see `workload::Scope`. + static SCOPE: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; +} +fn track(actor: ActorRef) -> ActorRef { + SCOPE.with(|s| s.borrow_mut().push(actor.child_handle())); + actor +} +/// Forget the last episode's actors, stopping them if it failed half-way. +pub fn clean_up(failed: bool) { + for actor in SCOPE.with(|s| std::mem::take(&mut *s.borrow_mut())) { + if failed { + actor.stop(); + } + } +} + +fn watcher() -> ActorRef { + track( + NWatcher { + _live: Live::new(), + downs: 0, + exits: 0, + } + .start(), + ) +} + +/// Exit within the limit: threads-mode `wait_exit` blocks without a timeout, +/// so poll the exit reason instead. +fn exit(actor: &ActorRef) -> Result { + let deadline = Instant::now() + LIMIT; + loop { + if let Some(reason) = actor.exit_reason() { + return Ok(reason); + } + ensure!(Instant::now() < deadline, "threads actor did not exit"); + std::thread::sleep(Duration::from_micros(200)); + } +} + +fn eventually(what: &str, mut check: impl FnMut() -> Result) -> R { + let deadline = Instant::now() + LIMIT; + while !check()? { + ensure!(Instant::now() < deadline, "timed out: {what}"); + std::thread::sleep(Duration::from_micros(500)); + } + Ok(()) +} + +fn finish(actor: &ActorRef, rng: &mut Rng) -> R { + match rng.below(4) { + 0 => { + ensure!(actor.request(Crash).is_err(), "crash request returned Ok"); + let reason = exit(actor)?; + ensure!( + matches!(reason, ExitReason::Panic(_)), + "crash exit {reason:?}" + ); + } + 1 => { + actor.child_handle().stop_with_reason(ExitReason::Shutdown); + let reason = exit(actor)?; + ensure!(reason == ExitReason::Shutdown, "shutdown exit {reason:?}"); + } + _ => { + actor.child_handle().stop(); + let reason = exit(actor)?; + ensure!(reason == ExitReason::Normal, "stop exit {reason:?}"); + } + } + Ok(()) +} + +fn stop_watcher(watcher: &ActorRef) -> R { + watcher.child_handle().stop(); + let reason = exit(watcher)?; + ensure!(reason == ExitReason::Normal, "watcher exit {reason:?}"); + Ok(()) +} + +pub fn run(index: usize, rng: &mut Rng, stats: &Stats) -> R { + match index { + 0 => churn(rng), + 1 => links_monitors(rng), + 2 => timers_streams(rng), + 3 => registry_requests(rng, stats), + 4 => burst(rng), + _ => unreachable!(), + } +} +pub const COUNT: usize = 5; + +fn churn(rng: &mut Rng) -> R { + let actors: Vec<_> = (0..1 + rng.below(4)) + .map(|_| track(NWorker::new().start())) + .collect(); + for actor in &actors { + for expected in 1..=1 + rng.below(3) { + let got = actor.request(Inc).map_err(e("inc"))?; + ensure!(got == expected, "inc returned {got}, expected {expected}"); + } + } + for actor in &actors { + finish(actor, rng)?; + } + Ok(()) +} + +fn links_monitors(rng: &mut Rng) -> R { + let watcher = watcher(); + let peer = track(NWorker::new().start()); + if rng.one_in(2) { + let trap = rng.one_in(2); + watcher.request(Trap(trap)).map_err(e("trap"))?; + watcher + .request(LinkTo(peer.child_handle())) + .map_err(e("link"))?; + let _ = peer.request(Crash); + exit(&peer)?; + if trap { + eventually("trapped exit delivered", || { + Ok(watcher.request(Counts).map_err(e("counts"))?.1 == 1) + })?; + stop_watcher(&watcher) + } else { + let reason = exit(&watcher)?; + ensure!(reason.is_abnormal(), "linked peer exit {reason:?}"); + Ok(()) + } + } else { + let monitor = watcher + .request(Watch(peer.child_handle())) + .map_err(e("watch"))?; + let expected = if rng.one_in(4) { + watcher.request(Unwatch(monitor)).map_err(e("unwatch"))?; + 0 + } else { + 1 + }; + finish(&peer, rng)?; + eventually("Down delivered", || { + Ok(watcher.request(Counts).map_err(e("counts"))?.0 == expected) + })?; + stop_watcher(&watcher) + } +} + +fn timers_streams(rng: &mut Rng) -> R { + let actor = track(NWorker::new().start()); + if rng.one_in(2) { + let once = 1 + rng.below(3); + let period = Duration::from_millis(1 + rng.below(3)); + actor + .request(ArmTimers(once, rng.below(2), period)) + .map_err(e("arm"))?; + std::thread::sleep(rng.ms(6)); + if rng.one_in(2) { + actor.request(CancelTimers).map_err(e("cancel"))?; + } + } else { + let n = 1 + rng.below(64); + drop(spawn_listener(actor.context(), (0..n).map(|_| Bump))); + eventually("finite listener drained", || { + Ok(actor.request(Get).map_err(e("get"))? == n) + })?; + } + finish(&actor, rng) +} + +fn registry_requests(rng: &mut Rng, stats: &Stats) -> R { + let actor = track(NWorker::new().start()); + let id = stats.next_id(); + let name = format!("soak-native-{id}"); + registry::register(&name, actor.clone()).map_err(e("register"))?; + let found = registry::whereis::>(&name).map(|a| a.id()); + ensure!(found == Some(actor.id()), "whereis {name} -> {found:?}"); + if !(leaks().registry && id.is_multiple_of(4)) { + registry::unregister(&name); + } + let slow = actor.request_with_timeout( + Slow(Duration::from_millis(20 + rng.below(20))), + Duration::from_millis(1), + ); + ensure!( + // A requester descheduled past the handler can see the reply first. + matches!(slow, Err(ActorError::RequestTimeout) | Ok(())), + "slow request: {slow:?}" + ); + ensure!(actor.request(Inc).map_err(e("inc"))? == 1, "inc"); + finish(&actor, rng)?; + ensure!( + matches!(actor.request(Inc), Err(ActorError::ActorStopped)), + "request to a stopped actor" + ); + Ok(()) +} + +fn burst(rng: &mut Rng) -> R { + let actor = track(NWorker::new().start()); + let n = 1_000 + rng.below(10_000); + for _ in 0..n { + actor.send(Bump).map_err(e("burst send"))?; + } + if !rng.one_in(3) { + let got = actor.request_with_timeout(Get, LIMIT).map_err(e("drain"))?; + ensure!(got == n, "drained {got} of {n}"); + } + finish(&actor, rng) +} diff --git a/concurrency/benches/soak/report.rs b/concurrency/benches/soak/report.rs new file mode 100644 index 00000000..1de0cb36 --- /dev/null +++ b/concurrency/benches/soak/report.rs @@ -0,0 +1,297 @@ +//! Thresholds, trend fitting and the pass/fail summary. +use std::{collections::HashMap, fmt::Write as _, path::Path}; + +/// Judged series, in CSV column order after `elapsed_s,phase`. RSS and heap +/// are in MiB; the rest are counts. +pub const SERIES: &[&str] = &[ + "rss_mb", + "heap_mb", + "threads", + "fds", + "tokio_tasks", + "registry", + "actors_live", + "actors_active", + "mailbox_depth", + "handlers_active", + "requests_active", +]; + +/// Flat `key = number` lines (valid TOML with dotted keys); `#` comments. +pub struct Thresholds(HashMap); +impl Thresholds { + pub fn load(path: &Path) -> Result { + let text = std::fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?; + let mut map = HashMap::new(); + for (n, line) in text.lines().enumerate() { + let line = line.split('#').next().unwrap_or("").trim(); + if line.is_empty() { + continue; + } + let (key, value) = line + .split_once('=') + .ok_or_else(|| format!("{}:{}: expected key = value", path.display(), n + 1))?; + let value: f64 = value + .trim() + .parse() + .map_err(|_| format!("{}:{}: not a number", path.display(), n + 1))?; + map.insert(key.trim().to_string(), value); + } + Ok(Self(map)) + } + pub fn get(&self, key: &str) -> Option { + self.0.get(key).copied() + } + pub fn or(&self, key: &str, default: f64) -> f64 { + self.get(key).unwrap_or(default) + } +} + +#[derive(Clone, Copy)] +pub enum Phase { + Baseline, + Warmup, + Steady, + Quiesce, +} +impl Phase { + pub fn name(self) -> &'static str { + match self { + Phase::Baseline => "baseline", + Phase::Warmup => "warmup", + Phase::Steady => "steady", + Phase::Quiesce => "quiesce", + } + } +} + +#[derive(Clone)] +pub struct Sample { + pub elapsed: f64, + pub phase: Phase, + pub values: [f64; 11], + pub started: u64, + pub scenarios: u64, + pub errors: u64, +} + +pub fn csv_header() -> String { + format!( + "elapsed_s,phase,{},actors_started,scenarios,errors", + SERIES.join(",") + ) +} +pub fn csv_row(s: &Sample) -> String { + let mut row = format!("{:.1},{}", s.elapsed, s.phase.name()); + for (i, v) in s.values.iter().enumerate() { + if i < 2 { + write!(row, ",{v:.3}").unwrap(); + } else { + write!(row, ",{v}").unwrap(); + } + } + write!(row, ",{},{},{}", s.started, s.scenarios, s.errors).unwrap(); + row +} + +/// Least-squares slope per hour and its standard error. +pub fn slope_per_hour(points: &[(f64, f64)]) -> Option<(f64, f64)> { + let n = points.len() as f64; + if points.len() < 3 { + return None; + } + let mx = points.iter().map(|p| p.0).sum::() / n; + let my = points.iter().map(|p| p.1).sum::() / n; + let sxx: f64 = points.iter().map(|p| (p.0 - mx).powi(2)).sum(); + if sxx == 0.0 { + return None; + } + let sxy: f64 = points.iter().map(|p| (p.0 - mx) * (p.1 - my)).sum(); + let slope = sxy / sxx; + let residual: f64 = points + .iter() + .map(|p| (p.1 - my - slope * (p.0 - mx)).powi(2)) + .sum(); + let se = (residual / (n - 2.0) / sxx).sqrt(); + Some((slope * 3600.0, se * 3600.0)) +} + +pub struct Verdict { + pub pass: bool, + pub text: String, +} + +pub struct RunInfo<'a> { + pub args: &'a str, + pub warmup_secs: f64, + pub runs: Vec<(&'static str, u64)>, + pub capacity: u64, + pub errors: u64, + pub first_errors: Vec, + pub quiesce_secs: f64, + pub quiesce_settled: bool, + pub lanes_joined: bool, + pub system_shutdown: Result<(), String>, +} + +/// The minimum of each of about 20 consecutive windows, at the window's +/// midpoint. Counts swing with the workload (a burst, a batch of Thread-backend +/// actors) and neighbouring samples are correlated, so a fit on raw samples +/// finds spurious trends; a leak raises the floor, which the workload does not. +fn floors(steady: &[&Sample], i: usize) -> Vec<(f64, f64)> { + let width = (steady.len() / 20).max(5); + steady + .chunks(width) + .filter(|w| w.len() * 2 >= width) + .map(|w| { + let mid = (w[0].elapsed + w[w.len() - 1].elapsed) / 2.0; + ( + mid, + w.iter().map(|s| s.values[i]).fold(f64::INFINITY, f64::min), + ) + }) + .collect() +} + +/// Judge every series: a significant trend above its per-hour limit after +/// warm-up fails, and so does a final quiesce value above baseline + slack. +pub fn judge(t: &Thresholds, samples: &[Sample], info: &RunInfo) -> Verdict { + let sigmas = t.or("trend.sigmas", 3.0); + let baseline = samples + .iter() + .rev() + .find(|s| matches!(s.phase, Phase::Baseline)) + .expect("baseline sample"); + let last = samples.last().expect("final sample"); + let steady: Vec<&Sample> = samples + .iter() + .filter(|s| matches!(s.phase, Phase::Steady)) + .collect(); + let mut pass = true; + let mut table = String::new(); + writeln!( + table, + "{:<16} {:>10} {:>10} {:>10} {:>12} {:>10} {:>10} {:<6} {:<6}", + "series", + "baseline", + "steady-max", + "quiesce", + "slope/h", + "±se/h", + "limit/h", + "trend", + "settle" + ) + .unwrap(); + for (i, name) in SERIES.iter().enumerate() { + let points = if *name == "rss_mb" { + steady.iter().map(|s| (s.elapsed, s.values[i])).collect() + } else { + floors(&steady, i) + }; + let fit = slope_per_hour(&points); + let limit = t.get(&format!("trend.{name}")); + // Growth the fit implies over the steady window: a bounded pool + // adding a few threads in a short run is a steep but small step. + let hours = steady + .last() + .zip(steady.first()) + .map_or(0.0, |(l, f)| (l.elapsed - f.elapsed) / 3600.0); + let min_growth = t.or(&format!("growth.{name}"), 0.0); + let trend_ok = match (fit, limit) { + (Some((slope, se)), Some(limit)) => { + !(slope > limit && slope > sigmas * se && slope * hours > min_growth) + } + _ => true, + }; + let slack = t.get(&format!("quiesce.{name}")); + let settle_ok = slack.is_none_or(|slack| last.values[i] <= baseline.values[i] + slack); + pass &= trend_ok && settle_ok; + let max = steady.iter().map(|s| s.values[i]).fold(f64::NAN, f64::max); + let (slope, se) = fit.map_or(("n/a".into(), "n/a".into()), |(s, e)| { + (format!("{s:.3}"), format!("{e:.3}")) + }); + writeln!( + table, + "{:<16} {:>10} {:>10} {:>10} {:>12} {:>10} {:>10} {:<6} {:<6}", + name, + num(baseline.values[i]), + num(max), + num(last.values[i]), + slope, + se, + limit.map_or("-".into(), num), + if limit.is_none() { + "-" + } else if trend_ok { + "ok" + } else { + "FAIL" + }, + match slack { + None => "-".to_string(), + Some(_) if settle_ok => "ok".into(), + Some(slack) => format!("FAIL(+{})", num(slack)), + }, + ) + .unwrap(); + } + let workload_ok = info.errors == 0 && info.lanes_joined && info.system_shutdown.is_ok(); + pass &= workload_ok; + let mut text = String::new(); + writeln!(text, "spawned soak: {}", if pass { "PASS" } else { "FAIL" }).unwrap(); + writeln!(text, "args: {}", info.args).unwrap(); + writeln!( + text, + "elapsed {:.0}s, warm-up {:.0}s, {} steady samples, quiesce {:.1}s ({})", + last.elapsed, + info.warmup_secs, + steady.len(), + info.quiesce_secs, + if info.quiesce_settled { + "settled" + } else { + "did not settle" + } + ) + .unwrap(); + writeln!( + text, + "actors started {}, scenarios {}, capacity rejections {}, workload errors {}", + last.started, + info.runs.iter().map(|r| r.1).sum::(), + info.capacity, + info.errors + ) + .unwrap(); + if !info.lanes_joined { + writeln!( + text, + "FAIL: a lane did not finish within the limit (stuck scenario)" + ) + .unwrap(); + } + if let Err(err) = &info.system_shutdown { + writeln!(text, "FAIL: long-lived system shutdown: {err}").unwrap(); + } + for err in &info.first_errors { + writeln!(text, "error: {err}").unwrap(); + } + text.push('\n'); + text.push_str(&table); + text.push_str("\nscenario runs:\n"); + for (name, count) in &info.runs { + writeln!(text, " {name:<28} {count}").unwrap(); + } + Verdict { pass, text } +} + +fn num(v: f64) -> String { + if v.is_nan() { + "n/a".into() + } else if v.fract() == 0.0 && v.abs() < 1e12 { + format!("{v:.0}") + } else { + format!("{v:.3}") + } +} diff --git a/concurrency/benches/soak/state.rs b/concurrency/benches/soak/state.rs new file mode 100644 index 00000000..f7656126 --- /dev/null +++ b/concurrency/benches/soak/state.rs @@ -0,0 +1,180 @@ +//! Harness-side counters, the seeded RNG, scenario statistics and leak injection. +use std::sync::{ + atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering::Relaxed}, + Arc, Mutex, OnceLock, +}; +use std::time::Duration; + +/// Bound for any single wait inside a scenario (an exit, a restart, a Down). +pub const LIMIT: Duration = Duration::from_secs(10); + +/// Actor states constructed and not yet dropped: "started minus fully stopped". +pub static LIVE: AtomicI64 = AtomicI64::new(0); +/// `started()` hooks run, both modes. +pub static STARTED: AtomicU64 = AtomicU64::new(0); + +/// Held in every harness actor; its drop is the "fully stopped" event. +pub struct Live(()); +impl Live { + pub fn new() -> Self { + LIVE.fetch_add(1, Relaxed); + Self(()) + } +} +impl Drop for Live { + fn drop(&mut self) { + LIVE.fetch_sub(1, Relaxed); + } +} + +/// SplitMix64: small, seedable and good enough to pick scenarios. +pub struct Rng(u64); +impl Rng { + pub fn new(seed: u64) -> Self { + Self(seed) + } + pub fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) + } + pub fn below(&mut self, n: u64) -> u64 { + self.next() % n.max(1) + } + /// True with probability 1/n. + pub fn one_in(&mut self, n: u64) -> bool { + self.below(n) == 0 + } + pub fn ms(&mut self, max: u64) -> Duration { + Duration::from_millis(self.below(max + 1)) + } +} + +/// Planted leaks for the harness's negative control (`--inject-leak`). +#[derive(Default, Debug, Clone, Copy)] +pub struct Leaks { + /// Never unregister one in four unmanaged registry names. + pub registry: bool, + /// Leak a 4 KiB `Arc` and an actor-state token on every supervised (re)start. + pub arc: bool, + /// Leak a pending Tokio task per supervisor cycle. + pub task: bool, +} +pub static LEAKS: OnceLock = OnceLock::new(); +static LEAKED: Mutex>> = Mutex::new(Vec::new()); +pub fn leaks() -> Leaks { + LEAKS.get().copied().unwrap_or_default() +} +/// Called from every supervised child factory. +pub fn leak_on_restart() { + if leaks().arc { + LEAKED + .lock() + .unwrap_or_else(|p| p.into_inner()) + .push(Arc::from(vec![7u8; 4096])); + std::mem::forget(Live::new()); + } +} + +pub const SCENARIOS: &[&str] = &[ + // async lanes (tasks mode) + "churn_async", + "churn_blocking", + "churn_thread", + "group", + "supervisor_one_for_one", + "supervisor_one_for_all", + "supervisor_rest_for_one", + "supervisor_nested", + "links", + "monitors", + "timers", + "registry", + "requests", + "streams", + "system_cycle", + "burst", + // threads lane (threads mode) + "threads_churn", + "threads_links_monitors", + "threads_timers_streams", + "threads_registry_requests", + "threads_burst", +]; +pub const ASYNC_SCENARIOS: usize = 16; + +pub struct Stats { + runs: Vec, + pub capacity: AtomicU64, + errors: AtomicU64, + first_errors: Mutex>, + pub stop: AtomicBool, + seq: AtomicU64, +} +impl Stats { + pub fn new() -> Self { + Self { + runs: SCENARIOS.iter().map(|_| AtomicU64::new(0)).collect(), + capacity: AtomicU64::new(0), + errors: AtomicU64::new(0), + first_errors: Mutex::new(Vec::new()), + stop: AtomicBool::new(false), + seq: AtomicU64::new(0), + } + } + pub fn ran(&self, scenario: usize) { + self.runs[scenario].fetch_add(1, Relaxed); + } + pub fn error(&self, what: impl Into) { + self.errors.fetch_add(1, Relaxed); + let mut first = self.first_errors.lock().unwrap_or_else(|p| p.into_inner()); + if first.len() < 20 { + let what = what.into(); + eprintln!("soak error: {what}"); + first.push(what); + } + } + pub fn errors(&self) -> u64 { + self.errors.load(Relaxed) + } + pub fn first_errors(&self) -> Vec { + self.first_errors + .lock() + .unwrap_or_else(|p| p.into_inner()) + .clone() + } + pub fn runs(&self) -> Vec<(&'static str, u64)> { + SCENARIOS + .iter() + .zip(&self.runs) + .map(|(n, c)| (*n, c.load(Relaxed))) + .collect() + } + pub fn total_runs(&self) -> u64 { + self.runs.iter().map(|c| c.load(Relaxed)).sum() + } + /// Process-unique suffix for registry names. + pub fn next_id(&self) -> u64 { + self.seq.fetch_add(1, Relaxed) + } + pub fn stopping(&self) -> bool { + self.stop.load(Relaxed) + } +} + +/// `return Err(..)` from a scenario unless the condition holds. +macro_rules! ensure { + ($cond:expr, $($fmt:tt)+) => { + if !$cond { + return Err(format!($($fmt)+)); + } + }; +} +pub(crate) use ensure; + +pub type R = Result<(), String>; +pub fn e(what: &'static str) -> impl FnOnce(T) -> String { + move |err| format!("{what}: {err:?}") +} diff --git a/concurrency/benches/soak/thresholds.toml b/concurrency/benches/soak/thresholds.toml new file mode 100644 index 00000000..940e53a3 --- /dev/null +++ b/concurrency/benches/soak/thresholds.toml @@ -0,0 +1,62 @@ +# Soak harness pass/fail thresholds (concurrency/benches/soak, docs/TESTING.md). +# Flat dotted keys; every value is a number. Delete a key to skip that check. + +# Warm-up: the first `fraction` of the run, capped at `max_secs`, is excluded +# from trend fits (allocator pools, runtime slabs and registry capacity grow +# to their working size). `--warmup` overrides both. +warmup.fraction = 0.25 +warmup.max_secs = 1800 + +# Trend: least-squares slope per hour after warm-up. rss_mb is fitted on +# every sample; the other series on their floor (the minimum of each of about +# 20 consecutive windows), because they swing with the workload and a leak +# raises the floor. A series fails when its slope exceeds the limit AND is at least +# `sigmas` standard errors above zero, so noise in short runs does not fail +# but a steady leak does. rss_mb and heap_mb are MiB per hour; the rest are +# counts per hour. For scale: ethrex's mailbox leak was about 62 MiB/h. +trend.sigmas = 3 +trend.rss_mb = 16 +trend.heap_mb = 4 +trend.threads = 2 +trend.fds = 2 +trend.tokio_tasks = 10 +trend.registry = 2 +trend.actors_live = 10 +trend.actors_active = 10 +trend.mailbox_depth = 100 +trend.handlers_active = 10 +trend.requests_active = 10 + +# Minimum growth: a trend also needs slope x steady-window hours above this +# many units (MiB for rss_mb and heap_mb) to fail. Bounded pools (Tokio's +# blocking pool, group workers) add a few threads or tasks in steps that a +# 10-minute run extrapolates to tens per hour; over a 4-hour run (3 steady +# hours) these floors keep the per-hour limits above as the binding check +# for steady leaks, and the quiesce check below catches anything left over. +growth.rss_mb = 4 +growth.heap_mb = 1 +growth.threads = 8 +growth.fds = 4 +growth.tokio_tasks = 5 +growth.registry = 0 +growth.actors_live = 5 +growth.actors_active = 5 +growth.mailbox_depth = 50 +growth.handlers_active = 3 +growth.requests_active = 3 + +# Quiesce: after the workload stops and the long-lived ActorSystem shuts +# down, each series must return to within `slack` of its baseline (sampled +# after one priming pass of every scenario) before `timeout_secs`. RSS has no +# quiesce check: the allocator keeps freed pages. +quiesce.timeout_secs = 60 +quiesce.heap_mb = 8 +quiesce.threads = 0 +quiesce.fds = 0 +quiesce.tokio_tasks = 0 +quiesce.registry = 0 +quiesce.actors_live = 0 +quiesce.actors_active = 0 +quiesce.mailbox_depth = 0 +quiesce.handlers_active = 0 +quiesce.requests_active = 0 diff --git a/concurrency/benches/soak/workload.rs b/concurrency/benches/soak/workload.rs new file mode 100644 index 00000000..63be5c5d --- /dev/null +++ b/concurrency/benches/soak/workload.rs @@ -0,0 +1,786 @@ +//! Tasks-mode scenarios. Each runs one bounded lifecycle episode and checks +//! its outcome; the lanes in `main.rs` pick them from a seeded RNG forever. +use crate::state::{e, ensure, leak_on_restart, leaks, Live, Rng, Stats, LIMIT, R, STARTED}; +use futures::StreamExt; +use spawned_concurrency::{ + message::Message, + registry, + supervision::{ + Backoff, ChildSpec, SupervisedChild, Supervisor, SupervisorConfig, SupervisorError, + SupervisorSpec, SupervisorStrategy, + }, + system::{ActorSystem, ExecutionGroup, SystemError}, + tasks::{ + send_after, send_interval, spawn_listener, Actor, ActorRef, ActorStart, Backend, Context, + Handler, TimerHandle, + }, + ActorError, ChildHandle, Down, Exit, ExitReason, MonitorRef, +}; +use spawned_rt::tasks as rt; +use std::{sync::Arc, time::Duration}; + +/// Long-lived placement shared by every lane: one system, an async group and a +/// native thread group. Supervisors and managed actors come and go inside it. +pub struct Env { + pub system: ActorSystem, + pub async_group: ExecutionGroup, + pub thread_group: ExecutionGroup, + pub stats: Arc, +} +impl Env { + pub fn new(stats: Arc) -> Result { + let system = ActorSystem::new()?; + let async_group = system.async_group("soak-async", 2)?; + let thread_group = system.thread_group("soak-native", 16)?; + Ok(Self { + system, + async_group, + thread_group, + stats, + }) + } +} + +// --------------------------------------------------------------------------- +// Actors and messages +// --------------------------------------------------------------------------- + +pub struct Worker { + _live: Live, + value: u64, + timers: Vec, +} +impl Worker { + pub fn new() -> Self { + Self { + _live: Live::new(), + value: 0, + timers: Vec::new(), + } + } +} +impl Actor for Worker { + async fn started(&mut self, _: &Context) { + STARTED.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } +} + +macro_rules! message { + ($name:ident $(($($field:ty),*))? => $result:ty) => { + pub struct $name $(($(pub $field),*))?; + impl Message for $name { + type Result = $result; + } + }; +} +message!(Inc => u64); +message!(Get => u64); +message!(Crash => ()); +message!(Slow(Duration) => ()); +message!(ArmTimers(u64, u64, Duration) => ()); +message!(CancelTimers => ()); +message!(RegisterManaged(String) => Result<(), String>); +#[derive(Clone)] +pub struct Bump; +impl Message for Bump { + type Result = (); +} + +impl Handler for Worker { + async fn handle(&mut self, _: Inc, _: &Context) -> u64 { + self.value += 1; + self.value + } +} +impl Handler for Worker { + async fn handle(&mut self, _: Get, _: &Context) -> u64 { + self.value + } +} +impl Handler for Worker { + async fn handle(&mut self, _: Bump, _: &Context) { + self.value += 1; + } +} +impl Handler for Worker { + async fn handle(&mut self, _: Crash, _: &Context) { + panic!("soak: injected crash"); + } +} +impl Handler for Worker { + async fn handle(&mut self, msg: Slow, _: &Context) { + rt::sleep(msg.0).await; + } +} +impl Handler for Worker { + async fn handle(&mut self, msg: ArmTimers, ctx: &Context) { + let ArmTimers(once, every, period) = msg; + for i in 0..once { + let delay = period * (i as u32 + 1); + self.timers.push(send_after(delay, ctx.clone(), Bump)); + } + for _ in 0..every { + self.timers.push(send_interval(period, ctx.clone(), Bump)); + } + } +} +impl Handler for Worker { + async fn handle(&mut self, _: CancelTimers, _: &Context) { + for timer in self.timers.drain(..) { + timer.cancellation_token.cancel(); + } + } +} +impl Handler for Worker { + async fn handle(&mut self, msg: RegisterManaged, ctx: &Context) -> Result<(), String> { + ctx.register_name(&msg.0).map_err(|e| e.to_string()) + } +} + +/// Observes other actors through monitors and links. +pub struct Watcher { + _live: Live, + downs: u64, + exits: u64, +} +impl Watcher { + pub fn new() -> Self { + Self { + _live: Live::new(), + downs: 0, + exits: 0, + } + } +} +impl Actor for Watcher { + async fn started(&mut self, _: &Context) { + STARTED.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + async fn exit_received(&mut self, _: Exit, _: &Context) { + self.exits += 1; + } +} +message!(Watch(ChildHandle) => MonitorRef); +message!(Unwatch(MonitorRef) => ()); +message!(LinkTo(ChildHandle) => ()); +message!(Unlink(ChildHandle) => ()); +message!(Trap(bool) => ()); +message!(Counts => (u64, u64)); +message!(SpawnLinked => ActorRef); +impl Handler for Watcher { + async fn handle(&mut self, msg: Watch, ctx: &Context) -> MonitorRef { + ctx.monitor(&msg.0) + } +} +impl Handler for Watcher { + async fn handle(&mut self, msg: Unwatch, ctx: &Context) { + ctx.demonitor(msg.0); + } +} +impl Handler for Watcher { + async fn handle(&mut self, msg: LinkTo, ctx: &Context) { + ctx.link(&msg.0); + } +} +impl Handler for Watcher { + async fn handle(&mut self, msg: Unlink, ctx: &Context) { + ctx.unlink(&msg.0); + } +} +impl Handler for Watcher { + async fn handle(&mut self, msg: Trap, ctx: &Context) { + ctx.trap_exit(msg.0); + } +} +impl Handler for Watcher { + async fn handle(&mut self, _: Counts, _: &Context) -> (u64, u64) { + (self.downs, self.exits) + } +} +impl Handler for Watcher { + async fn handle(&mut self, _: SpawnLinked, ctx: &Context) -> ActorRef { + Worker::new().start_linked(ctx) + } +} +impl Handler for Watcher { + async fn handle(&mut self, _: Down, _: &Context) { + self.downs += 1; + } +} + +// --------------------------------------------------------------------------- +// Cleanup of a failed episode +// --------------------------------------------------------------------------- + +/// What one episode started. An actor lives on when its last `ActorRef` drops +/// (as in Erlang), so when a scenario fails or is timed out half-way the lane +/// stops whatever it left behind; otherwise the harness itself would leak. +#[derive(Default)] +pub struct Scope { + actors: Vec, + supervisors: Vec, +} +impl Scope { + pub fn clean_up(&mut self) { + for actor in self.actors.drain(..) { + actor.stop(); + } + for supervisor in self.supervisors.drain(..) { + supervisor.stop(); + } + } +} +tokio::task_local! { + pub static SCOPE: Arc>; +} +fn track(actor: ActorRef) -> ActorRef { + let _ = SCOPE.try_with(|s| { + s.lock() + .unwrap_or_else(|p| p.into_inner()) + .actors + .push(actor.child_handle()) + }); + actor +} +fn track_supervisor(supervisor: &Supervisor) { + let _ = SCOPE.try_with(|s| { + s.lock() + .unwrap_or_else(|p| p.into_inner()) + .supervisors + .push(supervisor.clone()) + }); +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +fn backend(rng: &mut Rng) -> Backend { + [Backend::Async, Backend::Blocking, Backend::Thread][rng.below(3) as usize] +} + +async fn exit(actor: &ActorRef) -> Result { + rt::timeout(LIMIT, actor.wait_exit()) + .await + .map_err(|_| "actor did not exit within the limit".to_string()) +} + +/// Crash, stop or shut down the actor, then check the exit reason it reports. +async fn finish(actor: &ActorRef, rng: &mut Rng) -> R { + match rng.below(4) { + 0 => { + ensure!( + actor.request(Crash).await.is_err(), + "crash request returned Ok" + ); + let reason = exit(actor).await?; + ensure!( + matches!(reason, ExitReason::Panic(_)), + "crash exit {reason:?}" + ); + } + 1 => { + actor.child_handle().stop_with_reason(ExitReason::Shutdown); + let reason = exit(actor).await?; + ensure!(reason == ExitReason::Shutdown, "shutdown exit {reason:?}"); + } + _ => { + actor.child_handle().stop(); + let reason = exit(actor).await?; + ensure!(reason == ExitReason::Normal, "stop exit {reason:?}"); + } + } + Ok(()) +} + +async fn stop_watcher(watcher: &ActorRef) -> R { + watcher.child_handle().stop(); + let reason = exit(watcher).await?; + ensure!(reason == ExitReason::Normal, "watcher exit {reason:?}"); + Ok(()) +} + +/// Poll `check` until it returns true or the limit passes. +async fn eventually(what: &str, mut check: F) -> R +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let deadline = std::time::Instant::now() + LIMIT; + loop { + if check().await? { + return Ok(()); + } + ensure!(std::time::Instant::now() < deadline, "timed out: {what}"); + rt::sleep(Duration::from_millis(1)).await; + } +} + +// --------------------------------------------------------------------------- +// Scenarios +// --------------------------------------------------------------------------- + +pub async fn run(index: usize, rng: &mut Rng, env: &Env) -> R { + match index { + 0 => churn(rng, Backend::Async).await, + 1 => churn(rng, Backend::Blocking).await, + 2 => churn(rng, Backend::Thread).await, + 3 => group(rng, env).await, + 4 => supervisor(rng, env, SupervisorStrategy::OneForOne, false).await, + 5 => supervisor(rng, env, SupervisorStrategy::OneForAll, false).await, + 6 => supervisor(rng, env, SupervisorStrategy::RestForOne, false).await, + 7 => { + let strategy = [ + SupervisorStrategy::OneForOne, + SupervisorStrategy::OneForAll, + SupervisorStrategy::RestForOne, + ][rng.below(3) as usize]; + supervisor(rng, env, strategy, true).await + } + 8 => links(rng).await, + 9 => monitors(rng).await, + 10 => timers(rng).await, + 11 => registry_names(rng, env).await, + 12 => requests(rng).await, + 13 => streams(rng).await, + 14 => system_cycle(rng).await, + 15 => burst(rng).await, + _ => unreachable!(), + } +} + +/// Start a few actors on one backend, exercise requests, end each one. +async fn churn(rng: &mut Rng, backend: Backend) -> R { + let actors: Vec<_> = (0..1 + rng.below(6)) + .map(|_| track(Worker::new().start_with_backend(backend))) + .collect(); + for actor in &actors { + for expected in 1..=1 + rng.below(3) { + let got = actor.request(Inc).await.map_err(e("inc"))?; + ensure!(got == expected, "inc returned {got}, expected {expected}"); + } + } + for actor in &actors { + finish(actor, rng).await?; + } + Ok(()) +} + +/// Managed actors on the long-lived groups, some with owned registrations +/// that must disappear with the actor. +async fn group(rng: &mut Rng, env: &Env) -> R { + let group = if rng.one_in(3) { + &env.thread_group + } else { + &env.async_group + }; + let mut actors = Vec::new(); + for _ in 0..1 + rng.below(3) { + match group.start(Worker::new()) { + Ok(actor) => actors.push(track(actor)), + Err(SystemError::Capacity) => { + env.stats + .capacity + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + Err(err) => return Err(format!("group start: {err:?}")), + } + } + let mut names = Vec::new(); + for actor in &actors { + ensure!(actor.request(Inc).await.map_err(e("inc"))? == 1, "inc"); + if rng.one_in(2) { + let name = format!("soak-managed-{}", env.stats.next_id()); + actor + .request(RegisterManaged(name.clone())) + .await + .map_err(e("register"))??; + let found = registry::whereis::>(&name).map(|a| a.id()); + ensure!( + found == Some(actor.id()), + "managed name {name} -> {found:?}" + ); + names.push(name); + } + } + for actor in &actors { + finish(actor, rng).await?; + } + for name in names { + eventually("managed registration removed", || { + let gone = registry::whereis::>(&name).is_none(); + async move { Ok(gone) } + }) + .await?; + } + Ok(()) +} + +fn config(strategy: SupervisorStrategy) -> SupervisorConfig { + SupervisorConfig { + strategy, + max_restarts: 60_000, + backoff: Backoff { + initial: Duration::from_millis(1), + maximum: Duration::from_millis(2), + jitter: false, + ..Backoff::default() + }, + ..SupervisorConfig::default() + } +} + +fn child(name: String, group: &ExecutionGroup) -> ChildSpec { + ChildSpec::new(name, group, |_| async { + leak_on_restart(); + Ok::<_, String>(Worker::new()) + }) +} + +/// Three children (plus an optional nested subtree); crash some and wait for +/// their replacement generation; shut the tree down. +async fn supervisor(rng: &mut Rng, env: &Env, strategy: SupervisorStrategy, nested: bool) -> R { + if leaks().task { + rt::spawn(std::future::pending::<()>()); + } + let mut tree = env.system.supervisor(config(strategy)).map_err(e("tree"))?; + let mut children: Vec> = Vec::new(); + for i in 0..3 { + if nested && i == 1 { + let placement = env.async_group.clone(); + let published = Arc::new(std::sync::Mutex::new(None)); + let slot = published.clone(); + tree.add_supervisor(SupervisorSpec::new( + "subtree", + &env.async_group, + config(strategy), + move |subtree| { + let leaf = subtree.add(child("leaf".into(), &placement))?; + *slot.lock().unwrap_or_else(|p| p.into_inner()) = Some(leaf); + Ok::<_, SupervisorError>(()) + }, + )) + .map_err(e("add subtree"))?; + } + children.push( + tree.add(child(format!("child-{i}"), &env.async_group)) + .map_err(e("add child"))?, + ); + } + let supervisor = rt::timeout(LIMIT, tree.start()) + .await + .map_err(|_| "supervisor start timed out".to_string())? + .map_err(e("supervisor start"))?; + track_supervisor(&supervisor); + for _ in 0..1 + rng.below(3) { + let target = &children[rng.below(children.len() as u64) as usize]; + let before = target.snapshot().generation; + let actor = rt::timeout(LIMIT, target.wait_for_generation(before.saturating_sub(1))) + .await + .map_err(|_| "child never ready".to_string())? + .map_err(e("child ready"))?; + let generation = target.snapshot().generation; + let _ = actor.request(Crash).await; + let next = rt::timeout(LIMIT, target.wait_for_generation(generation)) + .await + .map_err(|_| format!("{strategy:?}: no restart after generation {generation}"))? + .map_err(e("restart"))?; + ensure!(next.id() != actor.id(), "restart reused the actor id"); + // The replacement may itself be replaced by a collateral restart. + let _ = next.request(Inc).await; + } + let report = supervisor.shutdown(LIMIT).await; + ensure!( + report.complete, + "supervisor shutdown incomplete: {report:?}" + ); + for child in &children { + ensure!( + child.current().is_err(), + "child still current after shutdown" + ); + } + Ok(()) +} + +/// Bidirectional links: trapped exits, propagated exits, `start_linked`, unlink. +async fn links(rng: &mut Rng) -> R { + let watcher = track(Watcher::new().start()); + let peer = track(Worker::new().start_with_backend(backend(rng))); + match rng.below(4) { + 0 => { + watcher.request(Trap(true)).await.map_err(e("trap"))?; + watcher + .request(LinkTo(peer.child_handle())) + .await + .map_err(e("link"))?; + let _ = peer.request(Crash).await; + exit(&peer).await?; + eventually("trapped exit delivered", || async { + Ok(watcher.request(Counts).await.map_err(e("counts"))?.1 == 1) + }) + .await?; + stop_watcher(&watcher).await + } + 1 => { + watcher + .request(LinkTo(peer.child_handle())) + .await + .map_err(e("link"))?; + let _ = peer.request(Crash).await; + exit(&peer).await?; + let reason = exit(&watcher).await?; + ensure!(reason.is_abnormal(), "linked peer exit {reason:?}"); + Ok(()) + } + 2 => { + peer.child_handle().stop(); + exit(&peer).await?; + watcher.request(Trap(true)).await.map_err(e("trap"))?; + let child = track(watcher.request(SpawnLinked).await.map_err(e("spawn"))?); + ensure!(child.request(Inc).await.map_err(e("inc"))? == 1, "inc"); + let _ = child.request(Crash).await; + exit(&child).await?; + eventually("start_linked exit delivered", || async { + Ok(watcher.request(Counts).await.map_err(e("counts"))?.1 == 1) + }) + .await?; + stop_watcher(&watcher).await + } + _ => { + let handle = peer.child_handle(); + watcher + .request(LinkTo(handle.clone())) + .await + .map_err(e("link"))?; + watcher.request(Unlink(handle)).await.map_err(e("unlink"))?; + let _ = peer.request(Crash).await; + exit(&peer).await?; + let (_, exits) = watcher.request(Counts).await.map_err(e("counts"))?; + ensure!(exits == 0, "exit delivered after unlink"); + stop_watcher(&watcher).await + } + } +} + +/// Monitors, demonitors, monitoring an already-dead actor, and a watcher +/// that stops before its targets. +async fn monitors(rng: &mut Rng) -> R { + let watcher = track(Watcher::new().start()); + let targets: Vec<_> = (0..1 + rng.below(4)) + .map(|_| track(Worker::new().start_with_backend(backend(rng)))) + .collect(); + let mut expected = 0; + for target in &targets { + let monitor = watcher + .request(Watch(target.child_handle())) + .await + .map_err(e("watch"))?; + if rng.one_in(4) { + watcher + .request(Unwatch(monitor)) + .await + .map_err(e("unwatch"))?; + } else { + expected += 1; + } + } + if rng.one_in(5) { + // The watcher goes first; its monitors must not outlive it. + stop_watcher(&watcher).await?; + for target in &targets { + finish(target, rng).await?; + } + return Ok(()); + } + for target in &targets { + finish(target, rng).await?; + } + if rng.one_in(2) { + watcher + .request(Watch(targets[0].child_handle())) + .await + .map_err(e("watch dead"))?; + expected += 1; + } + eventually("every Down delivered", || async { + Ok(watcher.request(Counts).await.map_err(e("counts"))?.0 == expected) + }) + .await?; + stop_watcher(&watcher).await +} + +/// `send_after` and `send_interval`, cancelled, fired, or live at stop. +async fn timers(rng: &mut Rng) -> R { + let actor = track(Worker::new().start_with_backend(backend(rng))); + let once = 1 + rng.below(4); + let period = Duration::from_millis(1 + rng.below(4)); + actor + .request(ArmTimers(once, rng.below(3), period)) + .await + .map_err(e("arm"))?; + rt::sleep(rng.ms(8)).await; + match rng.below(3) { + 0 => actor.request(CancelTimers).await.map_err(e("cancel"))?, + 1 => {} + _ => { + eventually("one-shot timers fired", || async { + Ok(actor.request(Get).await.map_err(e("get"))? >= once) + }) + .await?; + actor.request(CancelTimers).await.map_err(e("cancel"))?; + } + } + finish(&actor, rng).await +} + +/// Unmanaged registry: register, look up, reject duplicates, unregister. +async fn registry_names(rng: &mut Rng, env: &Env) -> R { + let actor = track(Worker::new().start()); + let leak = leaks().registry; + for _ in 0..1 + rng.below(4) { + let id = env.stats.next_id(); + let name = format!("soak-name-{id}"); + registry::register(&name, actor.clone()).map_err(e("register"))?; + let found = registry::whereis::>(&name).map(|a| a.id()); + ensure!(found == Some(actor.id()), "whereis {name} -> {found:?}"); + ensure!( + registry::register(&name, actor.clone()).is_err(), + "duplicate registration accepted" + ); + if leak && id.is_multiple_of(4) { + continue; + } + registry::unregister(&name); + ensure!( + registry::whereis::>(&name).is_none(), + "{name} still registered" + ); + } + finish(&actor, rng).await +} + +/// Requests that time out, requests dropped mid-flight, requests to a stopped actor. +async fn requests(rng: &mut Rng) -> R { + let actor = track(Worker::new().start_with_backend(backend(rng))); + let timed_out = actor + .request_with_timeout( + Slow(Duration::from_millis(20 + rng.below(20))), + Duration::from_millis(1), + ) + .await; + ensure!( + // A requester descheduled past the handler can see the reply first. + matches!(timed_out, Err(ActorError::RequestTimeout) | Ok(())), + "slow request: {timed_out:?}" + ); + if rng.one_in(2) { + // Dropping the request future cancels it. + let _ = rt::timeout( + Duration::from_millis(1), + actor.request(Slow(Duration::from_millis(10))), + ) + .await; + } + let got = actor + .request_with_timeout(Inc, LIMIT) + .await + .map_err(e("inc"))?; + ensure!(got == 1, "inc after timeouts returned {got}"); + finish(&actor, rng).await?; + ensure!( + matches!(actor.request(Inc).await, Err(ActorError::ActorStopped)), + "request to a stopped actor" + ); + ensure!(actor.send(Bump).is_err(), "send to a stopped actor"); + Ok(()) +} + +/// Stream listeners: a finite stream drained into the actor, and an endless +/// one that must end when the actor stops. +async fn streams(rng: &mut Rng) -> R { + let actor = track(Worker::new().start_with_backend(backend(rng))); + let n = 1 + rng.below(64); + drop(spawn_listener( + actor.context(), + futures::stream::iter((0..n).map(|_| Bump)), + )); + eventually("finite stream drained", || async { + Ok(actor.request(Get).await.map_err(e("get"))? == n) + }) + .await?; + if rng.one_in(2) { + let endless = futures::stream::repeat(()).then(|_| async { + rt::sleep(Duration::from_millis(1)).await; + Bump + }); + drop(spawn_listener(actor.context(), endless)); + rt::sleep(rng.ms(5)).await; + } + finish(&actor, rng).await +} + +/// A whole `ActorSystem` lifetime with actors still running at shutdown. +async fn system_cycle(rng: &mut Rng) -> R { + let system = ActorSystem::new().map_err(e("system"))?; + let group = system.async_group("cycle", 1).map_err(e("group"))?; + let native = if rng.one_in(2) { + Some( + system + .thread_group("cycle-native", 2) + .map_err(e("native"))?, + ) + } else { + None + }; + let mut actors = Vec::new(); + for _ in 0..1 + rng.below(4) { + actors.push(group.start(Worker::new()).map_err(e("start"))?); + } + if let Some(native) = &native { + actors.push(native.start(Worker::new()).map_err(e("native start"))?); + } + for actor in &actors { + ensure!(actor.request(Inc).await.map_err(e("inc"))? == 1, "inc"); + } + if rng.one_in(2) { + let mut tree = system + .supervisor(config(SupervisorStrategy::OneForOne)) + .map_err(e("tree"))?; + tree.add(child("cycle-child".into(), &group)) + .map_err(e("child"))?; + rt::timeout(LIMIT, tree.start()) + .await + .map_err(|_| "cycle supervisor start timed out".to_string())? + .map_err(e("cycle supervisor"))?; + } + let report = system.shutdown(LIMIT).await; + ensure!( + report.complete, + "system shutdown incomplete: groups {:?}, actors {:?}", + report.unfinished_groups, + report.unfinished_actors + ); + for actor in &actors { + ensure!(actor.exit_reason().is_some(), "actor alive after shutdown"); + } + Ok(()) +} + +/// Fill a mailbox with thousands of messages, then drain it or stop with it full. +async fn burst(rng: &mut Rng) -> R { + let actor = track(Worker::new().start_with_backend(backend(rng))); + let n = 1_000 + rng.below(20_000); + for _ in 0..n { + actor.send(Bump).map_err(e("burst send"))?; + } + if !rng.one_in(3) { + let got = actor + .request_with_timeout(Get, LIMIT) + .await + .map_err(e("drain"))?; + ensure!(got == n, "drained {got} of {n}"); + ensure!(actor.mailbox_depth() == 0, "mailbox not empty after drain"); + } + finish(&actor, rng).await +} diff --git a/docs/TESTING.md b/docs/TESTING.md index 2bc18e18..37860412 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -117,7 +117,7 @@ Status of each layer. "Planned" items are tracked in the | Spec and code in sync | `scripts/check-model-sync.py`, `verification/model-map.toml` | A PR changing modelled code without its model; a trace event no trace spec replays; a model no real run is checked against; trace coverage going down | `model-sync` job on every PR. See [model sync](#model-sync) | | Primitives | Loom | Atomic and memory-ordering bugs in cancellation, ready gate and wakeups | In CI; expand to reply slots and mailbox publication versus shutdown | | Whole runtime, deterministic | Simulation backend for `spawned-rt::tasks` (random, PCT, bounded DFS) | Async races, replayed from a seed with virtual time | In CI (`sim` job, [simulated suites in CI](#simulated-suites-in-ci)) | -| Resource stability | Soak tests | Memory, file descriptors, threads, tasks and registry entries growing over hours of actor churn and restarts | Planned | +| Resource stability | Soak harness (`concurrency/benches/soak`) | Memory, file descriptors, threads, tasks and registry entries growing over hours of actor churn and restarts | Run by hand before each release, never in CI ([soak harness](#soak-harness)) | | Behaviour drift | Behaviour snapshots | Silent changes to lifecycle traces, shutdown reports and exit reasons in about 30 standard scenarios | Planned | | Model-derived tests | Tests generated from the TLA+ models | Interesting model states (a crash during backoff while shutdown starts, a restart racing a deadline) that no hand-written test reaches | Planned, after trace coverage | | Time | Clock-jump tests | Timers and backoff misbehaving on paused time or large forward and backward clock jumps | Planned | @@ -1576,6 +1576,118 @@ change to a branch made from `spawned-v0.7`, dispatch the workflow with that ref, and land the spawned PR once both are green; then move the client branch. The `spawned-v0.7` client branches back long-lived draft PRs and are never merged from here. +## Soak harness + +Seconds-long tests cannot see a leak of a few bytes per restart; production +can (ethrex's mailbox leak grew 1.5 GB a day). `concurrency/benches/soak` runs +hours of seeded actor churn and fails when a resource grows. It is a +`harness = false` bench, so `cargo test` never builds or runs it; the +`--all-targets` clippy job keeps it compiling. It needs Linux (`/proc`). + +```sh +# on fede, from the repository root; release build with line tables +CARGO_INCREMENTAL=0 CARGO_PROFILE_BENCH_DEBUG=line-tables-only \ + nice -n 19 cargo bench -p spawned-concurrency --features metrics --bench soak --no-run +nice -n 19 taskset -c 26-29 target/release/deps/soak- \ + --duration 4h --seed 1 --out ~/spawned-work/soak/out/release-4h +``` + +`cargo bench ... --bench soak -- --duration 4h --seed 1` also works. Options: +`--duration` (`4h`, `10m`, `1h30m`), `--seed`, `--sample` (default: duration +/ 360, between 1 s and 10 s), `--warmup`, `--lanes` (async lanes, default 4), +`--workers` (Tokio workers, default 4), `--out`, `--config` (default +`concurrency/benches/soak/thresholds.toml`) and `--inject-leak`. Exit code 0 is +PASS, 1 FAIL, 2 a usage error. + +**Workload.** Four async lanes and one threads-mode lane each pick scenarios +from a seeded RNG until the duration ends; every scenario checks its own +outcome (exit reasons, replies, Down and Exit counts, registry contents) and +a scenario stuck for 60 s is an error. The scenarios: start, request and end +(crash, stop or `Shutdown`) actors on each backend (Async, Blocking, Thread) +and in threads mode; managed actors on a long-lived async group and thread +group with owned registrations; supervisors with each strategy, and nested, +crashing children and waiting for the replacement generation; links (trapped, +propagated, `start_linked`, unlink); monitors (demonitor, monitoring a dead +actor, the watcher dying first); `send_after` and `send_interval` cancelled, +fired or live at stop; unmanaged registry register, duplicate and unregister; +requests that time out, are dropped mid-flight or hit a stopped actor; +finite and endless stream listeners; a whole `ActorSystem` with running +actors and a supervisor shut down; and bursts of 1,000 to 21,000 messages +drained or discarded at stop. The seed fixes each lane's scenario sequence and +parameters; thread interleavings are not reproducible. A 10-minute run on fede +starts about 1.7 million actors, so a 4-hour run executes every path millions of +times. + +**Measurements**, one CSV row (`samples.csv`) per sample interval, plus a +`phase` column (`baseline`, `warmup`, `steady`, `quiesce`): + +| Column | Source | +|---|---| +| `rss_mb` | `VmRSS` in `/proc/self/status` | +| `heap_mb` | live heap bytes from a counting global allocator installed in the harness binary only | +| `threads`, `fds` | entries in `/proc/self/task` and `/proc/self/fd` | +| `tokio_tasks` | `RuntimeMetrics::num_alive_tasks()` of the harness runtime | +| `registry` | `registry::registered().len()` | +| `actors_live` | harness actor states constructed and not yet dropped (started minus fully stopped) | +| `actors_active`, `mailbox_depth`, `handlers_active`, `requests_active` | spawned's own `metrics` gauges | +| `actors_started`, `scenarios`, `errors` | progress counters (not judged) | + +Framework-internal `ActorRef` and `Context` clones are not countable through +public APIs; a leaked one keeps its actor's shared state and mailbox alive, so +it shows in `heap_mb` (and LeakSanitizer covers them at process exit, see +[leak tests](#leak-tests-and-leaksanitizer)). + +**Pass/fail** (`thresholds.toml`, every value documented there). The run +first primes every scenario once and lets the runtime settle, then takes the +baseline. After warm-up (a quarter of the run, at most 30 minutes) each series +gets a least-squares slope per hour: RSS over every sample, the other series +over their floor (the minimum of each of about 20 consecutive windows), since +counts swing with the workload and a leak raises the floor. A series fails +when the slope is above its limit *and* at least 3 standard errors above zero, +so short noisy runs do not fail but a steady leak does, and when the growth +it implies over the steady window exceeds a small floor (`growth.*`: 8 +threads, 5 tasks or actors, 4 MiB RSS, 1 MiB heap, 0 registry entries), so a +bounded pool adding a few threads in a short run does not fail either. Limits: RSS 16 MiB/h, +heap 4 MiB/h, threads, fds and registry 2/h, Tokio tasks, live actors and the +active-handler and request gauges 10/h, mailbox depth 100/h. At the end the lanes +stop, the long-lived system shuts down, and within 60 s every count must +return exactly to its baseline and the heap to within 8 MiB of it (RSS is not +checked here: the allocator keeps freed pages). Any scenario error, a stuck +lane or an incomplete system shutdown also fails the run. + +**Reading the results.** `summary.txt` (also printed) has the verdict, the +first scenario errors, a table with each series' baseline, steady maximum, +final quiesce value, slope ± standard error, limit and the two checks, and +the run count of every scenario. A `trend` FAIL with a clean `settle` usually +means growth that is released on shutdown (an unbounded cache or queue); a +`settle` FAIL means something outlived everything that owned it. Plot the +CSV columns against `elapsed_s` to see when growth starts. A real leak gets a +minimal repro and a failing test, never a raised threshold. + +**Negative control.** `--inject-leak registry,arc,task` (or `all`) plants +known leaks: one in four unmanaged registry names is never unregistered, every +supervised (re)start leaks a 4 KiB `Arc` and an actor-state token, and every +supervisor episode leaks a pending Tokio task. A 3-minute run with `all` must +FAIL on `registry`, `heap_mb`, `actors_live` and `tokio_tasks`; run it after +changing the harness or the thresholds. + +```sh +nice -n 19 taskset -c 26-29 target/release/deps/soak- \ + --duration 3m --seed 1 --inject-leak all --out ~/spawned-work/soak/out/self-test +``` + +Runs are by hand only: no schedule. Run a 4-hour soak before each release +([release checklist](#release-checklist)). + +## Release checklist + +Before tagging a release (`gh release create` triggers `release.yaml`, which +publishes to crates.io): + +1. CI green on the release PR, including the `client-compat` gate. +2. A 4-hour [soak](#soak-harness) on fede passes on the release commit; link + its `summary.txt` in the release PR. + ## CI hardening Batch 1 of the [database-grade testing program](../ROADMAP.md#database-grade-testing-program). From 7d7b33fdeb332a73f0b2779ad3eb973588e33702 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 15:28:45 -0300 Subject: [PATCH 169/208] test(soak): port the soak harness (#257) to the 0.8 API Handlers take PendingReply and answer through it; SupervisorConfig, Backoff and metrics::Config use their builders; a request to an actor past its death point is NoProc (0.8, OTP noproc), not ActorStopped. A 2-minute run on fede completes every scenario with no errors. --- concurrency/benches/soak/main.rs | 4 +- concurrency/benches/soak/native.rs | 119 +++++++++++++++---- concurrency/benches/soak/workload.rs | 167 +++++++++++++++++++++------ 3 files changed, 231 insertions(+), 59 deletions(-) diff --git a/concurrency/benches/soak/main.rs b/concurrency/benches/soak/main.rs index ee088e5e..0eb5cb1b 100644 --- a/concurrency/benches/soak/main.rs +++ b/concurrency/benches/soak/main.rs @@ -361,9 +361,7 @@ fn main() { }; LEAKS.set(args.leaks).unwrap(); // Counters and gauges only: no sampled clocks. - let _ = metrics::configure(metrics::Config { - timing_sample_every: 0, - }); + let _ = metrics::configure(metrics::Config::new().timing_sample_every(0)); // Injected crashes are expected millions of times; keep other panics loud. let default_hook = std::panic::take_hook(); std::panic::set_hook(Box::new(move |info| { diff --git a/concurrency/benches/soak/native.rs b/concurrency/benches/soak/native.rs index f102b86c..7975b460 100644 --- a/concurrency/benches/soak/native.rs +++ b/concurrency/benches/soak/native.rs @@ -50,33 +50,69 @@ impl Message for Bump { type Result = (); } impl Handler for NWorker { - fn handle(&mut self, _: Inc, _: &Context) -> u64 { - self.value += 1; - self.value + fn handle( + &mut self, + _: Inc, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let value = { + self.value += 1; + self.value + }; + reply.reply(value); } } impl Handler for NWorker { - fn handle(&mut self, _: Get, _: &Context) -> u64 { - self.value + fn handle( + &mut self, + _: Get, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let value = { self.value }; + reply.reply(value); } } impl Handler for NWorker { - fn handle(&mut self, _: Bump, _: &Context) { + fn handle( + &mut self, + _: Bump, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.value += 1; + reply.reply(()); } } impl Handler for NWorker { - fn handle(&mut self, _: Crash, _: &Context) { + fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { panic!("soak: injected crash"); } } impl Handler for NWorker { - fn handle(&mut self, msg: Slow, _: &Context) { + fn handle( + &mut self, + msg: Slow, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { std::thread::sleep(msg.0); + reply.reply(()); } } impl Handler for NWorker { - fn handle(&mut self, msg: ArmTimers, ctx: &Context) { + fn handle( + &mut self, + msg: ArmTimers, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { let ArmTimers(once, every, period) = msg; for i in 0..once { let delay = period * (i as u32 + 1); @@ -85,13 +121,20 @@ impl Handler for NWorker { for _ in 0..every { self.timers.push(send_interval(period, ctx.clone(), Bump)); } + reply.reply(()); } } impl Handler for NWorker { - fn handle(&mut self, _: CancelTimers, _: &Context) { + fn handle( + &mut self, + _: CancelTimers, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { for timer in self.timers.drain(..) { timer.cancellation_token.cancel(); } + reply.reply(()); } } @@ -114,33 +157,69 @@ message!(LinkTo(ChildHandle) => ()); message!(Trap(bool) => ()); message!(Counts => (u64, u64)); impl Handler for NWatcher { - fn handle(&mut self, msg: Watch, ctx: &Context) -> MonitorRef { - ctx.monitor(&msg.0) + fn handle( + &mut self, + msg: Watch, + reply: spawned_concurrency::PendingReply<'_, MonitorRef>, + ctx: &Context, + ) { + let value = { ctx.monitor(&msg.0) }; + reply.reply(value); } } impl Handler for NWatcher { - fn handle(&mut self, msg: Unwatch, ctx: &Context) { + fn handle( + &mut self, + msg: Unwatch, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.demonitor(msg.0); + reply.reply(()); } } impl Handler for NWatcher { - fn handle(&mut self, msg: LinkTo, ctx: &Context) { + fn handle( + &mut self, + msg: LinkTo, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.link(&msg.0); + reply.reply(()); } } impl Handler for NWatcher { - fn handle(&mut self, msg: Trap, ctx: &Context) { + fn handle( + &mut self, + msg: Trap, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.trap_exit(msg.0); + reply.reply(()); } } impl Handler for NWatcher { - fn handle(&mut self, _: Counts, _: &Context) -> (u64, u64) { - (self.downs, self.exits) + fn handle( + &mut self, + _: Counts, + reply: spawned_concurrency::PendingReply<'_, (u64, u64)>, + _: &Context, + ) { + let value = { (self.downs, self.exits) }; + reply.reply(value); } } impl Handler for NWatcher { - fn handle(&mut self, _: Down, _: &Context) { + fn handle( + &mut self, + _: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.downs += 1; + reply.reply(()); } } @@ -336,8 +415,8 @@ fn registry_requests(rng: &mut Rng, stats: &Stats) -> R { ensure!(actor.request(Inc).map_err(e("inc"))? == 1, "inc"); finish(&actor, rng)?; ensure!( - matches!(actor.request(Inc), Err(ActorError::ActorStopped)), - "request to a stopped actor" + matches!(actor.request(Inc), Err(ActorError::NoProc)), + "request to a stopped actor (NoProc past its death point)" ); Ok(()) } diff --git a/concurrency/benches/soak/workload.rs b/concurrency/benches/soak/workload.rs index 63be5c5d..20bcd948 100644 --- a/concurrency/benches/soak/workload.rs +++ b/concurrency/benches/soak/workload.rs @@ -87,33 +87,69 @@ impl Message for Bump { } impl Handler for Worker { - async fn handle(&mut self, _: Inc, _: &Context) -> u64 { - self.value += 1; - self.value + async fn handle( + &mut self, + _: Inc, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let value = { + self.value += 1; + self.value + }; + reply.reply(value); } } impl Handler for Worker { - async fn handle(&mut self, _: Get, _: &Context) -> u64 { - self.value + async fn handle( + &mut self, + _: Get, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let value = { self.value }; + reply.reply(value); } } impl Handler for Worker { - async fn handle(&mut self, _: Bump, _: &Context) { + async fn handle( + &mut self, + _: Bump, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.value += 1; + reply.reply(()); } } impl Handler for Worker { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { panic!("soak: injected crash"); } } impl Handler for Worker { - async fn handle(&mut self, msg: Slow, _: &Context) { + async fn handle( + &mut self, + msg: Slow, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { rt::sleep(msg.0).await; + reply.reply(()); } } impl Handler for Worker { - async fn handle(&mut self, msg: ArmTimers, ctx: &Context) { + async fn handle( + &mut self, + msg: ArmTimers, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { let ArmTimers(once, every, period) = msg; for i in 0..once { let delay = period * (i as u32 + 1); @@ -122,18 +158,31 @@ impl Handler for Worker { for _ in 0..every { self.timers.push(send_interval(period, ctx.clone(), Bump)); } + reply.reply(()); } } impl Handler for Worker { - async fn handle(&mut self, _: CancelTimers, _: &Context) { + async fn handle( + &mut self, + _: CancelTimers, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { for timer in self.timers.drain(..) { timer.cancellation_token.cancel(); } + reply.reply(()); } } impl Handler for Worker { - async fn handle(&mut self, msg: RegisterManaged, ctx: &Context) -> Result<(), String> { - ctx.register_name(&msg.0).map_err(|e| e.to_string()) + async fn handle( + &mut self, + msg: RegisterManaged, + reply: spawned_concurrency::PendingReply<'_, Result<(), String>>, + ctx: &Context, + ) { + let value = { ctx.register_name(&msg.0).map_err(|e| e.to_string()) }; + reply.reply(value); } } @@ -168,43 +217,91 @@ message!(Trap(bool) => ()); message!(Counts => (u64, u64)); message!(SpawnLinked => ActorRef); impl Handler for Watcher { - async fn handle(&mut self, msg: Watch, ctx: &Context) -> MonitorRef { - ctx.monitor(&msg.0) + async fn handle( + &mut self, + msg: Watch, + reply: spawned_concurrency::PendingReply<'_, MonitorRef>, + ctx: &Context, + ) { + let value = { ctx.monitor(&msg.0) }; + reply.reply(value); } } impl Handler for Watcher { - async fn handle(&mut self, msg: Unwatch, ctx: &Context) { + async fn handle( + &mut self, + msg: Unwatch, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.demonitor(msg.0); + reply.reply(()); } } impl Handler for Watcher { - async fn handle(&mut self, msg: LinkTo, ctx: &Context) { + async fn handle( + &mut self, + msg: LinkTo, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.link(&msg.0); + reply.reply(()); } } impl Handler for Watcher { - async fn handle(&mut self, msg: Unlink, ctx: &Context) { + async fn handle( + &mut self, + msg: Unlink, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.unlink(&msg.0); + reply.reply(()); } } impl Handler for Watcher { - async fn handle(&mut self, msg: Trap, ctx: &Context) { + async fn handle( + &mut self, + msg: Trap, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.trap_exit(msg.0); + reply.reply(()); } } impl Handler for Watcher { - async fn handle(&mut self, _: Counts, _: &Context) -> (u64, u64) { - (self.downs, self.exits) + async fn handle( + &mut self, + _: Counts, + reply: spawned_concurrency::PendingReply<'_, (u64, u64)>, + _: &Context, + ) { + let value = { (self.downs, self.exits) }; + reply.reply(value); } } impl Handler for Watcher { - async fn handle(&mut self, _: SpawnLinked, ctx: &Context) -> ActorRef { - Worker::new().start_linked(ctx) + async fn handle( + &mut self, + _: SpawnLinked, + reply: spawned_concurrency::PendingReply<'_, ActorRef>, + ctx: &Context, + ) { + let value = { Worker::new().start_linked(ctx) }; + reply.reply(value); } } impl Handler for Watcher { - async fn handle(&mut self, _: Down, _: &Context) { + async fn handle( + &mut self, + _: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.downs += 1; + reply.reply(()); } } @@ -417,17 +514,15 @@ async fn group(rng: &mut Rng, env: &Env) -> R { } fn config(strategy: SupervisorStrategy) -> SupervisorConfig { - SupervisorConfig { - strategy, - max_restarts: 60_000, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(2), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::new() + .strategy(strategy) + .max_restarts(60_000) + .backoff( + Backoff::new() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(2)) + .jitter(false), + ) } fn child(name: String, group: &ExecutionGroup) -> ChildSpec { @@ -689,8 +784,8 @@ async fn requests(rng: &mut Rng) -> R { ensure!(got == 1, "inc after timeouts returned {got}"); finish(&actor, rng).await?; ensure!( - matches!(actor.request(Inc).await, Err(ActorError::ActorStopped)), - "request to a stopped actor" + matches!(actor.request(Inc).await, Err(ActorError::NoProc)), + "request to a stopped actor (NoProc past its death point)" ); ensure!(actor.send(Bump).is_err(), "send to a stopped actor"); Ok(()) From dbd6dae0b3d6b6486c7da4a6b9def9259b08d276 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 15:28:45 -0300 Subject: [PATCH 170/208] verification: the temporal ForgetSignal control also drops GraceKillsRemaining --- verification/tla/negative_checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 69fed1f8..98387a36 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -587,7 +587,7 @@ def decide_at_once(events): model_fault("MCCoordinator", "ForgetFailure", args.tlc) model_fault("MCCoordinator", "ForgetSignal", args.tlc, invariant="JoinAfterAllSignalled") model_fault("MCCoordinator", "ForgetSignal", args.tlc, temporal=True, - drop=("JoinAfterAllSignalled",)) + drop=("JoinAfterAllSignalled", "GraceKillsRemaining")) model_fault("MCCoordinator", "RollbackReady", args.tlc, invariant="OrderedShutdown") # v0.8 grace expiry: no kill (a stuck trapping root holds the join), or a # kill in the ordered loop (stopped() skipped before the grace expired). From 0006aa38484ad6a8e27fd6d1b1dec03c2cf26093 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 15:31:28 -0300 Subject: [PATCH 171/208] fix(verification): the ForgetSignal liveness control also drops GraceKillsRemaining A root left unsignalled is also left unkilled at grace expiry, so the new GraceKillsRemaining invariant fired before the liveness check the control targets (CI's negative controls failed on MCCoordinator/ForgetSignal). --- verification/tla/negative_checks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 98387a36..d5ae84bd 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -586,6 +586,8 @@ def decide_at_once(events): model_fault("MCCoordinator", "ReleaseEarly", args.tlc) model_fault("MCCoordinator", "ForgetFailure", args.tlc) model_fault("MCCoordinator", "ForgetSignal", args.tlc, invariant="JoinAfterAllSignalled") + # A root left unsignalled is also left unkilled at grace expiry: drop + # GraceKillsRemaining too, so the run reaches the liveness check. model_fault("MCCoordinator", "ForgetSignal", args.tlc, temporal=True, drop=("JoinAfterAllSignalled", "GraceKillsRemaining")) model_fault("MCCoordinator", "RollbackReady", args.tlc, invariant="OrderedShutdown") From 0058b11721bd286dc2276b38ac839462f53bfd9f Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 15:31:28 -0300 Subject: [PATCH 172/208] verification: action coverage and floors for Lifecycle, DeferredJobs and DeferredReplies trace_coverage.py skipped the lifecycle-v1, jobs-v1 and replies-v1 schemas. They are now covered by action keys only: the model side is the set of log-event arms of TraceLifecycle.tla, TraceJobs.tla and TraceReplies.tla (each arm one model action or a named group), and a trace covers the arms its accepted TLC replay takes. No state space is explored for them, so the model-side cache and its digest are unchanged. coverage-baseline.json gains the covered keys (Lifecycle 17/17, DeferredJobs 10/10, DeferredReplies 5/5; the Supervision, Cohorts and Nested entries are left as they were), and model-map.toml sets the matching coverage_floor for each. README: the action-only models, the new job and lifecycle traces and controls. --- verification/model-map.toml | 3 + verification/tla/README.md | 26 +++++++-- verification/tla/coverage-baseline.json | 38 +++++++++++++ verification/tla/trace_coverage.py | 75 +++++++++++++++++++++---- 4 files changed, 128 insertions(+), 14 deletions(-) diff --git a/verification/model-map.toml b/verification/model-map.toml index 1e8c8401..57d25e34 100644 --- a/verification/model-map.toml +++ b/verification/model-map.toml @@ -113,6 +113,7 @@ code = [ "concurrency/src/mailbox_admission.rs", "concurrency/src/lifecycle_trace.rs", ] +coverage_floor = 17 # of 17 trace-spec arms (trace_coverage.py STATIC) [models.DeferredReplies] # Rank 11: reply slots, deadlines, deferral credits, owner exit and death. @@ -125,6 +126,7 @@ code = [ "concurrency/src/reply/**", "concurrency/src/lifecycle_trace.rs", ] +coverage_floor = 5 # of 5 trace-spec arms (trace_coverage.py STATIC) [models.DeferredJobs] # Rank 11: pipe jobs, keyed replacement, ActorId and cancellation fencing, @@ -140,6 +142,7 @@ code = [ "concurrency/src/threads/jobs.rs", "concurrency/src/lifecycle_trace.rs", ] +coverage_floor = 10 # of 10 trace-spec arms (trace_coverage.py STATIC) [models.DynamicSupervisor] # Rank 12a design model (PR #245); no code yet. diff --git a/verification/tla/README.md b/verification/tla/README.md index ac4dcbf1..c90752ba 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -138,7 +138,18 @@ in the same PR; the `model-sync` CI job enforces it Trace validation says every recorded run is a behavior of the model. Trace coverage asks the converse: which parts of the model's reachable behavior do the recorded runs never exercise? `trace_coverage.py` reports it per model with -trace validation (`Supervision`, `Cohorts`, `NestedSupervision`). +trace validation (`Supervision`, `Cohorts`, `NestedSupervision`, and by +action keys only `Lifecycle`, `DeferredJobs`, `DeferredReplies`). + +**Action-only models.** For `Lifecycle.tla`, `DeferredJobs.tla` and +`DeferredReplies.tla` the keys are the log-event arms of their trace spec +(`TraceLifecycle.tla`, `TraceJobs.tla`, `TraceReplies.tla`; each arm is one +model action or a named group of them, `A|Send`, `A|Fire`, ...): the model +side is the set of arms, a trace covers the arms its accepted replay takes, +and no state space is explored for them. `coverage-baseline.json` and the +`coverage_floor` of `verification/model-map.toml` hold them like the others. +Model actions without an arm are listed in the map's `[trace.unlogged]` with +the reason. ```bash cargo test -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace --test lifecycle_trace @@ -530,7 +541,7 @@ after a kill or after an untrapped signal, `Kill` instead of `Killed`, publication before D, a `Normal` signal stopping an actor, a late `Down` or link notice with the real reason, a notice after unlink, a `Down` after demonitor, a parent exit dispatched as an `Exit`, and a runtime drop ending -`Normal`: all eleven are rejected. `trace_coverage.py` skips this schema. +`Normal`: all eleven are rejected. Actor-to-actor messages (`Send`, recorded with their sender) add three controls: a send after the sender's death point, the death notice overtaking the earlier message, a message dispatched as an outside request. `trace_coverage.py` covers this schema by action keys (see [Trace coverage](#trace-coverage)). Source: [verification strategy](https://github.com/lambdaclass/spawned/blob/efba24340a01df82f872ec422b9dbaf48f897b04/legacy/docs/research/verification-strategy.md#acceptance). `Coordinator.tla` checks coordinator failure ownership independently of the actor @@ -673,7 +684,10 @@ under the job lock, the enqueue, the fence read), so the log is a linearization. `concurrency/tests/job_trace.rs` records seven scenarios (plain result, a finished keyed job replaced and fenced, emits with one credit, handle cancel, job deadline with the work still running, owner exit with a -running job, a blocking job emitting); `trace_check.py` checks them (schema +running job, a blocking job emitting, and delayed emits: a held result +released by its last fire in both modes, an emit already due, the job +deadline taking a held result, a cancel dropping a scheduled emit); +`trace_check.py` checks them (schema `jobs-v1`) against `TraceJobs.tla` with one owner instance and one key. The permit bound is not checked (`JobBound = J`): the code returns a job's credit just before `Finish`, and rejected admissions are not model actions. @@ -683,7 +697,11 @@ cargo test -p spawned-concurrency --features lifecycle-trace --test job_trace python3 verification/tla/trace_check.py target/lifecycle-traces ``` -`negative_checks.py` requires six edited job traces to be rejected: the +`negative_checks.py` requires twelve edited job traces to be rejected (six +for delayed emits: a release before the last fire, a terminal while an emit +is scheduled, a release by a fire that did not take the result, a delayed +emit after the deadline took the result, a fire without a schedule, a drop +while the job still accepts), and the first six: the replaced job's terminal message delivered, an emit after the terminal message, a second emit with its credit in flight, a second terminal message after the deadline's, a quiet finish without a cancel, and a dispatch after diff --git a/verification/tla/coverage-baseline.json b/verification/tla/coverage-baseline.json index 721b81ba..76750be6 100644 --- a/verification/tla/coverage-baseline.json +++ b/verification/tla/coverage-baseline.json @@ -91,5 +91,43 @@ "A|TJoined/leaf", "A|TJoined/owner", "A|TPropagate" + ], + "Lifecycle": [ + "A|Deliver", + "A|Demonitor", + "A|Die", + "A|Dispatch", + "A|Drop", + "A|Enqueue", + "A|Hook", + "A|Link", + "A|Monitor", + "A|Panic", + "A|Publish", + "A|Request", + "A|Send", + "A|Signal", + "A|Stop", + "A|Trap", + "A|Unlink" + ], + "DeferredJobs": [ + "A|Cancel", + "A|Dispatch", + "A|Emit", + "A|Exit", + "A|Finish", + "A|Fire", + "A|JobDeadline", + "A|Release", + "A|Schedule", + "A|Start" + ], + "DeferredReplies": [ + "A|Death", + "A|Discharge", + "A|Dispatch", + "A|Exit", + "A|Send" ] } diff --git a/verification/tla/trace_coverage.py b/verification/tla/trace_coverage.py index dcbbb4f7..0338170b 100644 --- a/verification/tla/trace_coverage.py +++ b/verification/tla/trace_coverage.py @@ -32,6 +32,13 @@ [--model-keys CACHE] [--json OUT] [--baseline FILE] [--write-baseline FILE] [--top N] +Lifecycle.tla, DeferredJobs.tla and DeferredReplies.tla are covered by +action keys only ("static" models): the model side is the set of log +event arms of their trace spec (TraceLifecycle.tla, TraceJobs.tla, +TraceReplies.tla; each arm is one model action or a named group of them), +and a trace covers the arms its accepted replay takes. No state space is +explored for them, so they add no TLC run to the model side. + Each --traces source is reported cumulatively in the given order (for example the dedicated trace tests, then the whole test suite). --baseline FILE fails (exit 1) if an action key listed there is no longer covered; coverage @@ -84,6 +91,26 @@ }, } +# Models covered by their trace spec's arms (action keys only): name -> +# (trace spec, schemas). See the module docstring. +STATIC = { + "Lifecycle": ("TraceLifecycle.tla", {"lifecycle-v1"}), + "DeferredJobs": ("TraceJobs.tla", {"jobs-v1"}), + "DeferredReplies": ("TraceReplies.tla", {"replies-v1"}), +} +ALL = list(MODELS) + list(STATIC) + + +def static_model_keys(): + """The arms (`e.a = "X"`) of each static model's trace spec.""" + keys = {} + for name, (spec, _) in STATIC.items(): + with open(os.path.join(HERE, spec)) as f: + arms = re.findall(r'e\.a = "(\w+)"', f.read()) + keys[name] = {"A|" + a for a in arms} + return keys + + # Steps the model takes that no trace event can name (fairness devices). UNOBSERVABLE = {"Nested": ("A|Deliver", "T|Deliver|")} @@ -244,9 +271,30 @@ def model_of(trace): for name, spec in MODELS.items(): if schema in spec["schemas"]: return name + for name, (_, schemas) in STATIC.items(): + if schema in schemas: + return name raise ValueError(f"unsupported trace schema: {schema}") +def static_trace_keys(trace, model, tlc): + """Replay TRACE with trace_check's TraceRun and return the arms its log + takes (only if TLC accepts it).""" + workdir = tempfile.mkdtemp(prefix="cov-trace-") + try: + trace_check.generate(trace, workdir) + cmd = shlex.split(tlc) + ["-deadlock", "-workers", "1", "-cleanup", + "-metadir", os.path.join(workdir, "states"), "TraceRun"] + run = subprocess.run(cmd, cwd=workdir, env=tlc_env(workdir), capture_output=True, text=True) + if "Invariant TraceIncomplete is violated" not in run.stdout + run.stderr: + return model, "REJECTED", set() + with open(os.path.join(workdir, "TraceRun.tla")) as f: + arms = set(re.findall(r'\ba \|-> "(\w+)"', f.read())) + return model, "ACCEPTED", {"A|" + a for a in arms} + finally: + shutil.rmtree(workdir, ignore_errors=True) + + COV_TRACE = {"Supervision": "CovTraceSupervision", "Cohorts": "CovTraceCohorts", "Nested": "CovTraceNested"} @@ -258,6 +306,8 @@ def trace_keys(path, tlc): model = model_of(trace) if not trace.get("events"): return model, "EMPTY", set() + if model in STATIC: + return static_trace_keys(trace, model, tlc) workdir = tempfile.mkdtemp(prefix="cov-trace-") try: trace_check.generate(trace, workdir) @@ -288,9 +338,10 @@ def trace_keys(path, tlc): shutil.rmtree(workdir, ignore_errors=True) -# Trace schemas validated by trace_check.py but without a coverage model -# (the deferred-replies models and Lifecycle.tla): skipped here. -UNCOVERED_SCHEMAS = {"jobs-v1", "replies-v1", "lifecycle-v1"} +# Trace schemas validated by trace_check.py but without a coverage model: +# skipped here (none now: the deferred-replies models and Lifecycle.tla are +# STATIC models). +UNCOVERED_SCHEMAS = set() def covered(path): @@ -341,6 +392,7 @@ def collect(paths): "Supervision": {"Permanent", "Transient", "Temporary", "failed", "0", "mid", "max", "reserved"}, "Cohorts": {"Permanent", "Transient", "Temporary", "before", "after", "origin", "none"}, "Nested": set(), + **{name: set() for name in STATIC}, } @@ -411,18 +463,19 @@ def main(): for name, prefixes in UNOBSERVABLE.items(): model[name] = {k for k in model[name] if not k.startswith(prefixes)} model = {name: with_coarse(name, keys) for name, keys in model.items()} + model.update(static_model_keys()) sources = [] for spec in args.traces: label, _, path = spec.rpartition("=") sources.append((label or path, collect([path]))) - covered = {m: set() for m in MODELS} + covered = {m: set() for m in ALL} verdicts = [] rows = [] with concurrent.futures.ThreadPoolExecutor(args.jobs) as pool: for label, files in sources: - counts = {m: [0, 0] for m in MODELS} + counts = {m: [0, 0] for m in ALL} for path, (name, verdict, keys) in zip(files, pool.map(lambda f: trace_keys(f, args.tlc), files)): verdicts.append({"source": label, "trace": os.path.basename(path), "model": name, "verdict": verdict, "keys": len(keys)}) @@ -432,7 +485,7 @@ def main(): covered[name] |= with_coarse(name, keys) elif verdict == "REJECTED": print(f"not counted (rejected by the model): {label}: {os.path.basename(path)}") - for name in MODELS: + for name in ALL: mk, tk = kinds(model[name]), kinds(covered[name] & model[name]) rows.append((label, name, counts[name], {k: (len(tk[k]), len(mk[k])) for k in KINDS})) @@ -446,8 +499,10 @@ def main(): beyond = covered[name] - model[name] print(f"{name}: model {state_counts[name]} states over {len(MODELS[name]['configs'])} configuration(s); " f"{len(beyond)} trace keys outside the configured bounds") + for name in STATIC: + print(f"{name}: action keys are the arms of {STATIC[name][0]}") - for name in MODELS: + for name in ALL: # Rank unreached actions and coarse transitions; each coarse key # stands for its unreached fine variants (count shown). missing = model[name] - covered[name] @@ -467,12 +522,12 @@ def main(): json.dump({ "model": {m: sorted(v) for m, v in model.items()}, "covered": {m: sorted(v) for m, v in covered.items()}, - "unreached": {m: sorted(model[m] - covered[m], key=lambda k: (-score(m, k), k)) for m in MODELS}, - "beyond": {m: sorted(covered[m] - model[m]) for m in MODELS}, + "unreached": {m: sorted(model[m] - covered[m], key=lambda k: (-score(m, k), k)) for m in ALL}, + "beyond": {m: sorted(covered[m] - model[m]) for m in ALL}, "traces": verdicts, }, f, indent=1) - actions = {m: sorted(k for k in covered[m] & model[m] if k.startswith("A|")) for m in MODELS} + actions = {m: sorted(k for k in covered[m] & model[m] if k.startswith("A|")) for m in ALL} if args.write_baseline: with open(args.write_baseline, "w") as f: json.dump(actions, f, indent=1) From c43c863dd089e63e8461a1f8077042aedf87547a Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 15:28:13 -0300 Subject: [PATCH 173/208] feat(trace): reply traces cover linked jobs, the downstream actor, shorten and the cleanup deadline TraceReplies.tla now takes every DeferredReplies.tla action that has a code counterpart; before, it validated one owner's reply slots without linked jobs or the downstream actor B. New ReplyEvents (lifecycle-trace only; nothing without the feature): - Shorten: shorten_deadline lowered the deadline, recorded under the slot lock (a new `sync::shortened` hook; no-ops in the Loom shims). - PipeReply: pipe_reply* launched a job ("linked") or the slot refused it ("refused"), in both modes. - JobNotify / JobEnd / JobCancel: a linked job's done or deadline message dispatched unfenced; its execution ending without its notification; its cancellation while it holds the token (caller gone, or the owner's exit, which cancels jobs as exit begins). JobState carries its slot and whether its notification was taken. - CallB / BDispatch / BReply: while a linked job's work runs, a thread-local link (owner, request) marks requests it makes to another actor B; B's deferral, in-turn answer or later answer of that request is recorded on the owner's trace. - Stuck: a supervisor's cleanup deadline before the child's death point (CleanupDeadline); one after it (a slow destructor) is not recorded. Tests (reply_trace.rs): shortened deadline; job result; job deadline then a refused second job; caller gone cancels the job; owner exit cancels it; downstream deferred then answered; downstream answered in its turn; a supervised child stuck in stopped(); threads mode: a blocking linked job asking a backend. Controls (negative_checks.py), one per event kind, each rejected: shorten after the reply, a second job without a credit, a notification taken twice, a job ending before its notification, a job cancelled while its caller waits, a downstream request with no job, B dispatching a request no job made, B answering before deferring, the cleanup deadline before exit. 84/84 traces accepted; every control and model counterexample holds. model-sync events: the nine DeferredReplies warnings are gone. --- concurrency/src/lifecycle_trace.rs | 83 +++++ concurrency/src/reply/jobs.rs | 88 ++++- concurrency/src/reply/mod.rs | 24 +- concurrency/src/reply/slot.rs | 1 + concurrency/src/supervision.rs | 10 + concurrency/src/tasks/jobs.rs | 33 +- concurrency/src/threads/jobs.rs | 20 +- concurrency/tests/job_core_loom.rs | 1 + concurrency/tests/reply_slot_loom.rs | 1 + concurrency/tests/reply_trace.rs | 519 ++++++++++++++++++++++++++- verification/tla/TraceReplies.tla | 75 +++- verification/tla/negative_checks.py | 63 ++++ verification/tla/trace_check.py | 3 +- 13 files changed, 890 insertions(+), 31 deletions(-) diff --git a/concurrency/src/lifecycle_trace.rs b/concurrency/src/lifecycle_trace.rs index a848c868..7ad24585 100644 --- a/concurrency/src/lifecycle_trace.rs +++ b/concurrency/src/lifecycle_trace.rs @@ -353,6 +353,12 @@ mod imp { if action == "Death" && !lock(&DEAD).insert(owner) { return; } + // A cleanup deadline after the death point (a slow destructor + // before publication) is not a step of the reply registry, which + // is sealed: the model's `CleanupDeadline` precedes death. + if action == "Stuck" && lock(&DEAD).contains(&owner) { + return; + } lock(&REPLIES) .entry(owner) .or_default() @@ -367,6 +373,78 @@ mod imp { lock(&REPLIES).remove(&owner).unwrap_or_default() } + // Linked jobs and their downstream requests (DeferredReplies.tla + // `JobCallB`, `BDispatch`, `BReply`): while a `pipe_reply*` job's work + // runs, the thread carries its link (owner instance, request slot); a + // request it makes to another actor B is recorded on the owner's trace, + // and B's decisions on that request are recorded there too. + thread_local! { + static LINK: Cell> = const { Cell::new(None) }; + } + /// B's slots requested from a linked job: slot -> (owner, request, deferred). + static DOWNSTREAM: Mutex> = Mutex::new(BTreeMap::new()); + + /// Run `f` (a linked job's work, or one poll of it) with its link set. + pub(crate) fn in_link(link: Option<(u64, u64)>, f: impl FnOnce() -> R) -> R { + struct Restore(Option<(u64, u64)>); + impl Drop for Restore { + fn drop(&mut self) { + LINK.with(|l| l.set(self.0)); + } + } + let _restore = Restore(LINK.with(|l| l.replace(link))); + f() + } + + /// A request slot `slot` was created at instance `owner`: if the current + /// thread runs a linked job of a watched instance, it is that request's + /// downstream call (`CallB`). + pub(crate) fn downstream_sent(owner: u64, slot: u64) { + let Some((a, request)) = LINK.with(Cell::get) else { + return; + }; + if a == owner { + return; + } + atomic(|| { + if watched(a) { + lock(&DOWNSTREAM).insert(slot, (a, request, false)); + crate::lifecycle_trace::reply(a, "CallB", request, ""); + } + }); + } + + /// B deferred a downstream request (`BDispatch` to `stateB`). + pub(crate) fn downstream_deferred(slot: u64) { + atomic(|| { + let found = lock(&DOWNSTREAM).get_mut(&slot).map(|entry| { + entry.2 = true; + (entry.0, entry.1) + }); + if let Some((a, request)) = found { + crate::lifecycle_trace::reply(a, "BDispatch", request, "stateB"); + } + }); + } + + /// A downstream request's slot left OPEN with `kind` (inside the slot's + /// trace section): B's answer in its turn (`BDispatch` to `done`) or a + /// later one (`BReply`). The caller's side (abandoned, timeout) is not + /// B's step: the model folds it into the job's end (`EndDown`). + pub(crate) fn downstream_closed(slot: u64, kind: &'static str) { + let Some((a, request, deferred)) = lock(&DOWNSTREAM).remove(&slot) else { + return; + }; + if matches!(kind, "abandoned" | "timeout") { + return; + } + if deferred { + crate::lifecycle_trace::reply(a, "BReply", request, kind); + } else { + crate::lifecycle_trace::reply(a, "BDispatch", request, "done"); + } + } + /// Remove and return the job events recorded for an owner instance. pub fn take_jobs(owner: u64) -> Vec { lock(&WATCHED).remove(&owner); @@ -801,6 +879,11 @@ mod imp { pub(crate) fn job(_: u64, _: impl FnOnce() -> super::JobEvent) {} #[allow(dead_code)] #[inline(always)] + pub(crate) fn in_link(_: Option<(u64, u64)>, f: impl FnOnce() -> R) -> R { + f() + } + #[allow(dead_code)] + #[inline(always)] pub(crate) fn life_section(_: u64, f: impl FnOnce() -> R) -> R { f() } diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index c67f71bb..218e8c2c 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -226,6 +226,13 @@ pub(crate) struct JobState { /// The job's number in lifecycle traces. #[cfg(feature = "lifecycle-trace")] trace: u64, + /// A `pipe_reply*` job: its request's slot in reply traces, and whether + /// its notification was taken (DeferredReplies.tla `JobNotify`, `JobEnds`, + /// `JobCancel`). + #[cfg(feature = "lifecycle-trace")] + link: std::sync::OnceLock, + #[cfg(feature = "lifecycle-trace")] + notified: std::sync::atomic::AtomicBool, /// Async cancellation for `tasks` jobs (drops the work future). #[cfg(feature = "async")] cancel: spawned_rt::tasks::CancellationToken, @@ -255,6 +262,10 @@ impl JobState { core: JobCore::new(emit_credits), #[cfg(feature = "lifecycle-trace")] trace: crate::lifecycle_trace::next_job(), + #[cfg(feature = "lifecycle-trace")] + link: std::sync::OnceLock::new(), + #[cfg(feature = "lifecycle-trace")] + notified: std::sync::atomic::AtomicBool::new(false), #[cfg(feature = "async")] cancel: spawned_rt::tasks::CancellationToken::new(), #[cfg(feature = "async")] @@ -273,6 +284,54 @@ impl JobState { &self.cancel } + /// A `pipe_reply*` job was launched for the request in `slot` (reply + /// traces: the model's `PipeReply`). + #[cfg(feature = "lifecycle-trace")] + pub(crate) fn linked(&self, slot: u64) { + trace::atomic(|| { + let _ = self.link.set(slot); + trace::reply(self.owner.raw(), "PipeReply", slot, "linked"); + }); + } + + // DeferredReplies.tla linked-job actions of a `pipe_reply*` job, recorded + // on its owner's reply trace inside the trace section. + + /// Its notification was taken: the token travels with it from now on. + fn link_notified(&self) { + #[cfg(feature = "lifecycle-trace")] + self.notified + .store(true, std::sync::atomic::Ordering::Relaxed); + } + + /// `JobNotify`: its done or deadline message is dispatched to the owner. + #[cfg_attr(not(feature = "lifecycle-trace"), allow(unused_variables))] + fn link_notify(&self, kind: &'static str) { + #[cfg(feature = "lifecycle-trace")] + if let Some(slot) = self.link.get() { + trace::reply(self.owner.raw(), "JobNotify", *slot, kind); + } + } + + /// `JobEnds`: execution ended without taking the notification. + fn link_end(&self) { + #[cfg(feature = "lifecycle-trace")] + if let Some(slot) = self.link.get() { + trace::reply(self.owner.raw(), "JobEnd", *slot, ""); + } + } + + /// `JobCancel`: cancelled while the token is still with the job (not + /// once its notification was taken). + fn link_cancel(&self) { + #[cfg(feature = "lifecycle-trace")] + if let Some(slot) = self.link.get() { + if !self.notified.load(std::sync::atomic::Ordering::Relaxed) { + trace::reply(self.owner.raw(), "JobCancel", *slot, ""); + } + } + } + /// Record a `DeferredJobs.tla` action of this job (lifecycle traces). #[cfg_attr(not(feature = "lifecycle-trace"), allow(unused_variables))] fn record(&self, event: impl FnOnce(u64) -> trace::JobEvent) { @@ -289,6 +348,7 @@ impl JobState { if self.core.cancel() { self.count_cancel(); self.record(|job| trace::JobEvent::new("Cancel", job)); + self.link_cancel(); } }); self.wake_cancelled(); @@ -349,6 +409,9 @@ impl JobState { let (taken, unsent) = self.core.notify(enqueue); if taken { crate::instrumentation::hooks::job_notified(end); + self.link_notified(); + } else if !deadline { + self.link_end(); } if !deadline { // Only a held result's terminal notifies without the @@ -387,15 +450,19 @@ impl JobState { match finish { Finish::Taken => { crate::instrumentation::hooks::job_notified(end); + self.link_notified(); self.record(|job| trace::JobEvent { x: "notified", ..trace::JobEvent::new("Finish", job) }); } - Finish::Refused => self.record(|job| trace::JobEvent { - x: "quiet", - ..trace::JobEvent::new("Finish", job) - }), + Finish::Refused => { + self.record(|job| trace::JobEvent { + x: "quiet", + ..trace::JobEvent::new("Finish", job) + }); + self.link_end(); + } Finish::Held => self.record(|job| trace::JobEvent { x: "held", ..trace::JobEvent::new("Finish", job) @@ -479,9 +546,12 @@ impl JobState { /// model's `Finish` when the notification was suppressed). #[cfg(feature = "async")] pub(crate) fn ended_cancelled(&self) { - self.record(|job| trace::JobEvent { - x: "quiet", - ..trace::JobEvent::new("Finish", job) + trace::atomic(|| { + self.record(|job| trace::JobEvent { + x: "quiet", + ..trace::JobEvent::new("Finish", job) + }); + self.link_end(); }); } @@ -499,6 +569,9 @@ impl JobState { fenced, ..trace::JobEvent::new("Dispatch", job) }); + if !fenced && matches!(kind, "done" | "deadline") { + self.link_notify(kind); + } fenced }) } @@ -562,6 +635,7 @@ impl Job for JobState { let (newly, held) = self.core.cancel_keeping_held(); if newly { self.count_cancel(); + self.link_cancel(); } let scheduled = std::mem::take(&mut *lock(&self.scheduled)); *lock(&self.exit_released) = Some((held, scheduled)); diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 87d517ab..2130117b 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -48,9 +48,28 @@ mod sync { #[cfg_attr(not(feature = "lifecycle-trace"), allow(unused_variables))] pub(crate) fn left_open(owner: &SlotOwner, slot: usize, kind: &'static str, token: bool) { crate::instrumentation::hooks::left_open(kind, token); + #[cfg(feature = "lifecycle-trace")] + { + if let Some(registry) = owner { + crate::lifecycle_trace::reply( + registry.owner().raw(), + "Discharge", + slot as u64, + kind, + ); + } + crate::lifecycle_trace::downstream_closed(slot as u64, kind); + } + } + + /// A deferred token's deadline was lowered (`slot`'s lock held): + /// lifecycle traces (the model's `Shorten`). + #[inline(always)] + #[cfg_attr(not(feature = "lifecycle-trace"), allow(unused_variables))] + pub(crate) fn shortened(owner: &SlotOwner, slot: usize) { #[cfg(feature = "lifecycle-trace")] if let Some(registry) = owner { - crate::lifecycle_trace::reply(registry.owner().raw(), "Discharge", slot as u64, kind); + crate::lifecycle_trace::reply(registry.owner().raw(), "Shorten", slot as u64, ""); } } @@ -178,6 +197,7 @@ pub(crate) fn reply_slot( #[cfg(feature = "lifecycle-trace")] if let Some(owner) = owner { crate::lifecycle_trace::reply(owner.owner().raw(), "Send", Arc::as_ptr(&slot) as u64, ""); + crate::lifecycle_trace::downstream_sent(owner.owner().raw(), Arc::as_ptr(&slot) as u64); } (SlotHandle(Some(slot.clone())), ReplyReceiver::new(slot)) } @@ -330,6 +350,8 @@ impl<'t, R: Send + 'static> PendingReply<'t, R> { let registry = self.registry.take().ok_or(DeferError::NoCaller)?; match slot.defer(registry.pool(), &now_ticks) { Ok(()) => { + #[cfg(feature = "lifecycle-trace")] + crate::lifecycle_trace::downstream_deferred(Arc::as_ptr(&slot) as u64); registry.register(&slot); registry.track_deadline(&slot); Ok(ReplyTo { diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index d09f523c..11d4cdbf 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -475,6 +475,7 @@ impl Slot { let lower = state(self.word()) == OPEN && at < self.deadline(); if lower { self.deadline.store(at, Ordering::Release); + super::sync::shortened(&self.owner, self as *const Self as usize); // Moved out, not cloned: no executor code under the lock. A // woken receiver re-registers when it polls again. ( diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs index ba0f80b5..81666d84 100644 --- a/concurrency/src/supervision.rs +++ b/concurrency/src/supervision.rs @@ -2319,6 +2319,15 @@ async fn run_child(spec: ChildSpec, view: Arc>, core: Arc reason, Err(_) => { stuck = true; + // Reply traces: the cleanup deadline + // passed before the child's death + // point (DeferredReplies `CleanupDeadline`). + crate::lifecycle_trace::reply( + child.id().raw(), + "Stuck", + 0, + "", + ); core.fail_cleanup_for( view.snapshot(), format!("actor cleanup stuck for {}", spec.name), @@ -2336,6 +2345,7 @@ async fn run_child(spec: ChildSpec, view: Arc>, core: Arc Context { { let (state, slot, token) = self.link_reply(&opts, token)?; let watch = opts.cancel_on_caller_gone.then(|| slot.clone()); + #[cfg(feature = "lifecycle-trace")] + let link = (self.id().raw(), Arc::as_ptr(&slot) as u64); let hold = Hold::Linked(LinkedJob(slot)); let job = JobCtx { state: state.clone(), @@ -898,6 +900,15 @@ impl Context { // User code; a panic here drops the hold, which detaches the // linked job from the slot. let work = work(job); + // Reply traces: requests the work makes are this request's + // downstream calls. + #[cfg(feature = "lifecycle-trace")] + let work = { + let mut work = Box::pin(work); + std::future::poll_fn(move |cx| { + crate::lifecycle_trace::in_link(Some(link), || work.as_mut().poll(cx)) + }) + }; if !self.admit_job(&state, &opts) { // Dropping the hold detaches the linked job from the slot. drop((hold, watch, work)); @@ -906,6 +917,8 @@ impl Context { error: PipeError::OwnerStopping, }); } + #[cfg(feature = "lifecycle-trace")] + state.linked(link.1); Ok(self.launch_async(opts, state, hold, work, linked_done(token, done), watch)) } @@ -946,6 +959,12 @@ impl Context { if opts.cancel_on_caller_gone { crate::reply::jobs::cancel_when_closed(&slot, &state); } + #[cfg(feature = "lifecycle-trace")] + let work = { + let link = (self.id().raw(), Arc::as_ptr(&slot) as u64); + state.linked(link.1); + move |job| crate::lifecycle_trace::in_link(Some(link), || work(job)) + }; Ok(self.launch_blocking(opts, state, hold, native, work, linked_done(token, done))) } @@ -968,7 +987,19 @@ impl Context { }); }; let slot = token.slot_arc().clone(); - match slot.attach(&now_ticks) { + let attached = slot.attach(&now_ticks); + // The slot refused the job (reply traces: `PipeReply` handing the + // token back). + #[cfg(feature = "lifecycle-trace")] + if attached.is_err() { + crate::lifecycle_trace::reply( + self.id().raw(), + "PipeReply", + Arc::as_ptr(&slot) as u64, + "refused", + ); + } + match attached { Ok(()) => Ok((state, slot, token)), Err(Refused::Closed) => Err(PipeReplyError::Closed(token)), Err(Refused::Overloaded) => Err(PipeReplyError::Failed { diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index c1562634..ba678cd6 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -600,6 +600,12 @@ impl Context { if opts.cancel_on_caller_gone { crate::reply::jobs::cancel_when_closed(&slot, &state); } + #[cfg(feature = "lifecycle-trace")] + let work = { + let link = (self.id().raw(), Arc::as_ptr(&slot) as u64); + state.linked(link.1); + move |job| crate::lifecycle_trace::in_link(Some(link), || work(job)) + }; let done: Done = Box::new(move |outcome| done(outcome, token)); Ok(self.launch_blocking(opts, state, hold, worker, work, done)) } @@ -624,7 +630,19 @@ impl Context { } let state = JobState::new(self.id(), opts.emit_credits); let slot = token.slot_arc().clone(); - match slot.attach(&now_ticks) { + let attached = slot.attach(&now_ticks); + // The slot refused the job (reply traces: `PipeReply` handing the + // token back). + #[cfg(feature = "lifecycle-trace")] + if attached.is_err() { + crate::lifecycle_trace::reply( + self.id().raw(), + "PipeReply", + Arc::as_ptr(&slot) as u64, + "refused", + ); + } + match attached { Ok(()) => Ok((state, slot, token)), Err(Refused::Closed) => Err(PipeReplyError::Closed(token)), Err(Refused::Overloaded) => Err(PipeReplyError::Failed { diff --git a/concurrency/tests/job_core_loom.rs b/concurrency/tests/job_core_loom.rs index fc805703..f7e1c704 100644 --- a/concurrency/tests/job_core_loom.rs +++ b/concurrency/tests/job_core_loom.rs @@ -21,6 +21,7 @@ mod sync { f() } pub(crate) fn left_open(_: &SlotOwner, _: usize, _: &'static str, _: bool) {} + pub(crate) fn shortened(_: &SlotOwner, _: usize) {} pub(crate) use super::model_arc::{JobArc, JobWeak}; /// Loom has no timed wait; the model never parks on a slot. diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index 28d509c1..2231bff5 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -16,6 +16,7 @@ mod sync { f() } pub(crate) fn left_open(_: &SlotOwner, _: usize, _: &'static str, _: bool) {} + pub(crate) fn shortened(_: &SlotOwner, _: usize) {} /// Loom has no timed wait; tests park only on slots without a deadline. pub(crate) fn wait_ticks<'a, T>( diff --git a/concurrency/tests/reply_trace.rs b/concurrency/tests/reply_trace.rs index 657dbbb5..7159647e 100644 --- a/concurrency/tests/reply_trace.rs +++ b/concurrency/tests/reply_trace.rs @@ -8,7 +8,12 @@ use spawned_concurrency::{ lifecycle_trace, message::Message, - tasks::{Actor, ActorRef, ActorStart, Context, Deadline, Handler, PendingReply, ReplyTo}, + supervision::{ChildSpec, SupervisorConfig}, + system::ActorSystem, + tasks::{ + Actor, ActorRef, ActorStart, Context, Deadline, Handler, JobError, JobOptions, JobOutcome, + PendingReply, ReplyTo, + }, ActorError, ExitReason, }; use spawned_rt::tasks::sleep; @@ -27,6 +32,8 @@ fn trace_dir() -> PathBuf { #[derive(Default)] struct Owner { stored: Vec>, + /// The downstream actor B its jobs call. + backend: Option>, } impl Actor for Owner { @@ -111,6 +118,149 @@ impl Handler for Owner { } } +/// Deferred, its deadline shortened, then stored. +struct ShortHold; +impl Message for ShortHold { + type Result = u32; +} +impl Handler for Owner { + async fn handle(&mut self, _: ShortHold, reply: PendingReply<'_, u32>, _: &Context) { + if let Ok(token) = reply.defer() { + token.shorten_deadline(Duration::from_secs(3)); + self.stored.push(token); + } + } +} + +/// Deferred into a `pipe_reply` job: `work_ms` of work under an optional job +/// deadline (`deadline_ms`), optionally cancelled when the caller gives up; +/// with `downstream`, the work asks the backend B (`Ask`, or `AskNow`). +struct Piped { + work_ms: u64, + deadline_ms: Option, + cancel_on_gone: bool, + downstream: Option, +} +impl Message for Piped { + type Result = u32; +} +/// The job's notification, with the token back. +struct JobDone { + res: JobOutcome, + token: ReplyTo, +} +impl Message for JobDone { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: Piped, reply: PendingReply<'_, u32>, ctx: &Context) { + let Ok(token) = reply.defer() else { return }; + let mut opts = JobOptions::new(); + if let Some(ms) = msg.deadline_ms { + opts = opts.deadline(Duration::from_millis(ms)); + } + if msg.cancel_on_gone { + opts = opts.cancel_on_caller_gone(); + } + let backend = msg + .downstream + .map(|later| (later, self.backend.clone().unwrap())); + let work_ms = msg.work_ms; + let piped = ctx.pipe_reply( + opts, + token, + move |_| async move { + let got = match backend { + Some((true, b)) => b.request(Ask).await.unwrap_or(0), + Some((false, b)) => b.request(AskNow).await.unwrap_or(0), + None => 5, + }; + sleep(Duration::from_millis(work_ms)).await; + got + }, + |res, token| JobDone { res, token }, + ); + if let Err(e) = piped { + drop(e.into_token()); + } + } +} +impl Handler for Owner { + async fn handle(&mut self, msg: JobDone, _: PendingReply<'_, ()>, ctx: &Context) { + match msg.res { + Ok(v) => { + msg.token.reply(v); + } + Err(JobError::DeadlineExceeded) => { + // The work still runs and holds the request's credit: a + // second job needs a fresh one, and BOUND = 1 refuses it. + let again = ctx.pipe_reply( + JobOptions::new(), + msg.token, + |_| async { 1u32 }, + |res, token| JobDone { res, token }, + ); + match again { + Ok(_) => panic!("a second job found a free credit"), + Err(e) => { + e.into_token().reply(9); + } + } + } + Err(_) => drop(msg.token), + } + } +} + +/// The downstream actor B: defers `Ask` until `Release`, answers `AskNow`. +#[derive(Default)] +struct Backend { + held: Vec>, +} +impl Actor for Backend { + const MAX_OUTSTANDING: usize = 1; +} +struct Ask; +impl Message for Ask { + type Result = u32; +} +struct AskNow; +impl Message for AskNow { + type Result = u32; +} +struct Release; +impl Message for Release { + type Result = (); +} +struct Held; +impl Message for Held { + type Result = usize; +} +impl Handler for Backend { + async fn handle(&mut self, _: Ask, reply: PendingReply<'_, u32>, _: &Context) { + if let Ok(token) = reply.defer() { + self.held.push(token); + } + } +} +impl Handler for Backend { + async fn handle(&mut self, _: AskNow, reply: PendingReply<'_, u32>, _: &Context) { + reply.reply(4); + } +} +impl Handler for Backend { + async fn handle(&mut self, _: Release, _: PendingReply<'_, ()>, _: &Context) { + for token in self.held.drain(..) { + token.reply(8); + } + } +} +impl Handler for Backend { + async fn handle(&mut self, _: Held, reply: PendingReply<'_, usize>, _: &Context) { + reply.reply(self.held.len()); + } +} + /// Start an owner whose reply slots are traced. fn start() -> ActorRef { let actor = Owner::default().start(); @@ -125,9 +275,8 @@ fn id(actor: &ActorRef) -> u64 { /// Write the owner's reply trace once it is dead. fn write(name: &str, owner: u64) -> Vec { let events = lifecycle_trace::take_replies(owner); - assert_eq!( - events.last().map(|e| e.action), - Some("Death"), + assert!( + events.iter().any(|e| e.action == "Death"), "{name}: the owner's death was not recorded" ); let json = format!( @@ -285,3 +434,365 @@ async fn panic_in_turn_answers_stopped() { sleep(Duration::from_millis(10)).await; write("panic", id(&actor)); } + +#[tokio::test] +async fn shortened_deadline_then_reply() { + let actor = start(); + let held = actor + .send_request(ShortHold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + actor.request(Sync).await.unwrap(); + actor.send(Flush::Reply).unwrap(); + assert!(matches!(held.await, Ok(7))); + let events = stop("shorten", actor).await; + in_order( + &events, + &["Dispatch", "Shorten", "Discharge:ok", "Exit", "Death"], + ); +} + +/// `expected` occurs in `events` in this order. +fn in_order(events: &[String], expected: &[&str]) { + let mut rest = events.iter(); + for want in expected { + assert!( + rest.any(|e| e == want), + "{want} missing or out of order: {events:?}" + ); + } +} + +fn piped(work_ms: u64) -> Piped { + Piped { + work_ms, + deadline_ms: None, + cancel_on_gone: false, + downstream: None, + } +} + +#[tokio::test] +async fn job_result_answers_the_caller() { + let actor = start(); + assert!(matches!(actor.request(piped(1)).await, Ok(5))); + let events = stop("job_done", actor).await; + assert_eq!( + events, + [ + "Send", + "Dispatch", + "PipeReply:linked", + "JobNotify:done", + "Discharge:ok", + "Exit", + "Death" + ] + ); +} + +#[tokio::test] +async fn job_deadline_hands_the_token_back() { + let actor = start(); + let got = actor + .request(Piped { + deadline_ms: Some(20), + ..piped(150) + }) + .await; + assert!(matches!(got, Ok(9)), "{got:?}"); + // The work runs on past its deadline, then ends quietly. + sleep(Duration::from_millis(250)).await; + let events = stop("job_deadline", actor).await; + assert_eq!( + events, + [ + "Send", + "Dispatch", + "PipeReply:linked", + "JobNotify:deadline", + "PipeReply:refused", + "Discharge:ok", + "JobEnd", + "Exit", + "Death" + ] + ); +} + +#[tokio::test] +async fn caller_gone_cancels_the_job() { + let actor = start(); + let held = actor + .send_request( + Piped { + cancel_on_gone: true, + ..piped(10_000) + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); + actor.request(Sync).await.unwrap(); + drop(held); + sleep(Duration::from_millis(50)).await; + let events = stop("job_cancelled", actor).await; + in_order( + &events, + &[ + "PipeReply:linked", + "Discharge:abandoned", + "JobCancel", + "JobEnd", + "Exit", + "Death", + ], + ); +} + +#[tokio::test] +async fn owner_exit_cancels_the_job() { + let actor = start(); + let held = actor + .send_request(piped(10_000), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + actor.request(Sync).await.unwrap(); + let owner = id(&actor); + actor.context().stop(); + let got = held.await; + assert!( + matches!(got, Err(ActorError::ActorStopped(ExitReason::Normal))), + "{got:?}" + ); + actor.join().await; + sleep(Duration::from_millis(50)).await; + let events = write("job_exit", owner); + in_order(&events, &["PipeReply:linked", "Exit", "JobCancel"]); + in_order(&events, &["JobCancel", "JobEnd"]); + in_order(&events, &["JobCancel", "Death"]); +} + +/// Start an owner with a backend B for its jobs. +fn start_with_backend() -> (ActorRef, ActorRef) { + let backend = Backend::default().start(); + let actor = Owner { + backend: Some(backend.clone()), + ..Owner::default() + } + .start(); + lifecycle_trace::watch_deferred(id(&actor)); + (actor, backend) +} + +#[tokio::test] +async fn downstream_request_deferred_then_answered() { + let (actor, backend) = start_with_backend(); + let held = actor + .send_request( + Piped { + downstream: Some(true), + ..piped(1) + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); + while backend.request(Held).await.unwrap() == 0 { + sleep(Duration::from_millis(2)).await; + } + backend.send(Release).unwrap(); + assert!(matches!(held.await, Ok(8))); + let events = stop("downstream_later", actor).await; + assert_eq!( + events, + [ + "Send", + "Dispatch", + "PipeReply:linked", + "CallB", + "BDispatch:stateB", + "BReply:ok", + "JobNotify:done", + "Discharge:ok", + "Exit", + "Death" + ] + ); +} + +#[tokio::test] +async fn downstream_request_answered_in_its_turn() { + let (actor, _backend) = start_with_backend(); + let got = actor + .request(Piped { + downstream: Some(false), + ..piped(1) + }) + .await; + assert!(matches!(got, Ok(4)), "{got:?}"); + let events = stop("downstream_now", actor).await; + assert_eq!( + events, + [ + "Send", + "Dispatch", + "PipeReply:linked", + "CallB", + "BDispatch:done", + "JobNotify:done", + "Discharge:ok", + "Exit", + "Death" + ] + ); +} + +/// A supervised child that traps exits (so its parent's shutdown runs +/// `stopped()`) and whose `stopped()` outlasts two shutdown timeouts. +struct Slow; +impl Actor for Slow { + async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); + } + async fn stopped(&mut self, _: &Context) { + sleep(Duration::from_millis(600)).await; + } +} +impl Handler for Slow { + async fn handle(&mut self, msg: Echo, reply: PendingReply<'_, u32>, _: &Context) { + reply.reply(msg.0); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cleanup_deadline_before_death_reports_stuck() { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("reply-trace-stuck", 1).unwrap(); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let child = tree + .add( + ChildSpec::new("slow", &group, |_| async { Ok::<_, String>(Slow) }) + .shutdown_timeout(Duration::from_millis(100)), + ) + .unwrap(); + let _supervisor = tree.start().await.unwrap(); + let actor = child.current().unwrap(); + let owner: u64 = actor.id().to_string().parse().unwrap(); + lifecycle_trace::watch_deferred(owner); + assert!(matches!(actor.request(Echo(2)).await, Ok(2))); + let report = system.shutdown(Duration::from_secs(5)).await; + assert!(report.complete); + let events = write("stuck", owner); + assert_eq!( + events, + ["Send", "Dispatch", "Discharge:ok", "Exit", "Stuck", "Death"] + ); +} + +/// Threads mode: a blocking linked job asks a backend B, which answers in +/// its turn; the job's result answers the caller. +mod threads_mode { + use super::{in_order, trace_dir}; + use spawned_concurrency::{ + lifecycle_trace, + message::Message, + threads::{ + Actor, ActorRef, ActorStart, Context, Handler, JobOptions, JobOutcome, PendingReply, + ReplyTo, + }, + }; + + struct Owner(ActorRef); + impl Actor for Owner { + const MAX_OUTSTANDING: usize = super::BOUND; + } + struct Backend; + impl Actor for Backend {} + + struct Ask; + impl Message for Ask { + type Result = u32; + } + impl Handler for Backend { + fn handle(&mut self, _: Ask, reply: PendingReply<'_, u32>, _: &Context) { + reply.reply(4); + } + } + + struct Piped; + impl Message for Piped { + type Result = u32; + } + struct JobDone { + res: JobOutcome, + token: ReplyTo, + } + impl Message for JobDone { + type Result = (); + } + impl Handler for Owner { + fn handle(&mut self, _: Piped, reply: PendingReply<'_, u32>, ctx: &Context) { + let Ok(token) = reply.defer() else { return }; + let backend = self.0.clone(); + if let Err(e) = ctx.pipe_reply_blocking( + JobOptions::new(), + token, + move |_| backend.request(Ask).unwrap_or(0), + |res, token| JobDone { res, token }, + ) { + drop(e.into_token()); + } + } + } + impl Handler for Owner { + fn handle(&mut self, msg: JobDone, _: PendingReply<'_, ()>, _: &Context) { + match msg.res { + Ok(v) => { + msg.token.reply(v); + } + Err(_) => drop(msg.token), + } + } + } + + #[test] + fn blocking_job_asks_downstream() { + let backend = Backend.start(); + let actor = Owner(backend).start(); + let owner: u64 = actor.id().to_string().parse().unwrap(); + lifecycle_trace::watch_deferred(owner); + assert!(matches!(actor.request(Piped), Ok(4))); + actor.context().stop(); + actor.join(); + let events = lifecycle_trace::take_replies(owner); + let json = format!( + "{{\"name\":\"replies_threads_downstream\",\"schema\":\"replies-v1\",\"bound\":{},\"events\":[\n{}\n]}}\n", + super::BOUND, + events.iter().map(|e| e.to_json()).collect::>().join(",\n") + ); + let dir = trace_dir(); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("replies_threads_downstream.json"), json).unwrap(); + let names: Vec = events + .iter() + .map(|e| { + if e.x.is_empty() { + e.action.to_string() + } else { + format!("{}:{}", e.action, e.x) + } + }) + .collect(); + in_order( + &names, + &[ + "Send", + "Dispatch", + "PipeReply:linked", + "CallB", + "BDispatch:done", + "JobNotify:done", + "Discharge:ok", + "Exit", + "Death", + ], + ); + } +} diff --git a/verification/tla/TraceReplies.tla b/verification/tla/TraceReplies.tla index 7f91c902..8a21b1fe 100644 --- a/verification/tla/TraceReplies.tla +++ b/verification/tla/TraceReplies.tla @@ -1,27 +1,59 @@ ----------------------------- MODULE TraceReplies ----------------------------- (***************************************************************************) -(* Trace validation for DeferredReplies.tla (reply slots, without linked *) -(* jobs and the downstream actor). *) +(* Trace validation for DeferredReplies.tla: one owner instance A's reply *) +(* slots, its pipe_reply jobs and their downstream requests to an actor B. *) (* *) (* With the `lifecycle-trace` feature, the reply transport records per *) -(* owner instance: Send (slot created), Dispatch (its turn starts), *) -(* Discharge (the slot's one OPEN -> terminal transition, recorded in the *) -(* trace section together with the CAS; a reply counts at WRITING), Exit *) -(* (the registry enters exiting, after committing the turn's drops) and *) -(* Death (the death point D: the reply gate is fenced and the registry *) -(* sealed). The answers death then gives and their wake-ups (Answer) are *) -(* not logged and are taken silently. *) +(* owner instance (concurrency/src/lifecycle_trace.rs, ReplyEvent; `r` is *) +(* the request's slot, numbered by trace_check.py at each Send): *) +(* Send slot created (Send) *) +(* Dispatch its request turn starts (DispatchA) *) +(* Discharge the slot's one OPEN -> terminal transition, in the trace *) +(* section with its CAS; a reply counts at WRITING (x: kind) *) +(* Shorten shorten_deadline lowered the deadline, under the slot lock *) +(* (Shorten; TLC picks the new deadline) *) +(* PipeReply pipe_reply* launched a job ("linked") or the slot refused *) +(* it ("refused": closed, or no credit for a second job) *) +(* JobNotify the job's done or deadline message is dispatched to A, not *) +(* fenced (JobNotify; x: done ends the job, deadline keeps it)*) +(* JobEnd the job's execution ended without taking its notification *) +(* (after a deadline notification, or cancelled) (JobEnds) *) +(* JobCancel the job was cancelled while it still held the token: its *) +(* caller gave up (cancel_on_caller_gone) or A's exit began *) +(* (JobCancel) *) +(* CallB the job's work created a request slot at another actor B *) +(* (JobCallB; a thread-local link set while the work runs) *) +(* BDispatch B deferred that request ("stateB") or answered it in its *) +(* turn ("done": a reply, Overloaded at B's bound, a drop) *) +(* BReply B answered the deferred request later (BReply) *) +(* Exit the registry enters exiting (exit begins, before stopped())*) +(* Stuck a supervisor's cleanup deadline passed before A's death *) +(* point (CleanupDeadline); one after it (a slow destructor *) +(* before publication) is not recorded: the registry is sealed*) +(* Death the death point D: the reply gate is fenced, the registry *) +(* sealed (Death). The answers death then gives and their *) +(* wake-ups (Answer) are not logged and are taken silently. *) (* *) (* Turn-internal steps leave no trace and are taken silently: Defer that *) (* takes a credit (or finds the slot closed), Store, DropInTurn, a turn *) -(* end that answers nothing, a drop while exiting (the slot becomes exit *) -(* pending and is answered by Death), and the clock. A Discharge is the model *) -(* action that discharges exactly that request with that kind. The model's *) -(* turns are request turns: later replies run between them (from *) -(* non-request turns, or stopped()). *) +(* end that answers nothing, a drop while exiting (exit pending, answered *) +(* by Death), and the clock. Sent messages (not requests) leave no trace: *) +(* the model's turns are request turns and JobNotify turns, and replies *) +(* from other turns or stopped() are LaterReply. B's caller-side closes *) +(* (its slot abandoned when the job's future is dropped, or expired) are *) +(* not B's steps: the model folds them into the job's end (EndDown). *) (* *) -(* trace_check.py instantiates N (requests), BoundA (MAX_OUTSTANDING) and *) -(* a clock generous enough for the trace; deadlines are chosen by TLC. *) +(* Not mapped (an event the model cannot take rejects the trace): *) +(* - pipe_reply from a non-request turn (a stored token), or a token *) +(* shortened inside a job: the model attaches and shortens only tokens *) +(* of the current request turn or stored ones; *) +(* - a linked job cancelled through its JobHandle while its caller still *) +(* waits: the model cancels only on caller give-up or owner exit; *) +(* - a second downstream request from one request's jobs. *) +(* *) +(* trace_check.py instantiates N (requests), BoundA (A's MAX_OUTSTANDING), *) +(* BoundB (B's, default 1) and a clock generous enough for the trace; *) +(* deadlines are chosen by TLC. *) (***************************************************************************) EXTENDS DeferredReplies, Sequences, TLC @@ -47,6 +79,17 @@ Step(e) == [] e.a = "Exit" -> \/ ExitBegin \/ Panic /\ UNCHANGED slot \/ phase = "exiting" /\ UNCHANGED vars + [] e.a = "Shorten" -> e.r \in Req /\ Shorten(e.r) + [] e.a = "PipeReply" -> e.r \in Req /\ PipeReply(e.r) + /\ holder'[e.r] = IF e.x = "linked" THEN "doneA" ELSE "localA" + [] e.a = "JobNotify" -> e.r \in Req /\ JobNotify(e.r) + /\ jobs'[e.r] = IF e.x = "done" THEN jobs[e.r] - 1 ELSE jobs[e.r] + [] e.a = "JobEnd" -> e.r \in Req /\ JobEnds(e.r) + [] e.a = "JobCancel" -> e.r \in Req /\ JobCancel(e.r) + [] e.a = "CallB" -> e.r \in Req /\ JobCallB(e.r) + [] e.a = "BDispatch" -> e.r \in Req /\ BDispatch(e.r) /\ down'[e.r] = e.x + [] e.a = "BReply" -> e.r \in Req /\ BReply(e.r) + [] e.a = "Stuck" -> CleanupDeadline [] e.a = "Death" -> Death [] OTHER -> FALSE diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index d5ae84bd..07aa357a 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -334,6 +334,69 @@ def stopped_before_death(events): require_rejected(by_name["replies_panic"], stopped_before_death, "exit error answered Stopped before the death point", tlc) + def at(events, action, x=None, start=0): + return next(i for i, e in enumerate(events) + if i >= start and e["a"] == action and (x is None or e["x"] == x)) + + def move(events, src, dst): + events.insert(dst, events.pop(src)) + + def shorten_closed(events): + # Shorten a slot that was already answered. + i = at(events, "Shorten") + r = events[i]["r"] + answered = next(j for j, e in enumerate(events) + if e["a"] == "Discharge" and e["r"] == r) + move(events, i, answered) + require_rejected(by_name["replies_shorten"], shorten_closed, + "deadline shortened after the reply", tlc) + + def second_job_shares(events): + # A second job for a request whose first job still runs, with no + # free credit (BOUND = 1): ShareRunning. + events[at(events, "PipeReply", "refused")]["x"] = "linked" + require_rejected(by_name["replies_job_deadline"], second_job_shares, + "second linked job without a credit", tlc) + + def notified_twice(events): + i = at(events, "JobNotify") + events.insert(i + 1, dict(events[i])) + require_rejected(by_name["replies_job_done"], notified_twice, + "job notification taken twice", tlc) + + def end_before_notify(events): + # The job's work ends while its deadline notification is pending. + move(events, at(events, "JobEnd"), at(events, "JobNotify")) + require_rejected(by_name["replies_job_deadline"], end_before_notify, + "job ended before its notification was taken", tlc) + + def cancel_while_waiting(events): + # Cancelled while its caller still waits (no give-up, no exit). + move(events, at(events, "JobCancel"), at(events, "Discharge", "abandoned")) + require_rejected(by_name["replies_job_cancelled"], cancel_while_waiting, + "linked job cancelled while its caller waits", tlc) + + def call_before_job(events): + move(events, at(events, "CallB"), at(events, "PipeReply")) + require_rejected(by_name["replies_downstream_now"], call_before_job, + "downstream request with no running job", tlc) + + def dispatch_without_call(events): + events.pop(at(events, "CallB")) + require_rejected(by_name["replies_downstream_now"], dispatch_without_call, + "B dispatches a request no job made", tlc) + + def reply_before_defer(events): + move(events, at(events, "BReply"), at(events, "BDispatch")) + require_rejected(by_name["replies_downstream_later"], reply_before_defer, + "B answers a request it has not deferred", tlc) + + def stuck_while_running(events): + # The cleanup deadline before exit began. + move(events, at(events, "Stuck"), at(events, "Exit")) + require_rejected(by_name["replies_stuck"], stuck_while_running, + "cleanup deadline before exit", tlc) + def lifecycle_trace_controls(traces, tlc): """Edits of recorded base-lifecycle traces that Lifecycle.tla must diff --git a/verification/tla/trace_check.py b/verification/tla/trace_check.py index 07d90fe2..da2eaa03 100644 --- a/verification/tla/trace_check.py +++ b/verification/tla/trace_check.py @@ -140,7 +140,8 @@ def generate_replies(trace, workdir): + ",\n ".join(records) + "\n>>\n====\n") with open(os.path.join(workdir, "TraceRun.cfg"), "w") as out: out.write("INIT TraceInit\nNEXT TraceNext\nCONSTANTS\n" - f" N = {n}\n BoundA = {int(trace['bound'])}\n BoundB = 1\n" + f" N = {n}\n BoundA = {int(trace['bound'])}\n" + f" BoundB = {int(trace.get('bound_b', 1))}\n" f" MaxTime = {clock}\n MaxWait = {clock}\n Fault = \"None\"\n" " Trace <- TraceLog\n" "INVARIANT Safety\nINVARIANT TraceIncomplete\n") From ee8c328e9906548831d9151412d9b39505531e70 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 15:38:55 -0300 Subject: [PATCH 174/208] verification: DeferredReplies coverage floor at 14 action keys (linked jobs, downstream actor, shorten, stuck) --- verification/model-map.toml | 2 +- verification/tla/coverage-baseline.json | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/verification/model-map.toml b/verification/model-map.toml index 70a1c82e..605591a9 100644 --- a/verification/model-map.toml +++ b/verification/model-map.toml @@ -126,7 +126,7 @@ code = [ "concurrency/src/reply/**", "concurrency/src/lifecycle_trace.rs", ] -coverage_floor = 5 # of 5 trace-spec arms (trace_coverage.py STATIC) +coverage_floor = 14 # of 14 action keys (TraceReplies arms) [models.DeferredJobs] # Rank 11: pipe jobs, keyed replacement, ActorId and cancellation fencing, diff --git a/verification/tla/coverage-baseline.json b/verification/tla/coverage-baseline.json index 76750be6..02676079 100644 --- a/verification/tla/coverage-baseline.json +++ b/verification/tla/coverage-baseline.json @@ -124,10 +124,19 @@ "A|Start" ], "DeferredReplies": [ + "A|BDispatch", + "A|BReply", + "A|CallB", "A|Death", "A|Discharge", "A|Dispatch", "A|Exit", - "A|Send" + "A|JobCancel", + "A|JobEnd", + "A|JobNotify", + "A|PipeReply", + "A|Send", + "A|Shorten", + "A|Stuck" ] } From 4667d633c8a51a69db88a7896971115ed20aa5fe Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 15:49:02 -0300 Subject: [PATCH 175/208] test(semantics): a request admitted after started() panicked gets NoProc (single-cpu race) --- concurrency/tests/semantics_gaps.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/concurrency/tests/semantics_gaps.rs b/concurrency/tests/semantics_gaps.rs index b65aefaa..74eba939 100644 --- a/concurrency/tests/semantics_gaps.rs +++ b/concurrency/tests/semantics_gaps.rs @@ -1556,6 +1556,14 @@ mod tasks_contract { if fault == Fault::Stopped { assert!(matches!(first, Ok(1)), "{place:?}"); actor.child_handle().stop(); + } else if fault == Fault::Started { + // Queued before the death point: ActorStopped; admitted + // after it (started() already panicked, as on one CPU): + // NoProc. + assert!( + matches!(first, Err(ActorError::ActorStopped(_) | ActorError::NoProc)), + "{place:?} {fault:?}: {first:?}" + ); } else { assert!( matches!(first, Err(ActorError::ActorStopped(_))), From 100fe993a8af5099fe5ea25d334084de27cc0028 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:23:26 -0300 Subject: [PATCH 176/208] fix(jobs): exit cancellation runs its callbacks after every caller is answered Final review (Codex P1 2, tasks mode): JobState::cancel_for_exit set the cancellation flag and also called the job's CancellationToken::cancel() and wake_emitters() synchronously, so wakers registered by `job.cancelled()` or a blocked `emit` (arbitrary code, e.g. a JobCtx exported to a helper) ran inside cancel_all, before the owner's callers were discharged. A waker waiting for one of those callers deadlocked teardown. The flag is still set at the death point (`is_cancelled()` and `emit` refuse from there); the token's cancel and the emitter wakes now run in dispose_cancelled, after end_discharge, ahead of the released payloads' destruction. The models already order callbacks after discharge (DeferredReplies WokenAfterDischarge); this brings the code in line. Regression (fails without the fix, 3 s timeout then assertion): death_boundary::tasks_mode::job_cancellation_wakes_after_every_caller_is_answered polls `job.cancelled()` on a helper with a waker that waits for a queued caller's answer, with a handshake that the waker is registered. --- concurrency/src/reply/jobs.rs | 22 ++++-- concurrency/tests/death_boundary.rs | 113 ++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 8 deletions(-) diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 218e8c2c..aa4da024 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -639,11 +639,10 @@ impl Job for JobState { } let scheduled = std::mem::take(&mut *lock(&self.scheduled)); *lock(&self.exit_released) = Some((held, scheduled)); - #[cfg(feature = "async")] - { - self.cancel.cancel(); - self.wake_emitters(); - } + // The cancellation flag is set (`is_cancelled`, `emit` refuse from + // here); the callbacks it owes (the `cancelled()` waiters' wakers + // and blocked emitters, arbitrary code) run in `dispose_cancelled`, + // after the owner's callers are answered. } fn is_running(&self) -> bool { self.core.is_running() @@ -809,11 +808,18 @@ impl JobSet { }); } - /// After the owner's death point answered and woke every caller: destroy - /// what the exit's cancellation released (held results, scheduled - /// emits; contained through each job's failure sink). Idempotent. + /// After the owner's death point answered and woke every caller: run + /// the callbacks the exit's cancellation owes (wake `cancelled()` + /// waiters and blocked emitters), then destroy what it released (held + /// results, scheduled emits; contained through each job's failure + /// sink). Idempotent. pub(crate) fn dispose_cancelled(&self) { let cancelled = std::mem::take(&mut *lock(&self.exit_cancelled)); + #[cfg(feature = "async")] + for job in &cancelled { + job.cancel.cancel(); + job.wake_emitters(); + } for job in &cancelled { let released = lock(&job.exit_released).take(); if let Some((held, scheduled)) = released { diff --git a/concurrency/tests/death_boundary.rs b/concurrency/tests/death_boundary.rs index fcb208df..8afd0aa6 100644 --- a/concurrency/tests/death_boundary.rs +++ b/concurrency/tests/death_boundary.rs @@ -406,6 +406,119 @@ mod tasks_mode { ); } + /// Pipes a job that hands its `JobCtx` to a helper and never ends. + struct ExportJob(mpsc::Sender>); + impl Message for ExportJob { + type Result = (); + } + struct JobDone; + impl Message for JobDone { + type Result = (); + } + impl Handler for Owner { + async fn handle( + &mut self, + msg: ExportJob, + reply: PendingReply<'_, ()>, + ctx: &Context, + ) { + let export = msg.0; + ctx.pipe( + spawned_concurrency::tasks::JobOptions::new(), + move |job| { + export.send(job).unwrap(); + std::future::pending::<()>() + }, + |_| JobDone, + ) + .unwrap(); + reply.reply(()); + } + } + impl Handler for Owner { + async fn handle(&mut self, _: JobDone, _: PendingReply<'_, ()>, _: &Context) {} + } + + /// Final review (Codex P1 2): the owner's exit cancels its jobs, but the + /// cancellation's callbacks (a `cancelled()` waiter's waker, arbitrary + /// code) run only after every caller is answered. Here a helper polls + /// `job.cancelled()` with a waker that waits for a queued caller's + /// answer; the exit used to run it inside cancellation, before that + /// caller could be answered. + #[test] + fn job_cancellation_wakes_after_every_caller_is_answered() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Mutex; + use std::task::{Wake, Waker}; + struct WaitForCaller { + caller: Mutex>, + answered: AtomicBool, + } + impl Wake for WaitForCaller { + fn wake(self: Arc) { + let began = Instant::now(); + while began.elapsed() < Duration::from_secs(3) { + if self.caller.lock().unwrap().try_recv().is_some() { + self.answered.store(true, Ordering::SeqCst); + return; + } + std::thread::sleep(Duration::from_millis(1)); + } + } + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let _guard = runtime.enter(); + let actor = Owner { + _state: inert(), + parked: Vec::new(), + } + .start(); + let (export, exported) = mpsc::channel(); + runtime.block_on(actor.request(ExportJob(export))).unwrap(); + let job = exported.recv_timeout(LIMIT).unwrap(); + let (unblock, blocked) = mpsc::channel(); + let (entered_tx, entered) = mpsc::channel(); + actor.send(Block(blocked, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + let caller = actor + .send_request(Queued(inert()), spawned_concurrency::Deadline::Never) + .unwrap(); + let waiter = Arc::new(WaitForCaller { + caller: Mutex::new(caller), + answered: AtomicBool::new(false), + }); + // Handshake: the helper's cancelled() is registered with the waker. + let (polled_tx, polled) = mpsc::channel(); + let (done_tx, done) = mpsc::channel::<()>(); + let helper = { + let waiter = waiter.clone(); + std::thread::spawn(move || { + let waker = Waker::from(waiter); + let mut cancelled = Box::pin(job.cancelled()); + assert!(cancelled + .as_mut() + .poll(&mut std::task::Context::from_waker(&waker)) + .is_pending()); + polled_tx.send(()).unwrap(); + let _ = done.recv_timeout(LIMIT * 2); + }) + }; + polled.recv_timeout(LIMIT).unwrap(); + actor.context().stop(); + unblock.send(()).unwrap(); + runtime.block_on(actor.join()); + assert!( + waiter.answered.load(Ordering::SeqCst), + "a job cancellation waker ran before every caller was answered" + ); + done_tx.send(()).unwrap(); + helper.join().unwrap(); + } + fn stopped(result: Option>, reason: &ExitReason) -> Option<()> { match result? { Err(ActorError::ActorStopped(r)) if r == *reason => Some(()), From c34ebf01e2bc6d812a86b2c10da3ec3a6cfeb46c Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:24:18 -0300 Subject: [PATCH 177/208] fix(reply): the death point's discharge retains every slot it meets until every caller is answered Final review (Codex P1 1, both modes): Registry::close upgraded each listed weak slot and dropped the temporary reference at once; for an already-answered slot close_slot kept nothing. If the caller dropped its receiver meanwhile, the close's reference was the last, and the unread reply's destructor (user code) ran inside the close loop, before later deferred slots and the queued mailbox requests were answered. While discharging, close_slot now retains every slot (closed here or already terminal) in the batch, and end_discharge releases them after running the wake-ups, i.e. after the mailbox discharge too. The deadline heap holds only weak references. The models' rule (no callback before discharge, DeferredReplies WokenAfterDischarge) is unchanged; payload destruction is abstracted there. Regression (fails without the fix in both modes): failpoints an_unread_reply_is_destroyed_after_every_caller_is_answered (tasks and native): new site `reply.close_upgraded` pauses the close holding the answered slot while the caller drops its receiver; the unread reply's destructor waits for a queued caller's answer. --- concurrency/src/failpoints.rs | 8 ++ concurrency/src/reply/registry.rs | 45 +++++-- concurrency/tests/failpoints.rs | 205 ++++++++++++++++++++++++++++++ docs/TESTING.md | 1 + docs/design/deferred-replies.md | 13 +- 5 files changed, 253 insertions(+), 19 deletions(-) diff --git a/concurrency/src/failpoints.rs b/concurrency/src/failpoints.rs index 59009320..55645c55 100644 --- a/concurrency/src/failpoints.rs +++ b/concurrency/src/failpoints.rs @@ -71,6 +71,14 @@ pub const SITES: &[Site] = &[ Delay models slow cleanup (shutdown deadlines); panic models a \ failing `stopped()`.", }, + Site { + name: "reply.close_upgraded", + has_error: false, + description: "The death point's close holds an upgraded deferred slot, before \ + closing it. Delay lets the caller drop its receiver meanwhile: an \ + unread reply must still be destroyed only after every caller is \ + answered (tasks and threads).", + }, Site { name: "monitor.down_before_dispatch", has_error: false, diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index ea2c2067..b42390f2 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -45,7 +45,10 @@ pub(crate) trait Closeable: Send + Sync { } /// A closed slot and its deferred wake-up. -type PendingWake = (Arc, Wake); +/// A slot met by the death point's discharge, retained (whatever its state: +/// an unread reply's value is destroyed only after every caller is answered) +/// with the wake-up its close collected, if it closed it. +type PendingWake = (Arc, Option); impl Expirable for Slot { fn pending_deadline(&self) -> Option { @@ -394,18 +397,24 @@ impl Registry { batch.get_or_insert_with(Vec::new); } - /// Every obligation of the death point is answered: run the collected - /// wake-ups (executor or user code), outside every lock. Later closes - /// wake at once. Idempotent. + /// Every obligation of the death point is answered (deferred slots and + /// the queued mailbox): run the collected wake-ups (executor or user + /// code) outside every lock, then release the retained slots (an unread + /// reply's destructor is user code). Later closes wake at once. + /// Idempotent. pub(crate) fn end_discharge(&self) { - let batch = self + let mut batch = self .discharge .lock() .unwrap_or_else(|p| p.into_inner()) - .take(); - for (slot, wake) in batch.into_iter().flatten() { - slot.run_wake(wake); + .take() + .unwrap_or_default(); + for (slot, wake) in &mut batch { + if let Some(wake) = wake.take() { + slot.run_wake(wake); + } } + drop(batch); } /// A turn destroyed unfinished (its runtime dropped the actor): its @@ -416,19 +425,24 @@ impl Registry { } /// Close one slot as `STOPPED`, deferring its wake-up while discharging. + /// While discharging, the slot itself is retained too, closed here or + /// already terminal: this may be the last strong reference to an unread + /// reply, whose destructor must not run before every caller is answered. fn close_slot(&self, slot: Arc) -> bool { - let Some(wake) = slot.close_collect() else { - return false; - }; + let wake = slot.close_collect(); + let closed = wake.is_some(); let mut batch = self.discharge.lock().unwrap_or_else(|p| p.into_inner()); match batch.as_mut() { Some(batch) => batch.push((slot, wake)), None => { drop(batch); - slot.run_wake(wake); + if let Some(wake) = wake { + slot.run_wake(wake); + } + drop(slot); } } - true + closed } /// Death: every open token listed, and every exit-pending slot, becomes @@ -448,7 +462,10 @@ impl Registry { let closed = tokens .into_iter() .filter_map(|t| t.upgrade()) - .filter(|s| self.close_slot(s.clone())) + .filter(|s| { + fail_point!("reply.close_upgraded"); + self.close_slot(s.clone()) + }) .count(); // The heap is dropped after the close (§5 step 3.4), and the service // forgets this owner, so dead actors leave nothing behind. diff --git a/concurrency/tests/failpoints.rs b/concurrency/tests/failpoints.rs index 0ec035a3..0b423ee3 100644 --- a/concurrency/tests/failpoints.rs +++ b/concurrency/tests/failpoints.rs @@ -1164,6 +1164,127 @@ async fn demonitor_flushes_a_down_in_dispatch() { observer.join().await; } +// ------------------------------------- death-boundary discharge (final review) + +type SharedReceiver = Arc>>>; + +/// An unread reply whose destructor waits (at most 3 s) for a queued +/// caller's answer and records whether it came. +struct WaitsForCaller { + caller: SharedReceiver, + answered: Arc, +} +impl Drop for WaitsForCaller { + fn drop(&mut self) { + let began = Instant::now(); + while began.elapsed() < Duration::from_secs(3) { + if let Some(receiver) = self.caller.lock().unwrap().as_mut() { + if receiver.try_recv().is_some() { + self.answered.store(1, Ordering::SeqCst); + return; + } + } + std::thread::sleep(Duration::from_millis(1)); + } + } +} +/// Deferred, then answered at once with a `WaitsForCaller` nobody reads. +struct AnswerLater(WaitsForCaller); +impl Message for AnswerLater { + type Result = WaitsForCaller; +} +/// Blocks the actor until released; reports entry. +struct Gate(std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>); +impl Message for Gate { + type Result = (); +} +struct Answer; +impl Message for Answer { + type Result = (); +} +struct Discharger; +impl Actor for Discharger {} +impl Handler for Discharger { + async fn handle( + &mut self, + msg: AnswerLater, + reply: spawned_concurrency::PendingReply<'_, WaitsForCaller>, + _: &Context, + ) { + reply.defer().unwrap().reply(msg.0); + } +} +impl Handler for Discharger { + async fn handle( + &mut self, + msg: Gate, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let _ = msg.1.send(()); + let _ = msg.0.recv_timeout(LIMIT); + reply.reply(()); + } +} +impl Handler for Discharger { + async fn handle( + &mut self, + _: Answer, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } +} + +/// Final review (Codex P1 1): the death point's close holds an upgraded, +/// already-answered deferred slot while its caller drops the receiver. The +/// close's reference was then the last: the unread reply's destructor ran +/// inside the close, before the queued caller was answered (and here waited +/// for that answer). Every slot the discharge meets is now retained until +/// every caller is answered. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_unread_reply_is_destroyed_after_every_caller_is_answered() { + let _scenario = FailScenario::setup(); + let actor = Discharger.start(); + let caller: SharedReceiver = Arc::default(); + let answered = Arc::new(AtomicUsize::new(0)); + let unread = actor + .send_request( + AnswerLater(WaitsForCaller { + caller: caller.clone(), + answered: answered.clone(), + }), + spawned_concurrency::Deadline::Never, + ) + .unwrap(); + let (release, gate) = std::sync::mpsc::channel(); + let (entered_tx, entered) = std::sync::mpsc::channel(); + actor.send(Gate(gate, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); // AnswerLater was handled + *caller.lock().unwrap() = Some( + actor + .send_request(Answer, spawned_concurrency::Deadline::Never) + .unwrap(), + ); + failpoints::configure( + "reply.close_upgraded", + FailPoint::new(Action::Delay(Duration::from_millis(300))).times(1), + ) + .unwrap(); + actor.context().stop(); + release.send(()).unwrap(); + reached("reply.close_upgraded", 1).await; + // The caller gives up on its unread reply while the close holds it. + drop(unread); + tokio::time::timeout(LIMIT * 2, actor.join()).await.unwrap(); + assert_eq!( + answered.load(Ordering::SeqCst), + 1, + "an unread reply was destroyed before every caller was answered" + ); +} + mod native { use super::*; use spawned_concurrency::threads::{ @@ -1446,4 +1567,88 @@ mod native { observer.context().stop(); observer.join(); } + + struct NativeDischarger; + impl NativeActor for NativeDischarger {} + impl NativeHandler for NativeDischarger { + fn handle( + &mut self, + msg: AnswerLater, + reply: spawned_concurrency::reply::PendingReply<'_, WaitsForCaller>, + _: &NativeContext, + ) { + reply.defer().unwrap().reply(msg.0); + } + } + impl NativeHandler for NativeDischarger { + fn handle( + &mut self, + msg: Gate, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &NativeContext, + ) { + let _ = msg.1.send(()); + let _ = msg.0.recv_timeout(LIMIT); + reply.reply(()); + } + } + impl NativeHandler for NativeDischarger { + fn handle( + &mut self, + _: Answer, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &NativeContext, + ) { + reply.reply(()); + } + } + fn wait_hits(site: &str) { + let began = Instant::now(); + while failpoints::stats(site).hits < 1 { + assert!(began.elapsed() < LIMIT, "site {site} not reached"); + std::thread::sleep(Duration::from_millis(2)); + } + } + + /// Threads mode of `an_unread_reply_is_destroyed_after_every_caller_is_answered`. + #[test] + fn an_unread_reply_is_destroyed_after_every_caller_is_answered() { + let _scenario = FailScenario::setup(); + let actor = NativeDischarger.start(); + let caller: SharedReceiver = Arc::default(); + let answered = Arc::new(AtomicUsize::new(0)); + let unread = actor + .send_request( + AnswerLater(WaitsForCaller { + caller: caller.clone(), + answered: answered.clone(), + }), + spawned_concurrency::Deadline::Never, + ) + .unwrap(); + let (release, gate) = std::sync::mpsc::channel(); + let (entered_tx, entered) = std::sync::mpsc::channel(); + actor.send(Gate(gate, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + *caller.lock().unwrap() = Some( + actor + .send_request(Answer, spawned_concurrency::Deadline::Never) + .unwrap(), + ); + failpoints::configure( + "reply.close_upgraded", + FailPoint::new(Action::Delay(Duration::from_millis(300))).times(1), + ) + .unwrap(); + actor.context().stop(); + release.send(()).unwrap(); + wait_hits("reply.close_upgraded"); + drop(unread); + actor.join(); + assert_eq!( + answered.load(Ordering::SeqCst), + 1, + "an unread reply was destroyed before every caller was answered" + ); + } } diff --git a/docs/TESTING.md b/docs/TESTING.md index 71d8e9ac..ff085c1c 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -1496,6 +1496,7 @@ and prints the seed when a test fails. Configuration is process-global. | `actor.exit_before_deliver` | no | A cloned context's exit signal racing its sender's death (tasks and threads) | | `link.before_install` | no | Peer exiting while a link is installed (tasks and threads) | | `monitor.before_install` | no | Target exiting while a monitor is installed (tasks and threads) | +| `reply.close_upgraded` | no | The death point's close holding an already-answered slot while its caller drops the receiver: the unread reply is destroyed only after every caller is answered (tasks and threads) | | `monitor.down_before_dispatch` | no | `demonitor` on another thread returning while the monitor's `Down` is in dispatch: it is flushed (tasks and threads) | `tests/failpoints.rs` drives every site and checks invariants through public diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index 7df2e4eb..bab7b548 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -400,11 +400,14 @@ current turn's slot) and a job set. The order is: 7. the exit reason is published, and monitors and links fire. Steps 1 to 4 run no user code: steps 3 and 4 only make terminal - transitions; the callers' wake-ups (wakers are executor or user code), - and a runtime drop's "stopped() skipped" log, run after the last answer - and before step 5. Cancellation in step 1 only marks the jobs: the - payloads it releases (held results, scheduled emits) are destroyed in - step 5. Steps 3 and 4 follow the seal instead of being one atomic step + transitions, and every slot they meet stays referenced (an unread + reply's value is user code when destroyed) until step 5. After the last + answer, and before step 5, run in this order: the callers' wake-ups + (executor or user code), a runtime drop's "stopped() + skipped" log, and the callbacks the jobs' cancellation owes (`cancelled()` + waiters, blocked emitters). Cancellation in step 1 only sets the jobs' + flag: the payloads it releases (held results, scheduled emits) are + destroyed in step 5. Steps 3 and 4 follow the seal instead of being one atomic step with it, but nothing between them depends on the requests they answer, so the models' atomic death is a sound abstraction of them (they commute left to the seal). From 4cbc032f0fae2c12715264fefb9cd9468d85b5ec Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:26:53 -0300 Subject: [PATCH 178/208] fix(reply): discharge notifies parked blocking receivers before running any waker Final review (Codex P1 3, both modes): end_discharge ran each slot's whole wake in turn, and within a slot the condvar notification came after the wakers. A queued caller's custom waker (arbitrary code) could wait for a caller parked in a blocking recv() whose notification was still queued behind it: with Deadline::Never neither proceeded. end_discharge now notifies every parked receiver in the batch first (no user code), then runs the wakers, then releases the retained slots; a slot's own Wake::run, and shorten(), also notify parked receivers before their wakers. Regression (fails without the fix in both modes): failpoints parked_receivers_wake_before_any_waker (tasks and native): caller B is parked in recv() (new handshake site `reply.recv_parked`, a hook under the slot's lock, a no-op in the Loom shims) and caller A's waker waits for B to return. Loom reply_slot/job_core/registry_turn pass. --- concurrency/src/failpoints.rs | 7 ++ concurrency/src/reply/mod.rs | 7 ++ concurrency/src/reply/registry.rs | 18 ++++- concurrency/src/reply/slot.rs | 28 +++++-- concurrency/tests/failpoints.rs | 109 +++++++++++++++++++++++++++ concurrency/tests/job_core_loom.rs | 1 + concurrency/tests/reply_slot_loom.rs | 1 + docs/TESTING.md | 1 + docs/design/deferred-replies.md | 5 +- 9 files changed, 163 insertions(+), 14 deletions(-) diff --git a/concurrency/src/failpoints.rs b/concurrency/src/failpoints.rs index 55645c55..4f3d2367 100644 --- a/concurrency/src/failpoints.rs +++ b/concurrency/src/failpoints.rs @@ -79,6 +79,13 @@ pub const SITES: &[Site] = &[ unread reply must still be destroyed only after every caller is \ answered (tasks and threads).", }, + Site { + name: "reply.recv_parked", + has_error: false, + description: "A blocking `ReplyReceiver::recv` is about to wait (under its slot's \ + lock). A handshake: once hit, the receiver is parked and only a \ + notification wakes it.", + }, Site { name: "monitor.down_before_dispatch", has_error: false, diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 2130117b..a18e2890 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -73,6 +73,13 @@ mod sync { } } + /// A blocking receiver is about to wait on its slot (under the slot's + /// lock): the failpoint site `reply.recv_parked`, a handshake for tests. + #[inline(always)] + pub(crate) fn parked() { + fail_point!("reply.recv_parked"); + } + /// Wait on `condvar` for at most `ticks` clock ticks (nanoseconds), or /// until notified when `None`. pub(crate) fn wait_ticks<'a, T>( diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index b42390f2..0248d4fc 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -40,6 +40,7 @@ pub(crate) trait Closeable: Send + Sync { /// Close without waking; the returned work runs with `run_wake`. fn close_collect(&self) -> Option; fn run_wake(&self, wake: Wake); + fn notify_parked(&self, wake: &mut Wake); fn terminate(&self, to: Terminal) -> bool; fn is_terminal(&self) -> bool; } @@ -67,6 +68,9 @@ impl Closeable for Slot { fn run_wake(&self, wake: Wake) { Slot::run_wake(self, wake) } + fn notify_parked(&self, wake: &mut Wake) { + Slot::notify_parked(self, wake) + } fn terminate(&self, to: Terminal) -> bool { Slot::terminate(self, to) } @@ -398,10 +402,11 @@ impl Registry { } /// Every obligation of the death point is answered (deferred slots and - /// the queued mailbox): run the collected wake-ups (executor or user - /// code) outside every lock, then release the retained slots (an unread - /// reply's destructor is user code). Later closes wake at once. - /// Idempotent. + /// the queued mailbox): run the collected wake-ups outside every lock, + /// parked blocking receivers first, so none waits behind an arbitrary + /// waker; then the wakers (executor or user code); then release the + /// retained slots (an unread reply's destructor is user code). Later + /// closes wake at once. Idempotent. pub(crate) fn end_discharge(&self) { let mut batch = self .discharge @@ -409,6 +414,11 @@ impl Registry { .unwrap_or_else(|p| p.into_inner()) .take() .unwrap_or_default(); + for (slot, wake) in &mut batch { + if let Some(wake) = wake { + slot.notify_parked(wake); + } + } for (slot, wake) in &mut batch { if let Some(wake) = wake.take() { slot.run_wake(wake); diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 11d4cdbf..969212e0 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -217,19 +217,24 @@ pub(crate) struct Wake { } impl Wake { - fn run(self, slot_parked: &Condvar) { - if let Some(pool) = self.pool { + /// Wake the slot's parked blocking receivers (no user code). Done first, + /// so a parked `recv()` never waits behind an arbitrary waker. + fn notify_parked(&mut self, slot_parked: &Condvar) { + if std::mem::take(&mut self.notify) { + slot_parked.notify_all(); + } + } + fn run(mut self, slot_parked: &Condvar) { + if let Some(pool) = self.pool.take() { pool.release(self.release); } + self.notify_parked(slot_parked); if let Some(waker) = self.waker { waker.wake(); } for watcher in self.watchers { watcher.wake(); } - if self.notify { - slot_parked.notify_all(); - } } } @@ -444,6 +449,12 @@ impl Slot { wake.run(&self.parked); } + /// The parked-receiver part of `wake`, run ahead of every waker of a + /// batch (see `Registry::end_discharge`). + pub(crate) fn notify_parked(&self, wake: &mut Wake) { + wake.notify_parked(&self.parked); + } + /// `[deadline]`: expire the slot if `now >= deadline`. Returns whether the /// deadline has passed (whoever won the transition). pub(crate) fn check_deadline(&self, now: &impl Fn() -> u64) -> bool { @@ -491,15 +502,15 @@ impl Slot { // Both sides re-arm their timers for the earlier deadline: the // token's waiters, the caller's receiver and parked `recv_blocking` // threads (which recompute their wait). + if parked { + self.parked.notify_all(); + } for watcher in watchers { watcher.wake(); } if let Some(waker) = waker { waker.wake(); } - if parked { - self.parked.notify_all(); - } self.check_deadline(now); lowered } @@ -740,6 +751,7 @@ impl Slot { continue; } inner.parked += 1; + super::sync::parked(); let mut inner = wait_ticks(&self.parked, inner, remaining); inner.parked -= 1; } diff --git a/concurrency/tests/failpoints.rs b/concurrency/tests/failpoints.rs index 0b423ee3..0d24a001 100644 --- a/concurrency/tests/failpoints.rs +++ b/concurrency/tests/failpoints.rs @@ -1285,6 +1285,62 @@ async fn an_unread_reply_is_destroyed_after_every_caller_is_answered() { ); } +/// Final review (Codex P1 3): the discharge batch wakes parked blocking +/// receivers before any arbitrary waker. Caller A's waker waits for caller +/// B, parked in `recv()`, to return; B used to be notified only after A's +/// waker, so neither proceeded. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn parked_receivers_wake_before_any_waker() { + use std::task::{Wake, Waker}; + struct WaitForB(Arc, Arc); + impl Wake for WaitForB { + fn wake(self: Arc) { + let began = Instant::now(); + while began.elapsed() < Duration::from_secs(3) { + if self.0.load(Ordering::SeqCst) == 1 { + self.1.store(1, Ordering::SeqCst); + return; + } + std::thread::sleep(Duration::from_millis(1)); + } + } + } + let _scenario = FailScenario::setup(); + let actor = Discharger.start(); + let (release, gate) = std::sync::mpsc::channel(); + let (entered_tx, entered) = std::sync::mpsc::channel(); + actor.send(Gate(gate, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + let never = spawned_concurrency::Deadline::Never; + let a = actor.send_request(Answer, never).unwrap(); + let b = actor.send_request(Answer, never).unwrap(); + let (b_returned, a_saw_b) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0))); + let waker = Waker::from(Arc::new(WaitForB(b_returned.clone(), a_saw_b.clone()))); + let mut a = Box::pin(std::future::IntoFuture::into_future(a)); + assert!( + std::future::Future::poll(a.as_mut(), &mut std::task::Context::from_waker(&waker)) + .is_pending() + ); + failpoints::configure("reply.recv_parked", FailPoint::new(Action::Off)).unwrap(); + let parked = { + let b_returned = b_returned.clone(); + std::thread::spawn(move || { + let _ = b.recv(); + b_returned.store(1, Ordering::SeqCst); + }) + }; + reached("reply.recv_parked", 1).await; // B is parked in recv() + actor.context().stop(); + release.send(()).unwrap(); + tokio::time::timeout(LIMIT * 2, actor.join()).await.unwrap(); + parked.join().unwrap(); + assert_eq!( + a_saw_b.load(Ordering::SeqCst), + 1, + "a parked blocking receiver was woken only after another caller's waker" + ); +} + mod native { use super::*; use spawned_concurrency::threads::{ @@ -1651,4 +1707,57 @@ mod native { "an unread reply was destroyed before every caller was answered" ); } + + /// Threads mode of `parked_receivers_wake_before_any_waker`. + #[test] + fn parked_receivers_wake_before_any_waker() { + use std::task::{Wake, Waker}; + struct WaitForB(Arc, Arc); + impl Wake for WaitForB { + fn wake(self: Arc) { + let began = Instant::now(); + while began.elapsed() < Duration::from_secs(3) { + if self.0.load(Ordering::SeqCst) == 1 { + self.1.store(1, Ordering::SeqCst); + return; + } + std::thread::sleep(Duration::from_millis(1)); + } + } + } + let _scenario = FailScenario::setup(); + let actor = NativeDischarger.start(); + let (release, gate) = std::sync::mpsc::channel(); + let (entered_tx, entered) = std::sync::mpsc::channel(); + actor.send(Gate(gate, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + let never = spawned_concurrency::Deadline::Never; + let a = actor.send_request(Answer, never).unwrap(); + let b = actor.send_request(Answer, never).unwrap(); + let (b_returned, a_saw_b) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0))); + let waker = Waker::from(Arc::new(WaitForB(b_returned.clone(), a_saw_b.clone()))); + let mut a = Box::pin(std::future::IntoFuture::into_future(a)); + assert!( + std::future::Future::poll(a.as_mut(), &mut std::task::Context::from_waker(&waker)) + .is_pending() + ); + failpoints::configure("reply.recv_parked", FailPoint::new(Action::Off)).unwrap(); + let parked = { + let b_returned = b_returned.clone(); + std::thread::spawn(move || { + let _ = b.recv(); + b_returned.store(1, Ordering::SeqCst); + }) + }; + wait_hits("reply.recv_parked"); + actor.context().stop(); + release.send(()).unwrap(); + actor.join(); + parked.join().unwrap(); + assert_eq!( + a_saw_b.load(Ordering::SeqCst), + 1, + "a parked blocking receiver was woken only after another caller's waker" + ); + } } diff --git a/concurrency/tests/job_core_loom.rs b/concurrency/tests/job_core_loom.rs index f7e1c704..41f69bd0 100644 --- a/concurrency/tests/job_core_loom.rs +++ b/concurrency/tests/job_core_loom.rs @@ -22,6 +22,7 @@ mod sync { } pub(crate) fn left_open(_: &SlotOwner, _: usize, _: &'static str, _: bool) {} pub(crate) fn shortened(_: &SlotOwner, _: usize) {} + pub(crate) fn parked() {} pub(crate) use super::model_arc::{JobArc, JobWeak}; /// Loom has no timed wait; the model never parks on a slot. diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index 2231bff5..88d6de03 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -17,6 +17,7 @@ mod sync { } pub(crate) fn left_open(_: &SlotOwner, _: usize, _: &'static str, _: bool) {} pub(crate) fn shortened(_: &SlotOwner, _: usize) {} + pub(crate) fn parked() {} /// Loom has no timed wait; tests park only on slots without a deadline. pub(crate) fn wait_ticks<'a, T>( diff --git a/docs/TESTING.md b/docs/TESTING.md index ff085c1c..3b9103f1 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -1497,6 +1497,7 @@ and prints the seed when a test fails. Configuration is process-global. | `link.before_install` | no | Peer exiting while a link is installed (tasks and threads) | | `monitor.before_install` | no | Target exiting while a monitor is installed (tasks and threads) | | `reply.close_upgraded` | no | The death point's close holding an already-answered slot while its caller drops the receiver: the unread reply is destroyed only after every caller is answered (tasks and threads) | +| `reply.recv_parked` | no | Handshake: a blocking `recv()` is parked, so only a notification wakes it (discharge notifies parked receivers before any waker; tasks and threads) | | `monitor.down_before_dispatch` | no | `demonitor` on another thread returning while the monitor's `Down` is in dispatch: it is flushed (tasks and threads) | `tests/failpoints.rs` drives every site and checks invariants through public diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index bab7b548..b22c45b0 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -402,8 +402,9 @@ current turn's slot) and a job set. The order is: Steps 1 to 4 run no user code: steps 3 and 4 only make terminal transitions, and every slot they meet stays referenced (an unread reply's value is user code when destroyed) until step 5. After the last - answer, and before step 5, run in this order: the callers' wake-ups - (executor or user code), a runtime drop's "stopped() + answer, and before step 5, run in this order: the parked blocking + receivers are notified (no user code, so none waits behind a waker), the + callers' wakers (executor or user code), a runtime drop's "stopped() skipped" log, and the callbacks the jobs' cancellation owes (`cancelled()` waiters, blocked emitters). Cancellation in step 1 only sets the jobs' flag: the payloads it releases (held results, scheduled emits) are From 04e342e008fa75a5627b58d283c908b917700c4c Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:27:24 -0300 Subject: [PATCH 179/208] docs: MIGRATION's Receiver and metrics::Snapshot guidance matches the sealed and non_exhaustive API; send_request is OTP's send_request/2 plus a deadline --- CHANGELOG.md | 2 +- concurrency/src/tasks/actor.rs | 6 +++--- concurrency/src/threads/actor.rs | 6 +++--- docs/GUIDE.md | 2 +- docs/MIGRATION.md | 19 ++++++++++++------- docs/design/deferred-replies.md | 2 +- 6 files changed, 21 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c892db34..d8115fe6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ `default_max_native_jobs` are no longer public. - Requests and deadlines: `request_raw` and `request_raw_until` become `send_request(msg, impl Into)` (OTP's - `gen_server:send_request/3`; on `Receiver`, a `Deadline`). 0.7's + `gen_server:send_request/2` with an explicit deadline; on `Receiver`, a `Deadline`). 0.7's `request_raw` had no deadline: `Deadline::Never` keeps that. `PendingReply::deadline`, `ReplyTo::deadline` and `ReplyReceiver::deadline` return `Deadline`, so a deadline forwards as is; diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 990d8ba0..8ceaeb45 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -716,7 +716,7 @@ impl Context { ) } - /// Send a request without waiting (`gen_server:send_request/3`) and get + /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline) and get /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) @@ -1156,7 +1156,7 @@ where /// spawned (a method can then be added without a breaking change). pub trait Receiver: Send + Sync + crate::__private::Sealed { fn send(&self, msg: M) -> Result<(), ActorError>; - /// Send a request without waiting (`gen_server:send_request/3`); its + /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline); its /// deadline is fixed now (the slot carries it). On a [`Recipient`], /// pass a `Deadline` (`Deadline::after(d)`, or `d.into()`). fn send_request( @@ -1256,7 +1256,7 @@ impl ActorRef { ) } - /// Send a request without waiting (`gen_server:send_request/3`) and get + /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline) and get /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index da2c6429..373a4849 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -589,7 +589,7 @@ impl Context { ) } - /// Send a request without waiting (`gen_server:send_request/3`) and get + /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline) and get /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) @@ -949,7 +949,7 @@ where /// spawned (a method can then be added without a breaking change). pub trait Receiver: Send + Sync + crate::__private::Sealed { fn send(&self, msg: M) -> Result<(), ActorError>; - /// Send a request without waiting (`gen_server:send_request/3`); its + /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline); its /// deadline is fixed now (the slot carries it). On a [`Recipient`], /// pass a `Deadline` (`Deadline::after(d)`, or `d.into()`). fn send_request( @@ -1051,7 +1051,7 @@ impl ActorRef { ) } - /// Send a request without waiting (`gen_server:send_request/3`) and get + /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline) and get /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 5993d0e7..213869a3 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -159,7 +159,7 @@ State that must be released on every exit, including kills and runtime drops, be | `ctx.send(msg)` | Send a fire-and-forget message to this actor (self-send) | | `ctx.request(msg)` | Send a request and wait for the reply (tasks: async, threads: blocking) | | `ctx.request_with_timeout(msg, duration)` | Send a request with a custom timeout | -| `ctx.send_request(msg, deadline)` | Send a request without waiting (`gen_server:send_request/3`) and get its `ReplyReceiver` (tasks: a future, threads: `recv()`/`try_recv()`) yielding `Result`; the deadline (a `Duration`, an `Instant` or `Deadline::Never`) is fixed at send | +| `ctx.send_request(msg, deadline)` | Send a request without waiting (`gen_server:send_request/2` with an explicit deadline) and get its `ReplyReceiver` (tasks: a future, threads: `recv()`/`try_recv()`) yielding `Result`; the deadline (a `Duration`, an `Instant` or `Deadline::Never`) is fixed at send | | `ctx.recipient::()` | Get a type-erased `Recipient` for this actor | | `ctx.actor_ref()` | Get an `ActorRef` from the context | | `ctx.id()` | This actor's `ActorId` | diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 8e1be472..e3cb0207 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -33,7 +33,7 @@ every change; this guide covers what you need to edit when upgrading. | `actor.request_raw(msg)` (0.7: no deadline) | `actor.send_request(msg, Deadline::Never)` (or a `Duration` / `Instant`) | | `actor.request_raw_until(msg, deadline)` *(0.8 pre-release)* | `actor.send_request(msg, deadline)` | | `Response::from_with_timeout(actor.request_raw(msg), t)` | `Response::from(actor.send_request(msg, t))` | -| `impl Receiver`: `fn request_raw(&self, msg)` | `fn send_request(&self, msg, deadline: Deadline)` | +| a `Recipient` call `recipient.request_raw(msg)` | `recipient.send_request(msg, deadline)` (`Deadline`); implementing `Receiver` is no longer possible (sealed, see below) | | `reply.deadline()` / `token.deadline()` / `rx.deadline()` returning `Option` *(0.8 pre-release)* | return `Deadline` (`Deadline::Never` for none); forward with `other.send_request(msg, token.deadline())` | | `token.with_deadline(at)` *(0.8 pre-release)* | `token.shorten_deadline(at)` (a `Duration` or `Instant`) | | `reply.has_caller()` *(0.8 pre-release)* | removed: `reply.defer()` returns `Err(DeferError::NoCaller)` for a send | @@ -238,21 +238,26 @@ a test, use `spawned_test::reply_pair()`. **Timeouts.** The request timeout starts at send, not at first poll. **Raw API.** `request_raw(msg)` is now `send_request(msg, deadline)` -(OTP's `gen_server:send_request/3`). It takes the deadline explicitly: a +(OTP's `gen_server:send_request/2`, plus an explicit deadline; OTP's +`/4` form takes a label and a request-id collection instead). It takes the deadline explicitly: a `Duration` from now, an `Instant`, or `Deadline::Never`. **0.7's `request_raw` had no deadline**: `send_request(msg, Deadline::Never)` keeps that unbounded wait, and `send_request(msg, DEFAULT_REQUEST_TIMEOUT)` gives the 5 s `request` uses. It returns a `ReplyReceiver` (a future in tasks mode, `recv()` in threads mode) yielding `Result`. `Response::from_with_timeout` is removed: `Response::from(actor.send_request(msg, t))`. -`Receiver` implementors implement `send_request(msg, Deadline)`. A -`recv_timeout` on the old native receiver has no counterpart: the deadline -is fixed when the request is sent. +`Receiver` is sealed: call `send_request(msg, Deadline)` through a +`Recipient`; a type of your own cannot implement it (wrap an `ActorRef` +or use `spawned_test`'s probe instead). A `recv_timeout` on the old native +receiver has no counterpart: the deadline is fixed when the request is +sent. **Metrics.** `metrics::Snapshot` gains `requests_reply_dropped_total` and `requests_overloaded_total`, the `deferred_*` and `jobs_*` fields (see the -GUIDE's Metrics section) and the `deferred_reply_latency` histogram. Code -that builds a `Snapshot` with a struct literal must add them. +GUIDE's Metrics section) and the `deferred_reply_latency` histogram. It is +`#[non_exhaustive]` and read-only: take one from `metrics::snapshot()` and +read its fields; a struct literal or an exhaustive pattern no longer +compiles (patterns need `..`). **Jobs (API refinements).** diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index b22c45b0..7f1a47da 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -172,7 +172,7 @@ There is one timeout reason, as in OTP. `DeadlineExceeded` exists only on | Today | Proposed | |---|---| -| `request_raw(msg) -> Result, _>`, no timeout | `send_request(msg, impl Into) -> Result, _>` (OTP's `gen_server:send_request/3`): a `Duration`, an `Instant` or `Deadline::Never` | +| `request_raw(msg) -> Result, _>`, no timeout | `send_request(msg, impl Into) -> Result, _>` (OTP's `gen_server:send_request/2` with an explicit deadline): a `Duration`, an `Instant` or `Deadline::Never` | | `Receiver::request_raw` | `send_request(msg, Deadline)` (required) | | erased `request(recipient, msg, timeout)` | computes the deadline before enqueueing, then `send_request` | | `Response::from(Result, _>)`, `Response::from_with_timeout` | `Response::from(Result, _>)`; `from_with_timeout` removed (the deadline is already in the slot) | From ca3b3d967f359d1bb468e1ead43b8006808e8d6d Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:27:49 -0300 Subject: [PATCH 180/208] docs(jobs): an async job's work is a factory run in the current turn; only its future is the job --- CHANGELOG.md | 3 +++ concurrency/src/tasks/jobs.rs | 15 +++++++++++++-- docs/GUIDE.md | 6 ++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8115fe6..e075d773 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +- Docs: an async job's `work` is a factory run at once in the current turn + (its panic is a handler panic; a job refused by the owner's exit has run + it); only its future is the job (`pipe` rustdoc, GUIDE). - **Behaviour (0.8)**: `demonitor` flushes, as OTP's `demonitor(Ref, [flush])`: once it returns, in both modes, no `Down` of that monitor is enqueued or handled, including one already in the mailbox. Framework `Down`s travel in diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index 7419a1e1..ba35f760 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -816,8 +816,19 @@ impl Context { /// `Task.async` + `handle_info` would. `work` receives the job's /// [`JobCtx`] (to check cancellation and `emit` fenced messages, bounded /// by `emit_credits`; the terminal message is always last) and returns - /// the future to run; `|_| fut` when it needs neither. A panic arrives - /// as `Err(JobError::Panicked)`. Holds one `MAX_OUTSTANDING` credit + /// the future to run; `|_| fut` when it needs neither. A panic in the + /// future arrives as `Err(JobError::Panicked)`. + /// + /// `work` itself is a factory that runs **at once, in this turn, on the + /// calling thread**: only the future it returns is the job. Code before + /// its `async` block runs in the handler, so a panic there is a handler + /// panic (the actor's panic boundary, not `JobError::Panicked`), and if + /// the owner's exit closes admission meanwhile the job is refused after + /// the factory ran (its future is dropped unpolled). Put fallible or slow + /// preparation inside the `async` block. (`pipe_blocking` runs its + /// closure on the job's thread, under the job's panic boundary.) + /// + /// Holds one `MAX_OUTSTANDING` credit /// while it executes. Rejects at once at a bound; never waits. On `Err`, /// `work` and `done` are dropped: use [`reserve_job`](Self::reserve_job) /// to keep them. diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 213869a3..cb5131cd 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -716,6 +716,12 @@ ctx.pipe_reply(JobOptions::new(), token, |_| async move { downstream?.await }, | `.emit_credits(n)` and, for `pipe_reply*` only, `.cancel_on_caller_gone()`. The work receives the job's `JobCtx` by value (`|_| fut` when it needs neither cancellation nor emits). +- For async jobs (`pipe`, `pipe_reply`), `work` is a factory that runs at + once, in the current turn: only the future it returns is the job. A panic + before its `async` block is a handler panic, not `JobError::Panicked`, + and a job the owner's exit refuses meanwhile has already run its factory. + Keep preparation inside the `async` block. Blocking jobs run their whole + closure on the job's thread, under the job's panic boundary. - `done(outcome)` runs in the owner's turn. `JobOutcome` is `Result`; `Err` carries `JobError::Panicked` or `JobError::DeadlineExceeded`. `JobError` and `PipeError` are From 6dab1bffc52cb2a5b2224ee4a4bd89a65af60df3 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:30:44 -0300 Subject: [PATCH 181/208] test: handshakes for the held-result and unused-native-permit regressions Final review (Codex P2 4): - death_boundary job_payloads_are_destroyed_after_every_caller_is_answered (held case) now waits until the job's execution is over (JobHandle::is_running false) before stopping the owner: JobCore::finish stores the held result before execution is marked over, so the exit provably meets a held terminal, not a still-running job. - New site `job.native_worker_unused` (threads mode: the parked thread of a dropped unused permit, before it exits) and failpoints::native::an_unused_permit_returns_its_slot_without_its_thread: with that thread held at its exit, the next reservation at MAX_NATIVE_JOBS = 1 must succeed at once. The 2,000-iteration test stays as a stress test. --- concurrency/src/failpoints.rs | 7 ++++ concurrency/src/threads/jobs.rs | 4 +++ concurrency/tests/death_boundary.rs | 50 ++++++++++++++++++----------- concurrency/tests/failpoints.rs | 32 ++++++++++++++++++ docs/TESTING.md | 1 + 5 files changed, 75 insertions(+), 19 deletions(-) diff --git a/concurrency/src/failpoints.rs b/concurrency/src/failpoints.rs index 4f3d2367..9a13379c 100644 --- a/concurrency/src/failpoints.rs +++ b/concurrency/src/failpoints.rs @@ -71,6 +71,13 @@ pub const SITES: &[Site] = &[ Delay models slow cleanup (shutdown deadlines); panic models a \ failing `stopped()`.", }, + Site { + name: "job.native_worker_unused", + has_error: false, + description: "Threads mode: the parked thread of a dropped, unused blocking-job \ + permit, before it exits. Delay shows the native slot is returned \ + by the drop itself, not when this thread ends.", + }, Site { name: "reply.close_upgraded", has_error: false, diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index ba678cd6..9798d2c1 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -426,6 +426,10 @@ impl Context { .spawn(move || { if let Ok(task) = started.recv() { task(); + } else { + // An unused permit was dropped: this thread only exits + // (the slot was already returned on the dropping thread). + fail_point!("job.native_worker_unused"); } }) .map_err(PipeError::SpawnFailed)?; diff --git a/concurrency/tests/death_boundary.rs b/concurrency/tests/death_boundary.rs index 8afd0aa6..7d10bf46 100644 --- a/concurrency/tests/death_boundary.rs +++ b/concurrency/tests/death_boundary.rs @@ -180,33 +180,45 @@ mod tasks_mode { answered: answered.clone(), }; let (scheduled_tx, scheduled) = mpsc::channel(); + let (handle_tx, handle) = mpsc::channel(); runtime.block_on(async { spawned_concurrency::__private::in_turn(&actor, move |_, ctx| { - ctx.pipe( - JobOptions::new(), - move |job| async move { - let far = spawned_concurrency::Instant::now() + LIMIT * 10; - if held { - // The result waits for this emit: held. - job.emit_at(far, Queued(inert())).await.unwrap(); - scheduled_tx.send(()).unwrap(); - Some(payload) - } else { - job.emit_at(far, Waiting(payload)).await.unwrap(); - scheduled_tx.send(()).unwrap(); - job.cancelled().await; - None - } - }, - |_| Queued(inert()), - ) - .unwrap(); + let job = ctx + .pipe( + JobOptions::new(), + move |job| async move { + let far = spawned_concurrency::Instant::now() + LIMIT * 10; + if held { + // The result waits for this emit: held. + job.emit_at(far, Queued(inert())).await.unwrap(); + scheduled_tx.send(()).unwrap(); + Some(payload) + } else { + job.emit_at(far, Waiting(payload)).await.unwrap(); + scheduled_tx.send(()).unwrap(); + job.cancelled().await; + None + } + }, + |_| Queued(inert()), + ) + .unwrap(); + handle_tx.send(job).unwrap(); }) .unwrap() .await .unwrap(); }); scheduled.recv_timeout(LIMIT).unwrap(); + if held { + // Handshake: the work returned with its emit still + // scheduled, so its result is held (`JobCore::finish` stores + // it before execution is marked over). + let job = handle.recv_timeout(LIMIT).unwrap(); + eventually("the held job finished executing", || { + (!job.is_running()).then_some(()) + }); + } let (unblock, blocked) = mpsc::channel(); let (entered_tx, entered) = mpsc::channel(); actor.send(Block(blocked, entered_tx)).unwrap(); diff --git a/concurrency/tests/failpoints.rs b/concurrency/tests/failpoints.rs index 0d24a001..ff0dd608 100644 --- a/concurrency/tests/failpoints.rs +++ b/concurrency/tests/failpoints.rs @@ -1760,4 +1760,36 @@ mod native { "a parked blocking receiver was woken only after another caller's waker" ); } + + /// Final review (Codex P2 4): dropping an unused blocking-job permit + /// returns its native slot on the dropping thread. With the parked + /// thread held at its exit (`job.native_worker_unused`), the next + /// reservation at `MAX_NATIVE_JOBS = 1` must still succeed; the slot + /// used to be released only when that thread ended. + #[test] + fn an_unused_permit_returns_its_slot_without_its_thread() { + struct One; + impl NativeActor for One { + const MAX_NATIVE_JOBS: Option = Some(1); + } + let _scenario = FailScenario::setup(); + failpoints::configure( + "job.native_worker_unused", + FailPoint::new(Action::Delay(Duration::from_millis(500))).times(1), + ) + .unwrap(); + let owner = One.start(); + let ctx = owner.context(); + let opts = spawned_concurrency::threads::JobOptions::new; + drop(ctx.reserve_blocking_job(opts()).unwrap()); + wait_hits("job.native_worker_unused"); // the old thread is held + let again = ctx.reserve_blocking_job(opts()); + assert!( + again.is_ok(), + "the slot waited for the unused permit's thread: {again:?}" + ); + drop(again); + ctx.stop(); + owner.join(); + } } diff --git a/docs/TESTING.md b/docs/TESTING.md index 3b9103f1..54b99bc6 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -1496,6 +1496,7 @@ and prints the seed when a test fails. Configuration is process-global. | `actor.exit_before_deliver` | no | A cloned context's exit signal racing its sender's death (tasks and threads) | | `link.before_install` | no | Peer exiting while a link is installed (tasks and threads) | | `monitor.before_install` | no | Target exiting while a monitor is installed (tasks and threads) | +| `job.native_worker_unused` | no | Threads mode: the parked thread of a dropped, unused blocking-job permit held at its exit; the native slot is already back (`an_unused_permit_returns_its_slot_without_its_thread`) | | `reply.close_upgraded` | no | The death point's close holding an already-answered slot while its caller drops the receiver: the unread reply is destroyed only after every caller is answered (tasks and threads) | | `reply.recv_parked` | no | Handshake: a blocking `recv()` is parked, so only a notification wakes it (discharge notifies parked receivers before any waker; tasks and threads) | | `monitor.down_before_dispatch` | no | `demonitor` on another thread returning while the monitor's `Down` is in dispatch: it is flushed (tasks and threads) | From 49f7833a019bacfc434db3dba36e47879e4423fe Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:38:12 -0300 Subject: [PATCH 182/208] fix(jobs): payloads dropped for an exiting owner are held until every caller is answered Final review (agy P1 1, both modes): a scheduled emit_at in a cancelled job's exit_released could still come due (during stopped(), or early in reach_death) on the deadline service, which refused it and dropped its user payload there, before the death point answered the callers. Other framework drops for the owner (a job's refused outcome, an unsent emit) had the same window. The owner's drop sink (mailbox::DropFailures, which already contains every such drop) gains a holding mode: hold() at the start of the exit, right before the jobs' cancellation (tasks and threads run loops, and ActorCleanup::reach_death for drops and panics); while holding, discard moves the value into the sink instead of dropping it, on whatever thread; ActorCleanup::discharged() releases (drops, contained) after the reply wake-ups and dispose_cancelled. The mailbox's own items use discard_now (they are destroyed after discharge anyway). Contained and the job Run gain 'static bounds every constructor already required. Regression (fails without the hold in both modes): failpoints emit_due_during_exit_is_destroyed_after_every_caller_is_answered (tasks and native): new site `job.emit_due` marks the emit coming due while stopped() is held; the payload's destructor records whether the queued caller was answered when it started. --- concurrency/src/actor_cleanup.rs | 9 ++ concurrency/src/failpoints.rs | 8 + concurrency/src/mailbox.rs | 57 +++++-- concurrency/src/reply/jobs.rs | 7 +- concurrency/src/tasks/actor.rs | 2 + concurrency/src/tasks/jobs.rs | 4 +- concurrency/src/threads/actor.rs | 2 + concurrency/src/threads/jobs.rs | 4 +- concurrency/tests/failpoints.rs | 246 +++++++++++++++++++++++++++++++ docs/TESTING.md | 1 + docs/design/deferred-replies.md | 5 +- 11 files changed, 326 insertions(+), 19 deletions(-) diff --git a/concurrency/src/actor_cleanup.rs b/concurrency/src/actor_cleanup.rs index 38b17dfe..e0f06f5a 100644 --- a/concurrency/src/actor_cleanup.rs +++ b/concurrency/src/actor_cleanup.rs @@ -134,6 +134,12 @@ impl ActorCleanup { if let Some(replies) = &self.replies { replies.exiting(); } + // From here until every caller is answered, the payloads framework + // code drops for this owner (a scheduled emit expiring, a job's + // refused outcome) are held, on whatever thread drops them. + if let Some(failures) = &self.failures { + failures.hold(); + } if let Some(jobs) = &self.jobs { jobs.cancel_all(); } @@ -208,6 +214,9 @@ impl ActorCleanup { if let Some(jobs) = &self.jobs { jobs.dispose_cancelled(); } + if let Some(failures) = &self.failures { + failures.release(); + } } fn close(&mut self, reason: ExitReason) -> ExitReason { let replies = self.replies.as_deref(); diff --git a/concurrency/src/failpoints.rs b/concurrency/src/failpoints.rs index 9a13379c..ba32a6dd 100644 --- a/concurrency/src/failpoints.rs +++ b/concurrency/src/failpoints.rs @@ -71,6 +71,14 @@ pub const SITES: &[Site] = &[ Delay models slow cleanup (shutdown deadlines); panic models a \ failing `stopped()`.", }, + Site { + name: "job.emit_due", + has_error: false, + description: "A scheduled `emit_at` comes due on the deadline service. A \ + handshake: one coming due during the owner's exit (`stopped()`) is \ + refused, and its payload is destroyed only after every caller is \ + answered (tasks and threads).", + }, Site { name: "job.native_worker_unused", has_error: false, diff --git a/concurrency/src/mailbox.rs b/concurrency/src/mailbox.rs index 89667e75..dc1ba219 100644 --- a/concurrency/src/mailbox.rs +++ b/concurrency/src/mailbox.rs @@ -7,26 +7,61 @@ use std::{ }, }; +/// An owner's sink for the user payloads its framework code drops (refused +/// or undelivered messages, job outcomes, scheduled emits): each drop is +/// contained, its first failure recorded. From the start of the owner's +/// exit until its death point has answered every caller, drops are held +/// instead ([`hold`](Self::hold), [`release`](Self::release)): no user +/// destructor runs before the last answer, on whatever thread drops it. #[derive(Default)] -pub(crate) struct DropFailures(Mutex>); +pub(crate) struct DropFailures { + first: Mutex>, + held: Mutex>>>, +} impl DropFailures { pub(crate) fn first(&self) -> Option { - self.0.lock().unwrap_or_else(|p| p.into_inner()).clone() + self.first.lock().unwrap_or_else(|p| p.into_inner()).clone() } /// Record a cleanup failure; the first one is kept. pub(crate) fn record(&self, message: String) { - self.0 + self.first .lock() .unwrap_or_else(|p| p.into_inner()) .get_or_insert(message); } - pub(crate) fn discard(&self, value: T) { + /// The owner's exit began: hold every later drop until `release`. + pub(crate) fn hold(&self) { + self.held + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get_or_insert_with(Vec::new); + } + /// Every caller was answered: drop what was held (contained), and drop + /// at once from now on. Idempotent. + pub(crate) fn release(&self) { + let held = self.held.lock().unwrap_or_else(|p| p.into_inner()).take(); + for value in held.into_iter().flatten() { + self.drop_contained(value); + } + } + pub(crate) fn discard(&self, value: T) { + let mut held = self.held.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(held) = held.as_mut() { + // Moved, not dropped: no user code under the lock. + held.push(Box::new(value)); + return; + } + drop(held); + self.drop_contained(value); + } + /// Drop at once (contained): the mailbox's own items, destroyed after + /// the death point's discharge. + pub(crate) fn discard_now(&self, value: T) { + self.drop_contained(value); + } + fn drop_contained(&self, value: T) { if let Err(payload) = catch_unwind(AssertUnwindSafe(|| drop(value))) { - let message = crate::error::panic_message(&*payload); - self.0 - .lock() - .unwrap_or_else(|p| p.into_inner()) - .get_or_insert(message); + self.record(crate::error::panic_message(&*payload)); dispose_panic(payload); } } @@ -126,7 +161,7 @@ pub(crate) mod tasks { /// Destroy drained items; a destructor panic is a cleanup failure. pub(crate) fn discard_all(&self, items: Vec) { for item in items { - self.failures.discard(item); + self.failures.discard_now(item); } } } @@ -209,7 +244,7 @@ pub(crate) mod threads { /// Destroy drained items; a destructor panic is a cleanup failure. pub(crate) fn discard_all(&self, items: Vec) { for item in items { - self.admission.failures.discard(item); + self.admission.failures.discard_now(item); } } } diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index aa4da024..5f24fde5 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -926,12 +926,12 @@ impl Expirable for JobDeadline { /// A value (user payload) whose drop is contained by the owner's failure /// sink: it may be dropped on the deadline service thread. -pub(crate) struct Contained { +pub(crate) struct Contained { value: Option, failures: Arc, } -impl Contained { +impl Contained { pub(crate) fn new(value: T, failures: Arc) -> Self { Self { value: Some(value), @@ -943,7 +943,7 @@ impl Contained { } } -impl Drop for Contained { +impl Drop for Contained { fn drop(&mut self) { if let Some(value) = self.value.take() { self.failures.discard(value); @@ -1033,6 +1033,7 @@ impl Expirable for ScheduledEmit { lock(&self.send).is_some().then_some(self.at) } fn expire(&self, _now: u64) { + fail_point!("job.emit_due"); if let Some(job) = self.job.upgrade() { self.settle(&job, true); } diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 8ceaeb45..afd96b44 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -2094,6 +2094,8 @@ async fn run_actor_lifecycle( } cancellation_token.cancel(); ctx.shared.replies.exiting(); + // Payload drops are held until every caller is answered (DropFailures). + ctx.shared.sender.failures().hold(); ctx.shared.jobs.cancel_all(); // stopped() is skipped when a kill, an untrapped exit signal or link // notice began the exit (decision 2); one that already began finishes. diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index ba35f760..f0dfb4fc 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -117,7 +117,7 @@ where // --------------------------------------------------------------------------- /// The parts shared by a job's completion path and its deadline. -struct Run { +struct Run { state: Arc, ctx: Context, done: Mutex>>, @@ -200,7 +200,7 @@ where } } -impl Drop for Run { +impl Drop for Run { /// The last reference may go on the deadline service thread (a deadline /// or a held result firing): `done` and its captures drop contained. fn drop(&mut self) { diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 373a4849..853852f6 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -1467,6 +1467,8 @@ fn run_actor_receiving( cancellation_token.cancel(); ctx.replies.exiting(); + // Payload drops are held until every caller is answered (DropFailures). + ctx.sender.failures().hold(); ctx.jobs.cancel_all(); // stopped() is skipped when a kill, an untrapped exit signal or link // notice began the exit (decision 2); one that already began finishes. diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index 9798d2c1..cc6da207 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -95,7 +95,7 @@ where // --------------------------------------------------------------------------- /// The parts shared by a job's completion path and its deadline. -struct Run { +struct Run { state: Arc, ctx: Context, done: Mutex>>, @@ -182,7 +182,7 @@ where } } -impl Drop for Run { +impl Drop for Run { /// The last reference may go on the deadline service thread (a deadline /// or a held result firing): `done` and its captures drop contained. fn drop(&mut self) { diff --git a/concurrency/tests/failpoints.rs b/concurrency/tests/failpoints.rs index ff0dd608..6d54518e 100644 --- a/concurrency/tests/failpoints.rs +++ b/concurrency/tests/failpoints.rs @@ -1341,6 +1341,147 @@ async fn parked_receivers_wake_before_any_waker() { ); } +// ------------------------------- job.emit_due (exit holds payload drops) + +/// A message whose destructor records, when it starts, whether a queued +/// caller was already answered. +struct Probe { + caller: SharedReceiver, + answered: Arc, + started: std::sync::mpsc::Sender<()>, +} +impl Drop for Probe { + fn drop(&mut self) { + if let Some(receiver) = self.caller.lock().unwrap().as_mut() { + if receiver.try_recv().is_some() { + self.answered.store(1, Ordering::SeqCst); + } + } + let _ = self.started.send(()); + } +} +struct Probed(#[allow(dead_code)] Probe); +impl Message for Probed { + type Result = (); +} +/// Its `stopped()` waits for the test's release. +struct SlowStop(std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>); +struct ScheduleProbe(Probe, std::sync::mpsc::Sender<()>); +impl Message for ScheduleProbe { + type Result = (); +} +impl Actor for SlowStop { + async fn stopped(&mut self, _: &Context) { + let _ = self.1.send(()); + let _ = self.0.recv_timeout(LIMIT); + } +} +impl Handler for SlowStop { + async fn handle( + &mut self, + msg: ScheduleProbe, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + let ScheduleProbe(probe, scheduled) = msg; + ctx.pipe( + spawned_concurrency::tasks::JobOptions::new(), + move |job| async move { + let at = spawned_concurrency::Instant::now() + Duration::from_millis(100); + job.emit_at(at, Probed(probe)).await.unwrap(); + let _ = scheduled.send(()); + job.cancelled().await; + }, + |_| Answer, + ) + .unwrap(); + reply.reply(()); + } +} +impl Handler for SlowStop { + async fn handle( + &mut self, + msg: Gate, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let _ = msg.1.send(()); + let _ = msg.0.recv_timeout(LIMIT); + reply.reply(()); + } +} +impl Handler for SlowStop { + async fn handle( + &mut self, + _: Answer, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } +} +impl Handler for SlowStop { + async fn handle( + &mut self, + _: Probed, + _: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + } +} + +/// Final review (agy P1 1): a scheduled `emit_at` that comes due while the +/// owner runs `stopped()` (its jobs already cancelled) is refused, and its +/// payload used to be destroyed right there on the deadline service, before +/// the queued caller was answered. The owner's payload drops are now held +/// from the start of its exit until every caller is answered. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn emit_due_during_exit_is_destroyed_after_every_caller_is_answered() { + let _scenario = FailScenario::setup(); + let (stop_release, stop_gate) = std::sync::mpsc::channel(); + let (stopping_tx, stopping) = std::sync::mpsc::channel(); + let actor = SlowStop(stop_gate, stopping_tx).start(); + let caller: SharedReceiver = Arc::default(); + let answered = Arc::new(AtomicUsize::new(0)); + let (started_tx, started) = std::sync::mpsc::channel(); + let (scheduled_tx, scheduled) = std::sync::mpsc::channel(); + let probe = Probe { + caller: caller.clone(), + answered: answered.clone(), + started: started_tx, + }; + actor + .request(ScheduleProbe(probe, scheduled_tx)) + .await + .unwrap(); + scheduled.recv_timeout(LIMIT).unwrap(); // the emit is armed + let (release, gate) = std::sync::mpsc::channel(); + let (entered_tx, entered) = std::sync::mpsc::channel(); + actor.send(Gate(gate, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + *caller.lock().unwrap() = Some( + actor + .send_request(Answer, spawned_concurrency::Deadline::Never) + .unwrap(), + ); + failpoints::configure("job.emit_due", FailPoint::new(Action::Off)).unwrap(); + actor.context().stop(); + release.send(()).unwrap(); + stopping.recv_timeout(LIMIT).unwrap(); // in stopped(), jobs cancelled + // The emit came due during stopped(). Its payload must not be destroyed + // now: give a wrong implementation the chance to, then let stopped() + // return. + reached("job.emit_due", 1).await; + let _ = started.recv_timeout(Duration::from_millis(500)); + stop_release.send(()).unwrap(); + tokio::time::timeout(LIMIT * 2, actor.join()).await.unwrap(); + assert_eq!( + answered.load(Ordering::SeqCst), + 1, + "a scheduled emit's payload was destroyed before every caller was answered" + ); +} + mod native { use super::*; use spawned_concurrency::threads::{ @@ -1792,4 +1933,109 @@ mod native { ctx.stop(); owner.join(); } + + struct NativeSlowStop(std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>); + impl NativeActor for NativeSlowStop { + fn stopped(&mut self, _: &NativeContext) { + let _ = self.1.send(()); + let _ = self.0.recv_timeout(LIMIT); + } + } + impl NativeHandler for NativeSlowStop { + fn handle( + &mut self, + msg: ScheduleProbe, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &NativeContext, + ) { + let ScheduleProbe(probe, scheduled) = msg; + ctx.pipe_blocking( + spawned_concurrency::threads::JobOptions::new(), + move |job| { + let at = spawned_concurrency::Instant::now() + Duration::from_millis(100); + job.emit_at_blocking(at, Probed(probe)).unwrap(); + let _ = scheduled.send(()); + while !job.is_cancelled() { + std::thread::sleep(Duration::from_millis(2)); + } + }, + |_| Answer, + ) + .unwrap(); + reply.reply(()); + } + } + impl NativeHandler for NativeSlowStop { + fn handle( + &mut self, + msg: Gate, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &NativeContext, + ) { + let _ = msg.1.send(()); + let _ = msg.0.recv_timeout(LIMIT); + reply.reply(()); + } + } + impl NativeHandler for NativeSlowStop { + fn handle( + &mut self, + _: Answer, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &NativeContext, + ) { + reply.reply(()); + } + } + impl NativeHandler for NativeSlowStop { + fn handle( + &mut self, + _: Probed, + _: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &NativeContext, + ) { + } + } + + /// Threads mode of `emit_due_during_exit_is_destroyed_after_every_caller_is_answered`. + #[test] + fn emit_due_during_exit_is_destroyed_after_every_caller_is_answered() { + let _scenario = FailScenario::setup(); + let (stop_release, stop_gate) = std::sync::mpsc::channel(); + let (stopping_tx, stopping) = std::sync::mpsc::channel(); + let actor = NativeSlowStop(stop_gate, stopping_tx).start(); + let caller: SharedReceiver = Arc::default(); + let answered = Arc::new(AtomicUsize::new(0)); + let (started_tx, started) = std::sync::mpsc::channel(); + let (scheduled_tx, scheduled) = std::sync::mpsc::channel(); + let probe = Probe { + caller: caller.clone(), + answered: answered.clone(), + started: started_tx, + }; + actor.request(ScheduleProbe(probe, scheduled_tx)).unwrap(); + scheduled.recv_timeout(LIMIT).unwrap(); + let (release, gate) = std::sync::mpsc::channel(); + let (entered_tx, entered) = std::sync::mpsc::channel(); + actor.send(Gate(gate, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + *caller.lock().unwrap() = Some( + actor + .send_request(Answer, spawned_concurrency::Deadline::Never) + .unwrap(), + ); + failpoints::configure("job.emit_due", FailPoint::new(Action::Off)).unwrap(); + actor.context().stop(); + release.send(()).unwrap(); + stopping.recv_timeout(LIMIT).unwrap(); + wait_hits("job.emit_due"); + let _ = started.recv_timeout(Duration::from_millis(500)); + stop_release.send(()).unwrap(); + actor.join(); + assert_eq!( + answered.load(Ordering::SeqCst), + 1, + "a scheduled emit's payload was destroyed before every caller was answered" + ); + } } diff --git a/docs/TESTING.md b/docs/TESTING.md index 54b99bc6..72505405 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -1496,6 +1496,7 @@ and prints the seed when a test fails. Configuration is process-global. | `actor.exit_before_deliver` | no | A cloned context's exit signal racing its sender's death (tasks and threads) | | `link.before_install` | no | Peer exiting while a link is installed (tasks and threads) | | `monitor.before_install` | no | Target exiting while a monitor is installed (tasks and threads) | +| `job.emit_due` | no | A scheduled `emit_at` coming due during the owner's exit: its payload is held until every caller is answered (`emit_due_during_exit_*`) | | `job.native_worker_unused` | no | Threads mode: the parked thread of a dropped, unused blocking-job permit held at its exit; the native slot is already back (`an_unused_permit_returns_its_slot_without_its_thread`) | | `reply.close_upgraded` | no | The death point's close holding an already-answered slot while its caller drops the receiver: the unread reply is destroyed only after every caller is answered (tasks and threads) | | `reply.recv_parked` | no | Handshake: a blocking `recv()` is parked, so only a notification wakes it (discharge notifies parked receivers before any waker; tasks and threads) | diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index 7f1a47da..184b0fdb 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -408,7 +408,10 @@ current turn's slot) and a job set. The order is: skipped" log, and the callbacks the jobs' cancellation owes (`cancelled()` waiters, blocked emitters). Cancellation in step 1 only sets the jobs' flag: the payloads it releases (held results, scheduled emits) are - destroyed in step 5. Steps 3 and 4 follow the seal instead of being one atomic step + destroyed in step 5. From the start of the exit (the jobs' cancellation) + to the end of discharge, every payload the framework drops for the owner + on any thread (a scheduled emit coming due, a job's refused outcome) is + held by the owner's drop sink and destroyed then too. Steps 3 and 4 follow the seal instead of being one atomic step with it, but nothing between them depends on the requests they answer, so the models' atomic death is a sound abstraction of them (they commute left to the seal). From e65af01a22416b8a5bf02c916e192006758911c1 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:43:12 -0300 Subject: [PATCH 183/208] fix(jobs): a keyed replacement cancels the old job in its Start step and drops its payloads outside the trace section Final review (agy P1 2): JobSetCore::admit cancelled the replaced keyed job (cancel_quietly) before JobSet::admit recorded the new job's Start, inside trace::atomic. So the old job's dropped emits were recorded before that Start (TraceJobs rejects it: in DeferredJobs.tla the old job stops accepting at Start(new), CancelCore(ToReplace)); a linked old job skipped link_cancel, so its caller's JobCancel was missing before the dropped token's Discharge; and its user payloads (held result, scheduled emits) were destroyed inside the process-wide trace section. JobSetCore::admit_replacing hands the replaced job back uncancelled. JobSet::admit records Start, then in the same section sets the old job's cancellation flag, counts it and records link_cancel (cancel_replaced); the suppressed held result and the scheduled emits are dropped after the section, outside every lock. JobSetCore::admit (cancel inside) remains for the Loom model. The model's ordering is unchanged: the code now matches it. Regression (fails without the fix): job_trace keyed_replacement_drops_the_old_emit_after_the_new_start: the old keyed job's emit is registered (handshake), a replacement follows; the recorded Start of the replacement must precede the old emit's dropped Fire (it recorded Start, Schedule, Fire, Start before). The new trace, jobs_keyed_replace_scheduled, is accepted by TraceJobs (13/13). --- concurrency/src/reply/job_core.rs | 25 ++++++++--- concurrency/src/reply/jobs.rs | 39 +++++++++++++--- concurrency/tests/job_trace.rs | 74 +++++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 12 deletions(-) diff --git a/concurrency/src/reply/job_core.rs b/concurrency/src/reply/job_core.rs index 22915583..b503fec6 100644 --- a/concurrency/src/reply/job_core.rs +++ b/concurrency/src/reply/job_core.rs @@ -403,12 +403,29 @@ impl JobSetCore { /// the owner's join counts it. The replaced job and the references /// compaction upgraded are cancelled and dropped after the locks are /// released. + #[allow(dead_code)] // used by the Loom model; the job set orders its own pub(crate) fn admit(&self, state: &Arc, key: Option) -> bool { + match self.admit_replacing(state, key) { + Some(previous) => { + if let Some(previous) = &previous { + previous.cancel(); + } + true + } + None => false, + } + } + + /// [`admit`](Self::admit) that hands the job its key replaced back + /// uncancelled (`Some(None)` when there was none; `None` when refused), + /// so the caller cancels it in the right order and drops its payloads + /// outside its own critical sections. + pub(crate) fn admit_replacing(&self, state: &Arc, key: Option) -> Option>> { let mut upgraded: Vec> = Vec::new(); let previous = { let mut list = lock(&self.jobs); if list.closed { - return false; + return None; } let previous = key.and_then(|key| lock(&self.keys).insert(key, state.clone())); let jobs = &mut list.jobs; @@ -426,12 +443,8 @@ impl JobSetCore { *lock(&self.running) += 1; previous }; - if let Some(previous) = &previous { - previous.cancel(); - } drop(upgraded); - drop(previous); - true + Some(previous) } /// A job's execution ended. Returns whether the set became idle (the diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 5f24fde5..94a1cd9a 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -360,6 +360,19 @@ impl JobState { self.core.finished(); } + /// A keyed replacement cancels this job (inside the trace section, + /// after the new job's `Start`): the flag, its count and a linked + /// caller's `JobCancel`; the suppressed held result comes back to be + /// dropped outside, with [`wake_cancelled`](Self::wake_cancelled). + fn cancel_replaced(&self) -> Option { + let (newly, held) = self.core.cancel_keeping_held(); + if newly { + self.count_cancel(); + self.link_cancel(); + } + held + } + fn cancel_quietly(&self) { if self.core.cancel() { self.count_cancel(); @@ -764,19 +777,33 @@ impl JobSet { /// run it. #[must_use] pub(crate) fn admit(&self, state: &Arc, key: Option, deadline: bool) -> bool { - trace::atomic(|| { + let admitted = trace::atomic(|| { let keyed = key.is_some(); - if !self.core.admit(state, key) { - return false; - } + let previous = self.core.admit_replacing(state, key)?; crate::instrumentation::hooks::job_started(); state.record(|job| trace::JobEvent { keyed, deadline, ..trace::JobEvent::new("Start", job) }); - true - }) + // The replaced job is cancelled in the same step as the new + // job's Start (the model's CancelCore(ToReplace)); a linked + // job's caller sees its JobCancel. Nothing is destroyed here. + Some(previous.map(|previous| { + let held = previous.cancel_replaced(); + (previous, held) + })) + }); + let Some(replaced) = admitted else { + return false; + }; + // Outside the trace section and every lock: the replaced job's + // suppressed result and scheduled emits (user payloads). + if let Some((previous, held)) = replaced { + drop(held); + previous.wake_cancelled(); + } + true } /// A job's execution ended. diff --git a/concurrency/tests/job_trace.rs b/concurrency/tests/job_trace.rs index 971a2554..1a6e9d3e 100644 --- a/concurrency/tests/job_trace.rs +++ b/concurrency/tests/job_trace.rs @@ -249,6 +249,48 @@ impl Handler for Owner { } } +/// A keyed job that schedules an emit far away and waits to be cancelled; +/// `armed` fires once the emit is registered. +struct KeyedScheduled(oneshot::Sender<()>); +impl Message for KeyedScheduled { + type Result = (); +} +impl Handler for Owner { + async fn handle( + &mut self, + msg: KeyedScheduled, + reply: PendingReply<'_, ()>, + ctx: &Context, + ) { + let armed = msg.0; + ctx.pipe( + JobOptions::new().key("k"), + |job: JobCtx| async move { + let at = spawned_concurrency::Instant::now() + Duration::from_secs(60); + job.emit_at(at, Emitted(5)).await.unwrap(); + let _ = armed.send(()); + job.cancelled().await; + 5u32 + }, + done("old_sched"), + ) + .unwrap(); + reply.reply(()); + } +} +/// Replace key `k` now. +struct ReplaceNow; +impl Message for ReplaceNow { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: ReplaceNow, reply: PendingReply<'_, ()>, ctx: &Context) { + ctx.pipe(JobOptions::new().key("k"), |_| async { 2u32 }, done("new")) + .unwrap(); + reply.reply(()); + } +} + /// Cancel the stored handles, then open the gates. struct CancelStored; impl Message for CancelStored { @@ -442,6 +484,38 @@ async fn job_deadline_takes_a_held_result() { ); } +/// Final review (agy P1 2): a keyed replacement cancels the old job in the +/// step of the new job's `Start` (the model's CancelCore(ToReplace)), so +/// the old job's dropped emit is recorded after that `Start`. The +/// replacement used to cancel first, so the drop preceded the `Start` and +/// TraceJobs rejected the trace; the old job's payloads also dropped inside +/// the trace section. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn keyed_replacement_drops_the_old_emit_after_the_new_start() { + let (actor, log) = start(); + let (armed_tx, armed) = oneshot::channel(); + actor.request(KeyedScheduled(armed_tx)).await.unwrap(); + armed.await.unwrap(); // the old job's emit is registered + actor.request(ReplaceNow).await.unwrap(); + wait_for(&log, "new:ok:2").await; + let actions = finish("keyed_replace_scheduled", actor, 16).await; + let second_start = actions + .iter() + .enumerate() + .filter(|(_, a)| *a == "Start") + .nth(1) + .map(|(i, _)| i) + .expect("the replacement's Start"); + let dropped_fire = actions + .iter() + .position(|a| a == "Fire") + .expect("the old job's dropped emit"); + assert!( + second_start < dropped_fire, + "the replaced job's emit dropped before the new Start: {actions:?}" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cancel_drops_a_scheduled_emit() { let (actor, log) = start(); From e9ab240552ffc6d880d5370e14493449e942471a Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:45:32 -0300 Subject: [PATCH 184/208] fix(jobs): a cancelled or deadline-displaced held result drops outside the trace section Final review (agy P2 3): JobState::cancel and JobState::notify (a job deadline overtaking a held result) called JobCore::cancel/notify inside trace::atomic, which dropped the suppressed HeldTerminal (user payload) there, before Cancel/JobCancel or JobDeadline were recorded. JobCore::notify_keeping_held hands the displaced result back (notify wraps it for the other callers); JobState::cancel uses cancel_keeping_held. Both now drop the held result after the trace section, after their events. Regression (fails without the fix): job_trace cancel_drops_a_held_result_outside_the_trace_section: a job's result (a probe) is held behind a far emit (handshake: the work returned); the probe's destructor needs the trace section from another thread within 1 s; cancelling the job used to drop it inside the section. Its trace jobs_cancel_held is accepted (14/14). --- concurrency/src/reply/job_core.rs | 18 +++++-- concurrency/src/reply/jobs.rs | 18 +++++-- concurrency/tests/job_trace.rs | 79 +++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 9 deletions(-) diff --git a/concurrency/src/reply/job_core.rs b/concurrency/src/reply/job_core.rs index b503fec6..d4d3ebfd 100644 --- a/concurrency/src/reply/job_core.rs +++ b/concurrency/src/reply/job_core.rs @@ -141,18 +141,28 @@ impl JobCore { /// caller after the lock is released. When taken, the caller wakes async /// emitters. pub(crate) fn notify(&self, enqueue: impl FnOnce() -> Option) -> (bool, Option) { + let (taken, unsent, held) = self.notify_keeping_held(enqueue); + // A deadline overtaking a held result: the result is dropped. + drop(held); + (taken, unsent) + } + + /// [`notify`](Self::notify) that hands a displaced held result back + /// (user payload) for the caller to drop outside its critical sections. + pub(crate) fn notify_keeping_held( + &self, + enqueue: impl FnOnce() -> Option, + ) -> (bool, Option, Option) { let (unsent, held) = { let mut inner = lock(&self.inner); if !inner.open() { - return (false, None); + return (false, None, None); } inner.notification = Notification::Notified; - // A deadline overtaking a held result: the result is dropped. (enqueue(), inner.held.take()) }; - drop(held); self.emitters.notify_all(); - (true, unsent) + (true, unsent, held) } /// The work's result: take the notification as [`notify`](Self::notify) diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs index 94a1cd9a..f99ca380 100644 --- a/concurrency/src/reply/jobs.rs +++ b/concurrency/src/reply/jobs.rs @@ -344,13 +344,18 @@ impl JobState { /// `Cancel` (the owner's exit and key replacement cancel through the job /// set instead, as part of `Exit` and `Start`). pub(crate) fn cancel(&self) { - trace::atomic(|| { - if self.core.cancel() { + let held = trace::atomic(|| { + let (newly, held) = self.core.cancel_keeping_held(); + if newly { self.count_cancel(); self.record(|job| trace::JobEvent::new("Cancel", job)); self.link_cancel(); } + held }); + // The suppressed held result (user payload): outside the trace + // section, after its Cancel and JobCancel are recorded. + drop(held); self.wake_cancelled(); } @@ -418,8 +423,8 @@ impl JobState { enqueue: impl FnOnce() -> Option, ) -> (bool, Option) { let deadline = end == "deadline"; - let (taken, unsent) = trace::atomic(|| { - let (taken, unsent) = self.core.notify(enqueue); + let (taken, unsent, held) = trace::atomic(|| { + let (taken, unsent, held) = self.core.notify_keeping_held(enqueue); if taken { crate::instrumentation::hooks::job_notified(end); self.link_notified(); @@ -437,8 +442,11 @@ impl JobState { } else if taken { self.record(|job| trace::JobEvent::new("JobDeadline", job)); } - (taken, unsent) + (taken, unsent, held) }); + // A held result the deadline displaced (user payload): dropped + // outside the trace section, after JobDeadline is recorded. + drop(held); #[cfg(feature = "async")] if taken { self.wake_emitters(); diff --git a/concurrency/tests/job_trace.rs b/concurrency/tests/job_trace.rs index 1a6e9d3e..f42efa95 100644 --- a/concurrency/tests/job_trace.rs +++ b/concurrency/tests/job_trace.rs @@ -291,6 +291,57 @@ impl Handler for Owner { } } +/// A held result whose destructor checks, from another thread, that the +/// process-wide trace section is free (it waits at most 1 s). +struct SectionProbe(Arc); +impl std::fmt::Debug for SectionProbe { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("SectionProbe") + } +} +impl Drop for SectionProbe { + fn drop(&mut self) { + let (entered_tx, entered) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + // Enters the trace section (no effect here: this binary does not + // record the base lifecycle). + let _ = lifecycle_trace::take_lifecycle(); + let _ = entered_tx.send(()); + }); + if entered.recv_timeout(Duration::from_secs(1)).is_ok() { + self.0.store(true, std::sync::atomic::Ordering::SeqCst); + } + } +} +/// A job whose result (a `SectionProbe`) is held behind a far emit; its +/// handle goes to the test and to `handles` (for `CancelStored`). +struct HeldProbe( + Arc, + std::sync::mpsc::Sender, +); +impl Message for HeldProbe { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: HeldProbe, reply: PendingReply<'_, ()>, ctx: &Context) { + let HeldProbe(free, handle_tx) = msg; + let handle = ctx + .pipe( + JobOptions::new(), + |job: JobCtx| async move { + let at = spawned_concurrency::Instant::now() + Duration::from_secs(60); + job.emit_at(at, Emitted(3)).await.unwrap(); + SectionProbe(free) + }, + done("held_probe"), + ) + .unwrap(); + handle_tx.send(handle.clone()).unwrap(); + self.handles.push(handle); + reply.reply(()); + } +} + /// Cancel the stored handles, then open the gates. struct CancelStored; impl Message for CancelStored { @@ -516,6 +567,34 @@ async fn keyed_replacement_drops_the_old_emit_after_the_new_start() { ); } +/// Final review (agy P2 3): cancelling a job drops its suppressed held +/// result (user payload) after the trace section, after its Cancel is +/// recorded; it used to drop inside the process-wide section, so a +/// destructor that needed the section (here, on another thread) waited for +/// its own caller. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cancel_drops_a_held_result_outside_the_trace_section() { + let (actor, _log) = start(); + let free = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let (handle_tx, handle) = std::sync::mpsc::channel(); + actor + .request(HeldProbe(free.clone(), handle_tx)) + .await + .unwrap(); + let job = handle.recv_timeout(Duration::from_secs(5)).unwrap(); + // Handshake: the work returned with its emit scheduled, so its result + // is held. + while job.is_running() { + sleep(Duration::from_millis(1)).await; + } + actor.request(CancelStored).await.unwrap(); + finish("cancel_held", actor, 16).await; + assert!( + free.load(std::sync::atomic::Ordering::SeqCst), + "a held result was destroyed inside the trace section" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cancel_drops_a_scheduled_emit() { let (actor, log) = start(); From 4b85fb0e7610f30b419f9e5bc90c82dcab157ebe Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:45:46 -0300 Subject: [PATCH 185/208] docs: shorten_deadline takes any Into (Duration, Instant or Deadline) --- concurrency/src/reply/mod.rs | 7 ++++--- docs/MIGRATION.md | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index a18e2890..08645a58 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -469,9 +469,10 @@ impl ReplyTo { Deadline::of_ticks(self.slot().deadline()) } - /// Shorten the request's deadline to `deadline` (a `Duration` from now - /// or an `Instant`); a later deadline, or [`Deadline::Never`], leaves it - /// unchanged. The waiting caller is woken to re-arm its timer. + /// Shorten the request's deadline to `deadline` (anything + /// `Into`: a `Duration` from now, an `Instant`, or a + /// [`Deadline`], e.g. another request's); a later deadline, or + /// [`Deadline::Never`], leaves it unchanged. The waiting caller is woken to re-arm its timer. pub fn shorten_deadline(&self, deadline: impl Into) { // Only a lowered deadline gets a heap entry, so repeated calls with // the same or a later deadline store nothing. diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index e3cb0207..921aed4c 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -35,7 +35,7 @@ every change; this guide covers what you need to edit when upgrading. | `Response::from_with_timeout(actor.request_raw(msg), t)` | `Response::from(actor.send_request(msg, t))` | | a `Recipient` call `recipient.request_raw(msg)` | `recipient.send_request(msg, deadline)` (`Deadline`); implementing `Receiver` is no longer possible (sealed, see below) | | `reply.deadline()` / `token.deadline()` / `rx.deadline()` returning `Option` *(0.8 pre-release)* | return `Deadline` (`Deadline::Never` for none); forward with `other.send_request(msg, token.deadline())` | -| `token.with_deadline(at)` *(0.8 pre-release)* | `token.shorten_deadline(at)` (a `Duration` or `Instant`) | +| `token.with_deadline(at)` *(0.8 pre-release)* | `token.shorten_deadline(at)` (a `Duration`, an `Instant` or a `Deadline`) | | `reply.has_caller()` *(0.8 pre-release)* | removed: `reply.defer()` returns `Err(DeferError::NoCaller)` for a send | | `PendingReply::detached()` *(0.8 pre-release)* | `spawned_test::detached_reply()` | | `PendingReplies::{purge, retained, live}` *(0.8 pre-release)* | `len()` / `is_empty()` (live tokens); purging is automatic on `insert` | From c2802359c5ee493008ba2547002853b0cb2d7ba3 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Wed, 11 Nov 2026 16:47:36 -0300 Subject: [PATCH 186/208] feat(reply): InsertError::kind() and into_parts() answer every refused insert InsertError is #[non_exhaustive], so a client matching Duplicate and Full still needed a wildcard arm that could only drop the token: a future variant would silently answer its caller ReplyDropped (seen porting ethrex rlpx/connection/server.rs). InsertError::kind() returns a #[non_exhaustive] InsertErrorKind, and into_parts() the reason and the token together, so a caller answers the token for any reason, known or not (`_ => token.overloaded()`). InsertErrorKind is exported with InsertError (crate root, tasks, threads). The doc comment, MIGRATION, GUIDE, the client mapping and the deferred_replies example show the pattern. Test: reply::pending::tests::a_refusal_gives_its_kind_and_its_token. --- CHANGELOG.md | 4 ++ concurrency/src/lib.rs | 4 +- concurrency/src/reply/mod.rs | 2 +- concurrency/src/reply/pending.rs | 60 +++++++++++++++++++ concurrency/src/tasks/mod.rs | 4 +- concurrency/src/threads/mod.rs | 4 +- docs/GUIDE.md | 4 +- docs/MIGRATION.md | 2 +- .../design/deferred-replies-client-mapping.md | 6 +- docs/design/deferred-replies.md | 2 +- .../capabilities/src/bin/deferred_replies.rs | 15 +++-- 11 files changed, 89 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e075d773..203bc4bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +- `InsertError::kind()` (a `#[non_exhaustive]` `InsertErrorKind`) and + `InsertError::into_parts()`: a refused insert's reason and token come + together, so a caller answers the token for any reason, including one + added later, instead of dropping it in a wildcard arm. - Docs: an async job's `work` is a factory run at once in the current turn (its panic is a handler panic; a job refused by the owner's exit has run it); only its future is the job (`pipe` rustdoc, GUIDE). diff --git a/concurrency/src/lib.rs b/concurrency/src/lib.rs index 835abc81..d1ba2a87 100644 --- a/concurrency/src/lib.rs +++ b/concurrency/src/lib.rs @@ -118,8 +118,8 @@ pub use error::{ActorError, ExitReason, NoProc}; pub use link::Exit; pub use monitor::{Down, MonitorRef}; pub use reply::{ - Deadline, DeferError, Delivery, InsertError, PendingReplies, PendingReply, ReplyReceiver, - ReplyTo, + Deadline, DeferError, Delivery, InsertError, InsertErrorKind, PendingReplies, PendingReply, + ReplyReceiver, ReplyTo, }; pub use response::{Response, DEFAULT_REQUEST_TIMEOUT}; pub use spawned_macros::{actor, protocol, send_handler}; diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 08645a58..5bd49d24 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -14,7 +14,7 @@ pub(crate) mod registry; pub(crate) mod slot; pub(crate) mod turn; -pub use pending::{InsertError, PendingReplies}; +pub use pending::{InsertError, InsertErrorKind, PendingReplies}; pub(crate) use registry::DEFAULT_MAX_OUTSTANDING; pub use slot::Delivery; diff --git a/concurrency/src/reply/pending.rs b/concurrency/src/reply/pending.rs index 2a07bfe2..94b70c36 100644 --- a/concurrency/src/reply/pending.rs +++ b/concurrency/src/reply/pending.rs @@ -7,6 +7,21 @@ use std::hash::Hash; /// Why [`PendingReplies::insert`] handed the token back. The handler decides /// the answer (typically `token.overloaded()` or a domain error) and still /// owns the token. +/// +/// The enum is `#[non_exhaustive]`; match on [`kind`](Self::kind) (or take +/// both with [`into_parts`](Self::into_parts)) so the token is answered +/// whatever the reason, including one added later, instead of being +/// dropped in a wildcard arm (which answers the caller `ReplyDropped`): +/// +/// ```ignore +/// if let Err(refused) = table.insert(id, token) { +/// let (kind, token) = refused.into_parts(); +/// match kind { +/// InsertErrorKind::Duplicate => token.reply(Err(Error::DuplicateId)), +/// _ => token.overloaded(), // Full, and any later reason +/// } +/// } +/// ``` #[derive(Debug)] #[non_exhaustive] pub enum InsertError { @@ -16,13 +31,34 @@ pub enum InsertError { Full(ReplyTo), } +/// Why an insert was refused, without the token ([`InsertError::kind`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum InsertErrorKind { + /// A live token is already stored under this key. + Duplicate, + /// The table holds `cap` live tokens. + Full, +} + impl InsertError { + /// Why the token was handed back. + pub fn kind(&self) -> InsertErrorKind { + match self { + Self::Duplicate(_) => InsertErrorKind::Duplicate, + Self::Full(_) => InsertErrorKind::Full, + } + } /// The token handed back. pub fn into_inner(self) -> ReplyTo { match self { Self::Duplicate(token) | Self::Full(token) => token, } } + /// The reason and the token together: every refusal can be answered. + pub fn into_parts(self) -> (InsertErrorKind, ReplyTo) { + (self.kind(), self.into_inner()) + } } /// Deferred tokens by key, capped: retained entries never exceed `cap`, @@ -196,4 +232,28 @@ mod tests { assert_eq!(once, expected); assert_eq!(twice, expected); } + + /// Every refusal hands back its reason and its token together, so a + /// caller can answer the token for any reason, known or not. + #[test] + fn a_refusal_gives_its_kind_and_its_token() { + let mut table = PendingReplies::with_capacity(1); + let (a, ra) = token(); + let (dup, rdup) = token(); + let (full, rfull) = token(); + table.insert(1, a).unwrap(); + let refused = table.insert(1, dup).unwrap_err(); + assert_eq!(refused.kind(), InsertErrorKind::Duplicate); + let (kind, token) = refused.into_parts(); + assert_eq!(kind, InsertErrorKind::Duplicate); + token.overloaded(); + assert!(matches!(rdup.recv(), Err(ActorError::Overloaded))); + let refused = table.insert(2, full).unwrap_err(); + let (kind, token) = refused.into_parts(); + assert_eq!(kind, InsertErrorKind::Full); + token.reply(9); + assert_eq!(rfull.recv().unwrap(), 9); + table.take(&1).unwrap().reply(1); + assert_eq!(ra.recv().unwrap(), 1); + } } diff --git a/concurrency/src/tasks/mod.rs b/concurrency/src/tasks/mod.rs index dd73d3db..074d221d 100644 --- a/concurrency/src/tasks/mod.rs +++ b/concurrency/src/tasks/mod.rs @@ -11,8 +11,8 @@ pub use crate::reply::jobs::{ JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, PipeError, PipeReplyError, }; pub use crate::reply::{ - Deadline, DeferError, Delivery, InsertError, PendingReplies, PendingReply, ReplyFuture, - ReplyReceiver, ReplyTo, + Deadline, DeferError, Delivery, InsertError, InsertErrorKind, PendingReplies, PendingReply, + ReplyFuture, ReplyReceiver, ReplyTo, }; pub use crate::response::Response; pub use crate::Instant; diff --git a/concurrency/src/threads/mod.rs b/concurrency/src/threads/mod.rs index 4bdd6644..37f44979 100644 --- a/concurrency/src/threads/mod.rs +++ b/concurrency/src/threads/mod.rs @@ -12,8 +12,8 @@ pub use crate::reply::jobs::{ JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, PipeError, PipeReplyError, }; pub use crate::reply::{ - Deadline, DeferError, Delivery, InsertError, PendingReplies, PendingReply, ReplyReceiver, - ReplyTo, + Deadline, DeferError, Delivery, InsertError, InsertErrorKind, PendingReplies, PendingReply, + ReplyReceiver, ReplyTo, }; pub use crate::response::Response; pub use crate::Instant; diff --git a/docs/GUIDE.md b/docs/GUIDE.md index cb5131cd..3903b6b6 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -645,7 +645,9 @@ Rules (the [design](design/deferred-replies.md) has the full contract): the caller can no longer be answered; the actor's deadline heap expires stored tokens on time and returns their credits. - `PendingReplies` is a bounded table of tokens by key (purges terminal - entries on insert; `InsertError::{Duplicate, Full}` hands the token back). + entries on insert; `InsertError::{Duplicate, Full}` hands the token back; `into_parts()` + gives the `InsertErrorKind` and the token together, so even a reason + added later can be answered). **Answering with a reason.** The framework's errors (`ReplyDropped`, `ActorStopped`, `Overloaded`, `RequestTimeout`) say what happened to the diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 921aed4c..d6fe06c0 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -39,7 +39,7 @@ every change; this guide covers what you need to edit when upgrading. | `reply.has_caller()` *(0.8 pre-release)* | removed: `reply.defer()` returns `Err(DeferError::NoCaller)` for a send | | `PendingReply::detached()` *(0.8 pre-release)* | `spawned_test::detached_reply()` | | `PendingReplies::{purge, retained, live}` *(0.8 pre-release)* | `len()` / `is_empty()` (live tokens); purging is automatic on `insert` | -| `InsertError>` *(0.8 pre-release)* | `InsertError` (`#[non_exhaustive]`) | +| `InsertError>` *(0.8 pre-release)* | `InsertError` (`#[non_exhaustive]`): match on `refused.kind()` (`InsertErrorKind`, also non-exhaustive) or take `let (kind, token) = refused.into_parts()`, so a wildcard arm still answers the token (e.g. `token.overloaded()`) instead of dropping it (`ReplyDropped`) | | `DEFAULT_MAX_OUTSTANDING` *(0.8 pre-release)* | removed from the public API (`Actor::MAX_OUTSTANDING` defaults to 1024) | | `system.shutdown(deadline)`, `supervisor.shutdown(deadline)`, `report.wait(deadline)` | unchanged calls; the `Duration` parameter is named `grace` (a deadline is an instant in 0.8) | | `Down { monitor_ref, reason }` (pattern) | `Down { monitor_ref, reason, .. }`; `Down` gains `actor: ActorId` (the monitored actor) and, like `Exit`, is `#[non_exhaustive]` | diff --git a/docs/design/deferred-replies-client-mapping.md b/docs/design/deferred-replies-client-mapping.md index 3dcf487d..3fe85a7d 100644 --- a/docs/design/deferred-replies-client-mapping.md +++ b/docs/design/deferred-replies-client-mapping.md @@ -86,8 +86,10 @@ async fn request(&mut self, msg: Request, reply: PendingReply<'_, Message>, ctx: let Ok(token) = reply.defer() else { return }; // Overloaded (caller told) or Closed (caller gone) match state.current_requests.insert(id, token) { Ok(()) => state.outbox.push(msg.message), // see note on the socket send - Err(InsertError::Duplicate(token)) => drop(token), // the new caller is told; the old one kept - Err(InsertError::Full(token)) => token.overloaded(), // caller gets Err(Overloaded) + Err(refused) => match refused.into_parts() { + (InsertErrorKind::Duplicate, token) => drop(token), // the new caller is told; the old one kept + (_, token) => token.overloaded(), // Full or a later reason: caller gets Err(Overloaded) + }, } } // response path diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index 184b0fdb..dbf74f2e 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -504,7 +504,7 @@ impl PendingReplies { pub fn drain(&mut self) -> impl Iterator)>; // live tokens, oldest first } #[non_exhaustive] -pub enum InsertError { Duplicate(ReplyTo), Full(ReplyTo) } // into_inner() +pub enum InsertError { Duplicate(ReplyTo), Full(ReplyTo) } // kind(), into_inner(), into_parts() ``` Retained entries never exceed `cap`, however fast tokens expire. The handler diff --git a/examples/capabilities/src/bin/deferred_replies.rs b/examples/capabilities/src/bin/deferred_replies.rs index 11049e0a..0134524a 100644 --- a/examples/capabilities/src/bin/deferred_replies.rs +++ b/examples/capabilities/src/bin/deferred_replies.rs @@ -13,7 +13,7 @@ //! rest with `ActorStopped`. use spawned_concurrency::message::Message; use spawned_concurrency::tasks::{ - Actor, ActorRef, ActorStart, Context, Deadline, Handler, InsertError, JobError, JobOptions, + Actor, ActorRef, ActorStart, Context, Deadline, Handler, InsertErrorKind, JobError, JobOptions, JobOutcome, PendingReplies, PendingReply, ReplyTo, }; use spawned_concurrency::{actor, protocol, ActorError, Response}; @@ -194,11 +194,14 @@ impl Connection { _: &Context, ) { let Ok(token) = reply.defer() else { return }; - match self.current_requests.insert(msg.id, token) { - Ok(()) => {} // the request goes out on the socket here - Err(InsertError::Duplicate(token)) => drop(token), // ReplyDropped; the first caller is kept - // `Full` (InsertError is non_exhaustive): answer Overloaded. - Err(error) => error.into_inner().overloaded(), + // The request goes out on the socket on Ok(()). A refusal hands back + // its reason and the token together, so every reason is answered. + if let Err(refused) = self.current_requests.insert(msg.id, token) { + let (kind, token) = refused.into_parts(); + match kind { + InsertErrorKind::Duplicate => drop(token), // ReplyDropped; the first caller is kept + _ => token.overloaded(), // Full, and any reason added later + } } } From fc7cddf0d006ffeb8aa3e8281a98752d0152895d Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 06:45:10 -0300 Subject: [PATCH 187/208] test(fuzz): main's linked-reason oracle accepts a late link's NoProc (0.8) #265's X3 linked-reason oracle (an actor stopped only through a link publishes its peer's reason, and a Normal peer exit does not stop it) predates 0.8's late links: a link made after the peer's death point delivers a NoProc exit signal, so a non-trapping caller ends NoProc whatever the peer's reason (E4), as the existing X3 Exit-reason oracle already accepts. With it, the corpus and 3000 generated inputs pass. --- fuzz/src/driver.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fuzz/src/driver.rs b/fuzz/src/driver.rs index 1fa46f29..e6d66175 100644 --- a/fuzz/src/driver.rs +++ b/fuzz/src/driver.rs @@ -1871,6 +1871,12 @@ impl Driver { } let detail = format!("w{} published {exit:?}, peer {peer:?}", a.instance); drop(s); + // 0.8 (E4): a link made after the peer's death point is a late + // link; a non-trapping caller gets the `NoProc` exit signal and ends + // `NoProc`, whatever the peer's reason was. + if exit == ExitReason::NoProc { + return; + } if *peer == ExitReason::Normal { self.violate( "X3: a non-trapping actor was stopped by a linked peer's Normal exit", From cca51051a11ed8aa6b0706f4d6c54ccfd7646e6b Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 07:46:16 -0300 Subject: [PATCH 188/208] test: bound the in-crate unit tests that wait on actors (fail by name, never hang) Planted-bug follow-up (#265): under planted bug 20, tasks::actor::tests::backend_blocking_handles_send_and_request hung the job instead of failing (its assertion failed, then the Tokio runtime's teardown waited forever for the Blocking backend's thread); other unit tests under concurrency/src also waited without a bound. concurrency/src/test_wait.rs gains HANG (120 s, as tests/support/deadline.rs), `bounded(name, body)` and `within(name, future)`. Every #[test] under concurrency/src whose body starts, calls, joins or waits on actors (162 tests in 18 files) runs its whole body (runtime and teardown included) through `bounded`, on a thread named after the test; a #[tokio::test] body goes through `within`. A body still running after 120 s fails the test by name; a panic in the body, an assertion or a #[should_panic] test's expected one, is re-raised unchanged. No unit test runs on paused Tokio time, so none is exempted. Checked: with planted bug 20 applied by hand (its 0.7 patch does not apply), backend_blocking_handles_send_and_request fails after 120 s with "timed out after 120s: ...backend_blocking_handles_send_and_request did not finish". Unit tests pass in the default, no-default and failpoints+lifecycle-trace+metrics feature sets (227, 110, 234). --- concurrency/src/child_handle.rs | 571 ++-- concurrency/src/link.rs | 107 +- concurrency/src/link_regression_tests.rs | 921 +++--- concurrency/src/mailbox.rs | 294 +- .../src/native_link_regression_tests.rs | 76 +- concurrency/src/reply/pending.rs | 108 +- concurrency/src/response.rs | 116 +- concurrency/src/supervision.rs | 1363 ++++---- concurrency/src/supervision/nested.rs | 255 +- concurrency/src/system.rs | 142 +- concurrency/src/tasks/actor.rs | 2794 +++++++++-------- concurrency/src/tasks/jobs.rs | 65 +- concurrency/src/tasks/stream_tests.rs | 328 +- concurrency/src/tasks/timer_tests.rs | 149 +- concurrency/src/test_wait.rs | 41 + concurrency/src/threads/actor.rs | 1725 +++++----- concurrency/src/threads/jobs.rs | 99 +- concurrency/src/threads/stream_tests.rs | 159 +- concurrency/src/threads/timer_tests.rs | 253 +- docs/TESTING.md | 9 +- 20 files changed, 5367 insertions(+), 4208 deletions(-) diff --git a/concurrency/src/child_handle.rs b/concurrency/src/child_handle.rs index 1c29caad..3b94d213 100644 --- a/concurrency/src/child_handle.rs +++ b/concurrency/src/child_handle.rs @@ -479,269 +479,332 @@ pub(crate) mod tests { #[test] fn child_handle_from_threads_actor_ref() { - use crate::message::Message; - use crate::threads::actor::{Actor, ActorStart, Context, Handler}; - - struct Worker; - struct Stop; - impl Message for Stop { - type Result = (); - } - impl Actor for Worker {} - impl Handler for Worker { - fn handle( - &mut self, - _msg: Stop, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - ctx.stop(); - reply.reply(()); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::child_handle_from_threads_actor_ref"), + move || { + use crate::message::Message; + use crate::threads::actor::{Actor, ActorStart, Context, Handler}; + + struct Worker; + struct Stop; + impl Message for Stop { + type Result = (); + } + impl Actor for Worker {} + impl Handler for Worker { + fn handle( + &mut self, + _msg: Stop, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.stop(); + reply.reply(()); + } + } - let actor = Worker.start(); - let handle = actor.child_handle(); + let actor = Worker.start(); + let handle = actor.child_handle(); - assert!(handle.is_alive()); - assert!(handle.exit_reason().is_none()); - assert_eq!(handle.id(), actor.id()); + assert!(handle.is_alive()); + assert!(handle.exit_reason().is_none()); + assert_eq!(handle.id(), actor.id()); - actor.send(Stop).unwrap(); - let reason = handle.wait_exit_blocking(); - assert_eq!(reason, ExitReason::Normal); - assert!(!handle.is_alive()); + actor.send(Stop).unwrap(); + let reason = handle.wait_exit_blocking(); + assert_eq!(reason, ExitReason::Normal); + assert!(!handle.is_alive()); + }, + ); } #[test] #[cfg(feature = "async")] fn child_handle_from_tasks_actor_ref() { - use crate::message::Message; - use crate::tasks::actor::{Actor, ActorStart, Context, Handler}; - - struct Worker; - struct Stop; - impl Message for Stop { - type Result = (); - } - impl Actor for Worker {} - impl Handler for Worker { - async fn handle( - &mut self, - _msg: Stop, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - ctx.stop(); - reply.reply(()); - } - } - - let runtime = spawned_rt::tasks::Runtime::new().unwrap(); - runtime.block_on(async { - let actor = Worker.start(); - let handle = actor.child_handle(); - - assert!(handle.is_alive()); - assert!(handle.exit_reason().is_none()); - assert_eq!(handle.id(), actor.id()); + crate::test_wait::bounded( + concat!(module_path!(), "::child_handle_from_tasks_actor_ref"), + move || { + use crate::message::Message; + use crate::tasks::actor::{Actor, ActorStart, Context, Handler}; + + struct Worker; + struct Stop; + impl Message for Stop { + type Result = (); + } + impl Actor for Worker {} + impl Handler for Worker { + async fn handle( + &mut self, + _msg: Stop, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.stop(); + reply.reply(()); + } + } - actor.send(Stop).unwrap(); - let reason = handle.wait_exit_async().await; - assert_eq!(reason, ExitReason::Normal); - assert!(!handle.is_alive()); - }); + let runtime = spawned_rt::tasks::Runtime::new().unwrap(); + runtime.block_on(async { + let actor = Worker.start(); + let handle = actor.child_handle(); + + assert!(handle.is_alive()); + assert!(handle.exit_reason().is_none()); + assert_eq!(handle.id(), actor.id()); + + actor.send(Stop).unwrap(); + let reason = handle.wait_exit_async().await; + assert_eq!(reason, ExitReason::Normal); + assert!(!handle.is_alive()); + }); + }, + ); } #[test] #[cfg(feature = "async")] fn child_handle_stop_from_tasks_actor() { - use crate::tasks::actor::ActorStart; - use tasks_fixtures::Idler; - - let runtime = spawned_rt::tasks::Runtime::new().unwrap(); - runtime.block_on(async { - let actor = Idler.start(); - let handle = actor.child_handle(); - - assert!(handle.is_alive()); - handle.stop(); - let reason = handle.wait_exit_async().await; - assert_eq!(reason, ExitReason::Normal); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::child_handle_stop_from_tasks_actor"), + move || { + use crate::tasks::actor::ActorStart; + use tasks_fixtures::Idler; + + let runtime = spawned_rt::tasks::Runtime::new().unwrap(); + runtime.block_on(async { + let actor = Idler.start(); + let handle = actor.child_handle(); + + assert!(handle.is_alive()); + handle.stop(); + let reason = handle.wait_exit_async().await; + assert_eq!(reason, ExitReason::Normal); + }); + }, + ); } #[test] #[cfg(feature = "async")] fn child_handle_wait_blocking_inside_multithread_runtime() { - use crate::tasks::actor::ActorStart; - use tasks_fixtures::Idler; - - // Multi-thread runtime — wait_exit_blocking should work via block_in_place - let runtime = spawned_rt::tasks::Runtime::new().unwrap(); - runtime.block_on(async { - let actor = Idler.start(); - let handle = actor.child_handle(); - handle.stop(); - - let reason = handle.wait_exit_blocking(); - assert_eq!(reason, ExitReason::Normal); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::child_handle_wait_blocking_inside_multithread_runtime" + ), + move || { + use crate::tasks::actor::ActorStart; + use tasks_fixtures::Idler; + + // Multi-thread runtime — wait_exit_blocking should work via block_in_place + let runtime = spawned_rt::tasks::Runtime::new().unwrap(); + runtime.block_on(async { + let actor = Idler.start(); + let handle = actor.child_handle(); + handle.stop(); + + let reason = handle.wait_exit_blocking(); + assert_eq!(reason, ExitReason::Normal); + }); + }, + ); } #[test] #[should_panic(expected = "non-multi-thread tokio runtime")] #[cfg(feature = "async")] fn child_handle_wait_blocking_panics_on_current_thread_runtime() { - use crate::tasks::actor::ActorStart; - use tasks_fixtures::Idler; - - let runtime = ::tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - - runtime.block_on(async { - let actor = Idler.start(); - let handle = actor.child_handle(); - // Calling wait_exit_blocking from within a current-thread runtime - // would deadlock the actor task — we panic with a clear message instead. - let _ = handle.wait_exit_blocking(); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::child_handle_wait_blocking_panics_on_current_thread_runtime" + ), + move || { + use crate::tasks::actor::ActorStart; + use tasks_fixtures::Idler; + + let runtime = ::tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + let actor = Idler.start(); + let handle = actor.child_handle(); + // Calling wait_exit_blocking from within a current-thread runtime + // would deadlock the actor task — we panic with a clear message instead. + let _ = handle.wait_exit_blocking(); + }); + }, + ); } #[test] #[cfg(feature = "async")] fn child_handle_wait_blocking_fast_path_on_current_thread_runtime() { - use crate::tasks::actor::ActorStart; - use tasks_fixtures::Idler; - - let runtime = ::tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - - // Fast path: if the actor has already exited, wait_exit_blocking is safe - // even on a current-thread runtime (no actual blocking happens). - runtime.block_on(async { - let actor = Idler.start(); - let handle = actor.child_handle(); - handle.stop(); - // Wait async first so the actor actually exits - let _ = handle.wait_exit_async().await; - // Now wait_exit_blocking takes the fast path - let reason = handle.wait_exit_blocking(); - assert_eq!(reason, ExitReason::Normal); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::child_handle_wait_blocking_fast_path_on_current_thread_runtime" + ), + move || { + use crate::tasks::actor::ActorStart; + use tasks_fixtures::Idler; + + let runtime = ::tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + // Fast path: if the actor has already exited, wait_exit_blocking is safe + // even on a current-thread runtime (no actual blocking happens). + runtime.block_on(async { + let actor = Idler.start(); + let handle = actor.child_handle(); + handle.stop(); + // Wait async first so the actor actually exits + let _ = handle.wait_exit_async().await; + // Now wait_exit_blocking takes the fast path + let reason = handle.wait_exit_blocking(); + assert_eq!(reason, ExitReason::Normal); + }); + }, + ); } // --- Panic observation tests --- #[test] fn child_handle_observes_panic_in_threads_actor() { - use crate::message::Message; - use crate::threads::actor::{Actor, ActorStart, Context, Handler}; - - struct Boomer; - struct Boom; - impl Message for Boom { - type Result = (); - } - impl Actor for Boomer {} - impl Handler for Boomer { - fn handle( - &mut self, - _msg: Boom, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("intentional panic in handler"); + crate::test_wait::bounded( + concat!( + module_path!(), + "::child_handle_observes_panic_in_threads_actor" + ), + move || { + use crate::message::Message; + use crate::threads::actor::{Actor, ActorStart, Context, Handler}; + + struct Boomer; + struct Boom; + impl Message for Boom { + type Result = (); + } + impl Actor for Boomer {} + impl Handler for Boomer { + fn handle( + &mut self, + _msg: Boom, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("intentional panic in handler"); + } + } } - } - } - let actor = Boomer.start(); - let handle = actor.child_handle(); - let _ = actor.send(Boom); + let actor = Boomer.start(); + let handle = actor.child_handle(); + let _ = actor.send(Boom); - let reason = handle.wait_exit_blocking(); - match reason { - ExitReason::Panic(msg) => { - assert!(msg.contains("intentional panic in handler"), "got: {msg}"); - } - other => panic!("expected Panic, got {other:?}"), - } - assert!(!handle.is_alive()); + let reason = handle.wait_exit_blocking(); + match reason { + ExitReason::Panic(msg) => { + assert!(msg.contains("intentional panic in handler"), "got: {msg}"); + } + other => panic!("expected Panic, got {other:?}"), + } + assert!(!handle.is_alive()); + }, + ); } #[test] #[cfg(feature = "async")] fn child_handle_observes_panic_in_tasks_actor() { - use crate::message::Message; - use crate::tasks::actor::{Actor, ActorStart, Context, Handler}; - - struct Boomer; - struct Boom; - impl Message for Boom { - type Result = (); - } - impl Actor for Boomer {} - impl Handler for Boomer { - async fn handle( - &mut self, - _msg: Boom, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("intentional panic in handler"); + crate::test_wait::bounded( + concat!( + module_path!(), + "::child_handle_observes_panic_in_tasks_actor" + ), + move || { + use crate::message::Message; + use crate::tasks::actor::{Actor, ActorStart, Context, Handler}; + + struct Boomer; + struct Boom; + impl Message for Boom { + type Result = (); } - } - } - - let runtime = spawned_rt::tasks::Runtime::new().unwrap(); - runtime.block_on(async { - let actor = Boomer.start(); - let handle = actor.child_handle(); - let _ = actor.send(Boom); - - let reason = handle.wait_exit_async().await; - match reason { - ExitReason::Panic(msg) => { - assert!(msg.contains("intentional panic in handler"), "got: {msg}"); + impl Actor for Boomer {} + impl Handler for Boomer { + async fn handle( + &mut self, + _msg: Boom, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("intentional panic in handler"); + } + } } - other => panic!("expected Panic, got {other:?}"), - } - assert!(!handle.is_alive()); - }); + + let runtime = spawned_rt::tasks::Runtime::new().unwrap(); + runtime.block_on(async { + let actor = Boomer.start(); + let handle = actor.child_handle(); + let _ = actor.send(Boom); + + let reason = handle.wait_exit_async().await; + match reason { + ExitReason::Panic(msg) => { + assert!(msg.contains("intentional panic in handler"), "got: {msg}"); + } + other => panic!("expected Panic, got {other:?}"), + } + assert!(!handle.is_alive()); + }); + }, + ); } #[test] #[cfg(feature = "async")] fn child_handle_observes_panic_in_started_callback() { - use crate::tasks::actor::{Actor, ActorStart, Context}; - - struct PanicStart; - impl Actor for PanicStart { - async fn started(&mut self, _ctx: &Context) { - panic!("intentional panic in started"); - } - } - - let runtime = spawned_rt::tasks::Runtime::new().unwrap(); - runtime.block_on(async { - let actor = PanicStart.start(); - let handle = actor.child_handle(); - - let reason = handle.wait_exit_async().await; - match reason { - ExitReason::Panic(msg) => { - assert!(msg.contains("intentional panic in started"), "got: {msg}"); + crate::test_wait::bounded( + concat!( + module_path!(), + "::child_handle_observes_panic_in_started_callback" + ), + move || { + use crate::tasks::actor::{Actor, ActorStart, Context}; + + struct PanicStart; + impl Actor for PanicStart { + async fn started(&mut self, _ctx: &Context) { + panic!("intentional panic in started"); + } } - other => panic!("expected Panic, got {other:?}"), - } - }); + + let runtime = spawned_rt::tasks::Runtime::new().unwrap(); + runtime.block_on(async { + let actor = PanicStart.start(); + let handle = actor.child_handle(); + + let reason = handle.wait_exit_async().await; + match reason { + ExitReason::Panic(msg) => { + assert!(msg.contains("intentional panic in started"), "got: {msg}"); + } + other => panic!("expected Panic, got {other:?}"), + } + }); + }, + ); } } @@ -751,47 +814,55 @@ mod closed_completion_tests { use crate::link::new_link_table; #[test] fn closed_without_reason_is_dead_and_all_waits_agree() { - #[cfg(not(feature = "async"))] - let modes = [true]; - #[cfg(feature = "async")] - let modes = [false, true]; - for threads in modes { - let (tx, rx) = spawned_rt::sync::watch::channel(None); - let target = super::tests::test_target( - ActorId::next(), - spawned_rt::threads::CancellationToken::new(), - ); - let completion = if threads { - Completion::Threads(rx) - } else { + crate::test_wait::bounded( + concat!( + module_path!(), + "::closed_without_reason_is_dead_and_all_waits_agree" + ), + move || { #[cfg(not(feature = "async"))] - unreachable!(); + let modes = [true]; #[cfg(feature = "async")] - Completion::Tasks(rx) - }; - let child = ChildHandle::new(target, completion, new_link_table()); - assert!(child.is_alive()); - drop(tx); - assert_eq!(child.exit_reason(), Some(ExitReason::Killed)); - assert!(!child.is_alive()); - assert_eq!( - futures::executor::block_on(child.wait_exit_async()), - ExitReason::Killed - ); - // Already-exited fast path must not enter another executor or reject - // a current-thread runtime merely because no exit value was published. - assert_eq!( - futures::executor::block_on(async { child.wait_exit_blocking() }), - ExitReason::Killed - ); - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - rt.block_on(async { - assert_eq!(child.wait_exit_blocking(), ExitReason::Killed); - }); - } + let modes = [false, true]; + for threads in modes { + let (tx, rx) = spawned_rt::sync::watch::channel(None); + let target = super::tests::test_target( + ActorId::next(), + spawned_rt::threads::CancellationToken::new(), + ); + let completion = if threads { + Completion::Threads(rx) + } else { + #[cfg(not(feature = "async"))] + unreachable!(); + #[cfg(feature = "async")] + Completion::Tasks(rx) + }; + let child = ChildHandle::new(target, completion, new_link_table()); + assert!(child.is_alive()); + drop(tx); + assert_eq!(child.exit_reason(), Some(ExitReason::Killed)); + assert!(!child.is_alive()); + assert_eq!( + futures::executor::block_on(child.wait_exit_async()), + ExitReason::Killed + ); + // Already-exited fast path must not enter another executor or reject + // a current-thread runtime merely because no exit value was published. + assert_eq!( + futures::executor::block_on(async { child.wait_exit_blocking() }), + ExitReason::Killed + ); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(async { + assert_eq!(child.wait_exit_blocking(), ExitReason::Killed); + }); + } + }, + ); } #[test] fn closure_preserves_the_final_published_reason() { diff --git a/concurrency/src/link.rs b/concurrency/src/link.rs index da16c90b..978c70e6 100644 --- a/concurrency/src/link.rs +++ b/concurrency/src/link.rs @@ -778,57 +778,72 @@ mod tests { } #[test] fn reason_precedence_until_death() { - // A late non-kill signal replaces a requested reason (decision 8); - // a kill supersedes it and nothing replaces Killed. - let mut r = ExitRequest::default(); - assert!(r.request(ExitReason::Normal)); - assert!(r.untrapped(ExitReason::Shutdown)); - assert_eq!(r.reason(), Some(&ExitReason::Shutdown)); - assert!( - r.begin_hook(), - "a late signal does not skip a self-exit's stopped()" + crate::test_wait::bounded( + concat!(module_path!(), "::reason_precedence_until_death"), + move || { + // A late non-kill signal replaces a requested reason (decision 8); + // a kill supersedes it and nothing replaces Killed. + let mut r = ExitRequest::default(); + assert!(r.request(ExitReason::Normal)); + assert!(r.untrapped(ExitReason::Shutdown)); + assert_eq!(r.reason(), Some(&ExitReason::Shutdown)); + assert!( + r.begin_hook(), + "a late signal does not skip a self-exit's stopped()" + ); + r.panic(ExitReason::Panic("stopped".into())); + assert_eq!(r.reason(), Some(&ExitReason::Panic("stopped".into()))); + assert!(r.kill()); + r.panic(ExitReason::Panic("after kill".into())); + assert!(!r.untrapped(ExitReason::Shutdown)); + assert!(r.request(ExitReason::Normal)); + assert_eq!(r.die(), ExitReason::Killed); + assert!(!r.kill() && !r.untrapped(ExitReason::Shutdown)); + // Runtime drop: Killed unless a non-Normal reason was requested. + let mut r = ExitRequest::default(); + r.request(ExitReason::Normal); + assert_eq!(r.runtime_drop(), ExitReason::Killed); + let mut r = ExitRequest::default(); + r.request(ExitReason::Shutdown); + assert_eq!(r.runtime_drop(), ExitReason::Shutdown); + }, ); - r.panic(ExitReason::Panic("stopped".into())); - assert_eq!(r.reason(), Some(&ExitReason::Panic("stopped".into()))); - assert!(r.kill()); - r.panic(ExitReason::Panic("after kill".into())); - assert!(!r.untrapped(ExitReason::Shutdown)); - assert!(r.request(ExitReason::Normal)); - assert_eq!(r.die(), ExitReason::Killed); - assert!(!r.kill() && !r.untrapped(ExitReason::Shutdown)); - // Runtime drop: Killed unless a non-Normal reason was requested. - let mut r = ExitRequest::default(); - r.request(ExitReason::Normal); - assert_eq!(r.runtime_drop(), ExitReason::Killed); - let mut r = ExitRequest::default(); - r.request(ExitReason::Shutdown); - assert_eq!(r.runtime_drop(), ExitReason::Shutdown); } #[test] fn link_from_an_exiting_or_dead_caller_is_inert() { - let seen = Arc::new(AtomicUsize::new(0)); - // Exit begun: no record on either side, even to a live peer. - let (a, at, b, bt) = pair(&seen); - lock_request(&a.request).request(ExitReason::Normal); - assert_eq!( - register_link(&a, &at, &b, &bt, false), - LinkOutcome::Unchanged - ); - assert!(at.lock().unwrap().entries.is_empty() && bt.lock().unwrap().entries.is_empty()); - // Past D, even to a dead peer: no NoProc notice, no try_link error. - let (c, ct, d, dt) = pair(&seen); - lock_request(&c.request).die(); - propagate_exit(c.id, &ct, &ExitReason::Normal); - propagate_exit(d.id, &dt, &ExitReason::Normal); - assert_eq!( - register_link(&c, &ct, &d, &dt, false), - LinkOutcome::Unchanged + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_from_an_exiting_or_dead_caller_is_inert" + ), + move || { + let seen = Arc::new(AtomicUsize::new(0)); + // Exit begun: no record on either side, even to a live peer. + let (a, at, b, bt) = pair(&seen); + lock_request(&a.request).request(ExitReason::Normal); + assert_eq!( + register_link(&a, &at, &b, &bt, false), + LinkOutcome::Unchanged + ); + assert!( + at.lock().unwrap().entries.is_empty() && bt.lock().unwrap().entries.is_empty() + ); + // Past D, even to a dead peer: no NoProc notice, no try_link error. + let (c, ct, d, dt) = pair(&seen); + lock_request(&c.request).die(); + propagate_exit(c.id, &ct, &ExitReason::Normal); + propagate_exit(d.id, &dt, &ExitReason::Normal); + assert_eq!( + register_link(&c, &ct, &d, &dt, false), + LinkOutcome::Unchanged + ); + assert_eq!( + register_link(&c, &ct, &d, &dt, true), + LinkOutcome::Unchanged + ); + assert_eq!(seen.load(Ordering::SeqCst), 0); + }, ); - assert_eq!( - register_link(&c, &ct, &d, &dt, true), - LinkOutcome::Unchanged - ); - assert_eq!(seen.load(Ordering::SeqCst), 0); } #[test] fn self_link_is_noop() { diff --git a/concurrency/src/link_regression_tests.rs b/concurrency/src/link_regression_tests.rs index 4708b999..91426709 100644 --- a/concurrency/src/link_regression_tests.rs +++ b/concurrency/src/link_regression_tests.rs @@ -25,54 +25,62 @@ fn runtime() -> tokio::runtime::Runtime { #[test] fn linking_between_death_point_and_publication_delivers_noproc() { - let rt = runtime(); - let events = Arc::new(Mutex::new(Vec::new())); - let observer = { - let _entered = rt.enter(); - Observer(events.clone()).start() - }; - observer.context().trap_exit(true); - let peer_id = ActorId::next(); - let peer_links = link::new_link_table(); - let (_completion_tx, completion_rx) = spawned_rt::tasks::watch::channel(None); - // This is the actual state between the death point D and publication: - // a registration after D reports NoProc, not the real reason (E4). - let real = ExitReason::Panic("closing before completion".into()); - link::propagate_exit(peer_id, &peer_links, &real); - let reason = ExitReason::NoProc; - let peer = ChildHandle::new( - crate::child_handle::tests::test_target( - peer_id, - spawned_rt::threads::CancellationToken::new(), + crate::test_wait::bounded( + concat!( + module_path!(), + "::linking_between_death_point_and_publication_delivers_noproc" ), - crate::child_handle::Completion::Tasks(completion_rx), - peer_links, - ); - observer.context().link(&peer); - let delivered = rt.block_on(async { - tokio::time::timeout(crate::test_wait::WAIT, async { - loop { - if !events.lock().unwrap().is_empty() { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .is_ok() - }); - observer.child_handle().stop(); - rt.block_on(observer.wait_exit()); - assert!( - delivered, - "exit lost between link-table drain and completion publication" - ); - assert_eq!( - *events.lock().unwrap(), - vec![Exit { - from: peer_id, - reason - }] + move || { + let rt = runtime(); + let events = Arc::new(Mutex::new(Vec::new())); + let observer = { + let _entered = rt.enter(); + Observer(events.clone()).start() + }; + observer.context().trap_exit(true); + let peer_id = ActorId::next(); + let peer_links = link::new_link_table(); + let (_completion_tx, completion_rx) = spawned_rt::tasks::watch::channel(None); + // This is the actual state between the death point D and publication: + // a registration after D reports NoProc, not the real reason (E4). + let real = ExitReason::Panic("closing before completion".into()); + link::propagate_exit(peer_id, &peer_links, &real); + let reason = ExitReason::NoProc; + let peer = ChildHandle::new( + crate::child_handle::tests::test_target( + peer_id, + spawned_rt::threads::CancellationToken::new(), + ), + crate::child_handle::Completion::Tasks(completion_rx), + peer_links, + ); + observer.context().link(&peer); + let delivered = rt.block_on(async { + tokio::time::timeout(crate::test_wait::WAIT, async { + loop { + if !events.lock().unwrap().is_empty() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .is_ok() + }); + observer.child_handle().stop(); + rt.block_on(observer.wait_exit()); + assert!( + delivered, + "exit lost between link-table drain and completion publication" + ); + assert_eq!( + *events.lock().unwrap(), + vec![Exit { + from: peer_id, + reason + }] + ); + }, ); } @@ -80,38 +88,47 @@ struct Idler; impl Actor for Idler {} #[test] fn never_polled_actor_propagates_runtime_shutdown_to_peer() { - #[cfg(feature = "backend-experiments")] - { - let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); - if selector == "sharded" - || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) - .is_some() - { - return; // Those process-owned pools deliberately outlive the caller runtime. - } - } + crate::test_wait::bounded( + concat!( + module_path!(), + "::never_polled_actor_propagates_runtime_shutdown_to_peer" + ), + move || { + #[cfg(feature = "backend-experiments")] + { + let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); + if selector == "sharded" + || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) + .is_some() + { + return; // Those process-owned pools deliberately outlive the caller runtime. + } + } - let owner = runtime(); - let survivor = runtime(); - let target = { - let _entered = owner.enter(); - Idler.start() - }; - let peer = { - let _entered = survivor.enter(); - Idler.start() - }; - peer.context().link(&target.child_handle()); - // The owner runtime never polls its actor. Dropping it must still close links. - drop(owner); - let result = survivor - .block_on(async { tokio::time::timeout(crate::test_wait::WAIT, peer.wait_exit()).await }); - peer.child_handle().stop(); - survivor.block_on(peer.wait_exit()); - assert_eq!( - result, - Ok(ExitReason::Killed), - "aborted actor failed to signal its live peer" + let owner = runtime(); + let survivor = runtime(); + let target = { + let _entered = owner.enter(); + Idler.start() + }; + let peer = { + let _entered = survivor.enter(); + Idler.start() + }; + peer.context().link(&target.child_handle()); + // The owner runtime never polls its actor. Dropping it must still close links. + drop(owner); + let result = survivor.block_on(async { + tokio::time::timeout(crate::test_wait::WAIT, peer.wait_exit()).await + }); + peer.child_handle().stop(); + survivor.block_on(peer.wait_exit()); + assert_eq!( + result, + Ok(ExitReason::Killed), + "aborted actor failed to signal its live peer" + ); + }, ); } @@ -132,34 +149,43 @@ impl crate::tasks::Handler for Idler { #[test] fn polled_actor_propagates_runtime_shutdown_to_peer() { - #[cfg(feature = "backend-experiments")] - { - let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); - if selector == "sharded" - || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) - .is_some() - { - return; - } - } - let owner = runtime(); - let survivor = runtime(); - let target = { - let _entered = owner.enter(); - Idler.start() - }; - owner.block_on(target.request(Ping)).unwrap(); - let peer = { - let _entered = survivor.enter(); - Idler.start() - }; - peer.context().link(&target.child_handle()); - drop(owner); - let result = survivor - .block_on(async { tokio::time::timeout(Duration::from_secs(1), peer.wait_exit()).await }); - peer.child_handle().stop(); - survivor.block_on(peer.wait_exit()); - assert_eq!(result, Ok(ExitReason::Killed)); + crate::test_wait::bounded( + concat!( + module_path!(), + "::polled_actor_propagates_runtime_shutdown_to_peer" + ), + move || { + #[cfg(feature = "backend-experiments")] + { + let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); + if selector == "sharded" + || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) + .is_some() + { + return; + } + } + let owner = runtime(); + let survivor = runtime(); + let target = { + let _entered = owner.enter(); + Idler.start() + }; + owner.block_on(target.request(Ping)).unwrap(); + let peer = { + let _entered = survivor.enter(); + Idler.start() + }; + peer.context().link(&target.child_handle()); + drop(owner); + let result = survivor.block_on(async { + tokio::time::timeout(Duration::from_secs(1), peer.wait_exit()).await + }); + peer.child_handle().stop(); + survivor.block_on(peer.wait_exit()); + assert_eq!(result, Ok(ExitReason::Killed)); + }, + ); } struct MonitorObserver(Option>); @@ -177,37 +203,45 @@ impl crate::tasks::Handler for MonitorObserver { } #[test] fn never_polled_actor_notifies_monitor_on_runtime_shutdown() { - #[cfg(feature = "backend-experiments")] - { - let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); - if selector == "sharded" - || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) - .is_some() - { - return; - } - } - let owner = runtime(); - let survivor = runtime(); - let target = { - let _entered = owner.enter(); - Idler.start() - }; - let (tx, rx) = spawned_rt::tasks::oneshot::channel(); - let observer = { - let _entered = survivor.enter(); - MonitorObserver(Some(tx)).start() - }; - { - let _entered = survivor.enter(); - observer.context().monitor(&target.child_handle()); - } - drop(owner); - let result = - survivor.block_on(async { tokio::time::timeout(Duration::from_secs(1), rx).await }); - observer.child_handle().stop(); - survivor.block_on(observer.wait_exit()); - assert_eq!(result.unwrap().unwrap().reason, ExitReason::Killed); + crate::test_wait::bounded( + concat!( + module_path!(), + "::never_polled_actor_notifies_monitor_on_runtime_shutdown" + ), + move || { + #[cfg(feature = "backend-experiments")] + { + let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); + if selector == "sharded" + || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) + .is_some() + { + return; + } + } + let owner = runtime(); + let survivor = runtime(); + let target = { + let _entered = owner.enter(); + Idler.start() + }; + let (tx, rx) = spawned_rt::tasks::oneshot::channel(); + let observer = { + let _entered = survivor.enter(); + MonitorObserver(Some(tx)).start() + }; + { + let _entered = survivor.enter(); + observer.context().monitor(&target.child_handle()); + } + drop(owner); + let result = + survivor.block_on(async { tokio::time::timeout(Duration::from_secs(1), rx).await }); + observer.child_handle().stop(); + survivor.block_on(observer.wait_exit()); + assert_eq!(result.unwrap().unwrap().reason, ExitReason::Killed); + }, + ); } struct Stop; @@ -257,37 +291,45 @@ impl crate::tasks::Handler for Stopping { /// still finishes, and the linked survivor receives the new reason. #[test] fn late_link_failure_relabels_an_already_requested_normal_stop() { - let rt = runtime(); - let (entered_tx, entered_rx) = spawned_rt::tasks::oneshot::channel(); - let (release_tx, release_rx) = spawned_rt::tasks::oneshot::channel(); - let (stopping, failing, survivor) = { - let _entered = rt.enter(); - ( - Stopping { - entered: Some(entered_tx), - release: Some(release_rx), - } - .start(), - Idler.start(), - Idler.start(), - ) - }; - stopping.context().link(&failing.child_handle()); - stopping.context().link(&survivor.child_handle()); - stopping.send(Stop).unwrap(); - rt.block_on(entered_rx).unwrap(); - failing.send(Crash).unwrap(); - assert!(matches!( - rt.block_on(failing.wait_exit()), - ExitReason::Panic(_) - )); - release_tx.send(()).unwrap(); - let relabelled = rt.block_on(stopping.wait_exit()); - assert!( - matches!(&relabelled, ExitReason::Panic(m) if m.contains("linked failure")), - "got {relabelled:?}" + crate::test_wait::bounded( + concat!( + module_path!(), + "::late_link_failure_relabels_an_already_requested_normal_stop" + ), + move || { + let rt = runtime(); + let (entered_tx, entered_rx) = spawned_rt::tasks::oneshot::channel(); + let (release_tx, release_rx) = spawned_rt::tasks::oneshot::channel(); + let (stopping, failing, survivor) = { + let _entered = rt.enter(); + ( + Stopping { + entered: Some(entered_tx), + release: Some(release_rx), + } + .start(), + Idler.start(), + Idler.start(), + ) + }; + stopping.context().link(&failing.child_handle()); + stopping.context().link(&survivor.child_handle()); + stopping.send(Stop).unwrap(); + rt.block_on(entered_rx).unwrap(); + failing.send(Crash).unwrap(); + assert!(matches!( + rt.block_on(failing.wait_exit()), + ExitReason::Panic(_) + )); + release_tx.send(()).unwrap(); + let relabelled = rt.block_on(stopping.wait_exit()); + assert!( + matches!(&relabelled, ExitReason::Panic(m) if m.contains("linked failure")), + "got {relabelled:?}" + ); + assert_eq!(rt.block_on(survivor.wait_exit()), relabelled); + }, ); - assert_eq!(rt.block_on(survivor.wait_exit()), relabelled); } struct LinkedAtStart { @@ -307,280 +349,325 @@ impl Actor for LinkedAtStart { } #[test] fn start_linked_registers_before_async_started() { - let rt = tokio::runtime::Builder::new_multi_thread() - .worker_threads(2) - .enable_all() - .build() - .unwrap(); - let parent = { - let _entered = rt.enter(); - Idler.start() - }; - for _ in 0..32 { - let (tx, rx) = spawned_rt::tasks::oneshot::channel(); - let child = { - let _entered = rt.enter(); - LinkedAtStart { - parent: parent.id(), - seen: Some(tx), + crate::test_wait::bounded( + concat!( + module_path!(), + "::start_linked_registers_before_async_started" + ), + move || { + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let parent = { + let _entered = rt.enter(); + Idler.start() + }; + for _ in 0..32 { + let (tx, rx) = spawned_rt::tasks::oneshot::channel(); + let child = { + let _entered = rt.enter(); + LinkedAtStart { + parent: parent.id(), + seen: Some(tx), + } + .start_linked(&parent.context()) + }; + assert!(rt.block_on(rx).unwrap()); + assert_eq!(rt.block_on(child.wait_exit()), ExitReason::Normal); } - .start_linked(&parent.context()) - }; - assert!(rt.block_on(rx).unwrap()); - assert_eq!(rt.block_on(child.wait_exit()), ExitReason::Normal); - } - parent.child_handle().stop(); - rt.block_on(parent.wait_exit()); + parent.child_handle().stop(); + rt.block_on(parent.wait_exit()); + }, + ); } #[test] fn never_polled_monitor_owner_releases_waiter_and_refuses_late_registration() { - #[cfg(feature = "backend-experiments")] - { - let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); - if selector == "sharded" - || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) - .is_some() - { - return; - } - } - let owner_rt = runtime(); - let survivor = runtime(); - let (tx, _rx) = spawned_rt::tasks::oneshot::channel(); - let owner = { - let _entered = owner_rt.enter(); - MonitorObserver(Some(tx)).start() - }; - let target = { - let _entered = survivor.enter(); - Idler.start() - }; - let handle = target.child_handle(); - let probe = handle.lifetime_probe(); - { - let _entered = survivor.enter(); - owner.context().monitor(&handle); - } - drop(handle); - drop(owner_rt); - survivor.block_on(async { - tokio::time::timeout(Duration::from_secs(1), async { - while probe.upgrade().is_some() { - tokio::task::yield_now().await; + crate::test_wait::bounded( + concat!( + module_path!(), + "::never_polled_monitor_owner_releases_waiter_and_refuses_late_registration" + ), + move || { + #[cfg(feature = "backend-experiments")] + { + let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); + if selector == "sharded" + || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) + .is_some() + { + return; + } } - }) - .await - .unwrap(); - }); - let handle = target.child_handle(); - let probe = handle.lifetime_probe(); - { - let _entered = survivor.enter(); - owner.context().monitor(&handle); - } - drop(handle); - assert!( - probe.upgrade().is_none(), - "dead owner retained a new monitor waiter" + let owner_rt = runtime(); + let survivor = runtime(); + let (tx, _rx) = spawned_rt::tasks::oneshot::channel(); + let owner = { + let _entered = owner_rt.enter(); + MonitorObserver(Some(tx)).start() + }; + let target = { + let _entered = survivor.enter(); + Idler.start() + }; + let handle = target.child_handle(); + let probe = handle.lifetime_probe(); + { + let _entered = survivor.enter(); + owner.context().monitor(&handle); + } + drop(handle); + drop(owner_rt); + survivor.block_on(async { + tokio::time::timeout(Duration::from_secs(1), async { + while probe.upgrade().is_some() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + }); + let handle = target.child_handle(); + let probe = handle.lifetime_probe(); + { + let _entered = survivor.enter(); + owner.context().monitor(&handle); + } + drop(handle); + assert!( + probe.upgrade().is_none(), + "dead owner retained a new monitor waiter" + ); + target.child_handle().stop(); + survivor.block_on(target.wait_exit()); + assert_eq!(owner.child_handle().exit_reason(), Some(ExitReason::Killed)); + }, ); - target.child_handle().stop(); - survivor.block_on(target.wait_exit()); - assert_eq!(owner.child_handle().exit_reason(), Some(ExitReason::Killed)); } #[test] fn linked_runtime_abort_is_trappable() { - #[cfg(feature = "backend-experiments")] - { - let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); - if selector == "sharded" - || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) - .is_some() - { - return; - } - } - let owner = runtime(); - let survivor = runtime(); - let events = Arc::new(Mutex::new(Vec::new())); - let target = { - let _entered = owner.enter(); - Idler.start() - }; - let peer = { - let _entered = survivor.enter(); - Observer(events.clone()).start() - }; - peer.context().trap_exit(true); - peer.context().link(&target.child_handle()); - drop(owner); - survivor.block_on(async { - tokio::time::timeout(crate::test_wait::WAIT, async { - while events.lock().unwrap().is_empty() { - tokio::time::sleep(Duration::from_millis(1)).await; + crate::test_wait::bounded( + concat!(module_path!(), "::linked_runtime_abort_is_trappable"), + move || { + #[cfg(feature = "backend-experiments")] + { + let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); + if selector == "sharded" + || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) + .is_some() + { + return; + } } - }) - .await - .unwrap(); - }); - assert!(peer.child_handle().is_alive()); - assert_eq!( - *events.lock().unwrap(), - vec![Exit { - from: target.id(), - reason: ExitReason::Killed - }] + let owner = runtime(); + let survivor = runtime(); + let events = Arc::new(Mutex::new(Vec::new())); + let target = { + let _entered = owner.enter(); + Idler.start() + }; + let peer = { + let _entered = survivor.enter(); + Observer(events.clone()).start() + }; + peer.context().trap_exit(true); + peer.context().link(&target.child_handle()); + drop(owner); + survivor.block_on(async { + tokio::time::timeout(crate::test_wait::WAIT, async { + while events.lock().unwrap().is_empty() { + tokio::time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .unwrap(); + }); + assert!(peer.child_handle().is_alive()); + assert_eq!( + *events.lock().unwrap(), + vec![Exit { + from: target.id(), + reason: ExitReason::Killed + }] + ); + peer.child_handle().stop(); + assert_eq!(survivor.block_on(peer.wait_exit()), ExitReason::Normal); + }, ); - peer.child_handle().stop(); - assert_eq!(survivor.block_on(peer.wait_exit()), ExitReason::Normal); } #[test] fn never_polled_completion_waits_for_actor_payload_destruction() { - #[cfg(feature = "backend-experiments")] - { - let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); - if selector == "sharded" - || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) - .is_some() - { - return; - } - } - struct BlockingDrop { - entered: std::sync::mpsc::Sender<()>, - release: std::sync::mpsc::Receiver<()>, - } - impl Actor for BlockingDrop {} - impl Drop for BlockingDrop { - fn drop(&mut self) { - let _ = self.entered.send(()); - let _ = self.release.recv(); - } - } - let owner = runtime(); - let (entered_tx, entered_rx) = std::sync::mpsc::channel(); - let (release_tx, release_rx) = std::sync::mpsc::channel(); - let actor = { - let _entered = owner.enter(); - BlockingDrop { - entered: entered_tx, - release: release_rx, - } - .start() - }; - let shutdown = std::thread::spawn(move || drop(owner)); - let entered = entered_rx.recv_timeout(Duration::from_secs(2)); - let before = actor.child_handle().exit_reason(); - let _ = release_tx.send(()); - shutdown.join().unwrap(); - entered.unwrap(); - assert_eq!( - before, None, - "completion published while the actor's destructor was still running" + crate::test_wait::bounded( + concat!( + module_path!(), + "::never_polled_completion_waits_for_actor_payload_destruction" + ), + move || { + #[cfg(feature = "backend-experiments")] + { + let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); + if selector == "sharded" + || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) + .is_some() + { + return; + } + } + struct BlockingDrop { + entered: std::sync::mpsc::Sender<()>, + release: std::sync::mpsc::Receiver<()>, + } + impl Actor for BlockingDrop {} + impl Drop for BlockingDrop { + fn drop(&mut self) { + let _ = self.entered.send(()); + let _ = self.release.recv(); + } + } + let owner = runtime(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let actor = { + let _entered = owner.enter(); + BlockingDrop { + entered: entered_tx, + release: release_rx, + } + .start() + }; + let shutdown = std::thread::spawn(move || drop(owner)); + let entered = entered_rx.recv_timeout(Duration::from_secs(2)); + let before = actor.child_handle().exit_reason(); + let _ = release_tx.send(()); + shutdown.join().unwrap(); + entered.unwrap(); + assert_eq!( + before, None, + "completion published while the actor's destructor was still running" + ); + assert_eq!(actor.child_handle().exit_reason(), Some(ExitReason::Killed)); + }, ); - assert_eq!(actor.child_handle().exit_reason(), Some(ExitReason::Killed)); } #[test] fn linked_runtime_abort_cancels_an_idle_threads_actor() { - #[cfg(feature = "backend-experiments")] - { - let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); - if selector == "sharded" - || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) - .is_some() - { - return; - } - } - use crate::threads::{Actor as ThreadActor, ActorStart as ThreadStart}; - struct ThreadPeer; - impl ThreadActor for ThreadPeer {} - impl crate::threads::Handler for ThreadPeer { - fn handle( - &mut self, - _: Ping, - reply: crate::reply::PendingReply<'_, ()>, - _: &crate::threads::Context, - ) { - reply.reply(()); - } - } - let owner = runtime(); - let waiter = runtime(); - let target = { - let _entered = owner.enter(); - Idler.start() - }; - let peer = ThreadPeer.start(); - peer.context().link(&target.child_handle()); - drop(owner); - let result = waiter.block_on(async { - tokio::time::timeout( - Duration::from_secs(2), - peer.child_handle().wait_exit_async(), - ) - .await - }); - peer.child_handle().stop(); - if result.is_err() { - let _ = peer.send(Ping); // Wake cleanup even if the cancellation callback regresses. - } - peer.child_handle().wait_exit_blocking(); - assert_eq!(result, Ok(ExitReason::Killed)); + crate::test_wait::bounded( + concat!( + module_path!(), + "::linked_runtime_abort_cancels_an_idle_threads_actor" + ), + move || { + #[cfg(feature = "backend-experiments")] + { + let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); + if selector == "sharded" + || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) + .is_some() + { + return; + } + } + use crate::threads::{Actor as ThreadActor, ActorStart as ThreadStart}; + struct ThreadPeer; + impl ThreadActor for ThreadPeer {} + impl crate::threads::Handler for ThreadPeer { + fn handle( + &mut self, + _: Ping, + reply: crate::reply::PendingReply<'_, ()>, + _: &crate::threads::Context, + ) { + reply.reply(()); + } + } + let owner = runtime(); + let waiter = runtime(); + let target = { + let _entered = owner.enter(); + Idler.start() + }; + let peer = ThreadPeer.start(); + peer.context().link(&target.child_handle()); + drop(owner); + let result = waiter.block_on(async { + tokio::time::timeout( + Duration::from_secs(2), + peer.child_handle().wait_exit_async(), + ) + .await + }); + peer.child_handle().stop(); + if result.is_err() { + let _ = peer.send(Ping); // Wake cleanup even if the cancellation callback regresses. + } + peer.child_handle().wait_exit_blocking(); + assert_eq!(result, Ok(ExitReason::Killed)); + }, + ); } #[test] fn dropping_runtime_with_large_polled_link_cycle_reclaims_every_actor() { - #[cfg(feature = "backend-experiments")] - { - let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); - if selector == "sharded" - || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) - .is_some() - { - return; - } - } - use std::sync::atomic::{AtomicUsize, Ordering}; - struct Node(Arc); - impl Actor for Node {} - impl crate::tasks::Handler for Node { - async fn handle( - &mut self, - _: Ping, - reply: crate::reply::PendingReply<'_, ()>, - _: &Context, - ) { - reply.reply(()); - } - } - impl Drop for Node { - fn drop(&mut self) { - self.0.fetch_add(1, Ordering::SeqCst); - } - } - let owner = runtime(); - let dropped = Arc::new(AtomicUsize::new(0)); - let actors: Vec<_> = { - let _entered = owner.enter(); - (0..4096).map(|_| Node(dropped.clone()).start()).collect() - }; - for i in 0..actors.len() { - actors[i] - .context() - .link(&actors[(i + 1) % actors.len()].child_handle()); - } - owner.block_on(async { - for actor in &actors { - actor.request(Ping).await.unwrap(); - } - }); - drop(owner); - assert_eq!(dropped.load(Ordering::SeqCst), actors.len()); - assert!(actors - .iter() - .all(|a| a.child_handle().exit_reason() == Some(ExitReason::Killed))); + crate::test_wait::bounded( + concat!( + module_path!(), + "::dropping_runtime_with_large_polled_link_cycle_reclaims_every_actor" + ), + move || { + #[cfg(feature = "backend-experiments")] + { + let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); + if selector == "sharded" + || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) + .is_some() + { + return; + } + } + use std::sync::atomic::{AtomicUsize, Ordering}; + struct Node(Arc); + impl Actor for Node {} + impl crate::tasks::Handler for Node { + async fn handle( + &mut self, + _: Ping, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } + } + impl Drop for Node { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + let owner = runtime(); + let dropped = Arc::new(AtomicUsize::new(0)); + let actors: Vec<_> = { + let _entered = owner.enter(); + (0..4096).map(|_| Node(dropped.clone()).start()).collect() + }; + for i in 0..actors.len() { + actors[i] + .context() + .link(&actors[(i + 1) % actors.len()].child_handle()); + } + owner.block_on(async { + for actor in &actors { + actor.request(Ping).await.unwrap(); + } + }); + drop(owner); + assert_eq!(dropped.load(Ordering::SeqCst), actors.len()); + assert!(actors + .iter() + .all(|a| a.child_handle().exit_reason() == Some(ExitReason::Killed))); + }, + ); } diff --git a/concurrency/src/mailbox.rs b/concurrency/src/mailbox.rs index dc1ba219..044e7e3a 100644 --- a/concurrency/src/mailbox.rs +++ b/concurrency/src/mailbox.rs @@ -300,152 +300,186 @@ mod tests { #[test] fn accepted_sends_are_drained_even_when_senders_outlive_receiver() { - struct Item { - dropped: Arc, - accepted: bool, - } - impl Drop for Item { - fn drop(&mut self) { - if self.accepted { - self.dropped.fetch_add(1, Ordering::SeqCst); + crate::test_wait::bounded( + concat!( + module_path!(), + "::accepted_sends_are_drained_even_when_senders_outlive_receiver" + ), + move || { + struct Item { + dropped: Arc, + accepted: bool, } - } - } - macro_rules! check { - ($channel:path) => { - for _ in 0..50 { - let (sender, receiver) = $channel(); - let accepted = Arc::new(AtomicUsize::new(1)); - let dropped = Arc::new(AtomicUsize::new(0)); - assert!(sender - .send(Item { - dropped: dropped.clone(), - accepted: true - }) - .is_ok()); - let gate = Arc::new(std::sync::Barrier::new(3)); - let publishers: Vec<_> = (0..2) - .map(|_| { - let sender = sender.clone(); - let accepted = accepted.clone(); - let dropped = dropped.clone(); - let gate = gate.clone(); - std::thread::spawn(move || { - gate.wait(); - for _ in 0..64 { - match sender.send(Item { - dropped: dropped.clone(), - accepted: true, - }) { - Ok(()) => { - accepted.fetch_add(1, Ordering::SeqCst); - } - Err(mut rejected) => { - rejected.0.accepted = false; - } - } - } - }) - }) - .collect(); - gate.wait(); - drop(receiver); - for publisher in publishers { - publisher.join().unwrap(); + impl Drop for Item { + fn drop(&mut self) { + if self.accepted { + self.dropped.fetch_add(1, Ordering::SeqCst); + } } - assert_eq!( - accepted.load(Ordering::SeqCst), - dropped.load(Ordering::SeqCst) - ); - assert!(sender.failures().first().is_none()); } - }; - } - check!(threads::channel); - #[cfg(feature = "async")] - check!(tasks::channel); + macro_rules! check { + ($channel:path) => { + for _ in 0..50 { + let (sender, receiver) = $channel(); + let accepted = Arc::new(AtomicUsize::new(1)); + let dropped = Arc::new(AtomicUsize::new(0)); + assert!(sender + .send(Item { + dropped: dropped.clone(), + accepted: true + }) + .is_ok()); + let gate = Arc::new(std::sync::Barrier::new(3)); + let publishers: Vec<_> = (0..2) + .map(|_| { + let sender = sender.clone(); + let accepted = accepted.clone(); + let dropped = dropped.clone(); + let gate = gate.clone(); + std::thread::spawn(move || { + gate.wait(); + for _ in 0..64 { + match sender.send(Item { + dropped: dropped.clone(), + accepted: true, + }) { + Ok(()) => { + accepted.fetch_add(1, Ordering::SeqCst); + } + Err(mut rejected) => { + rejected.0.accepted = false; + } + } + } + }) + }) + .collect(); + gate.wait(); + drop(receiver); + for publisher in publishers { + publisher.join().unwrap(); + } + assert_eq!( + accepted.load(Ordering::SeqCst), + dropped.load(Ordering::SeqCst) + ); + assert!(sender.failures().first().is_none()); + } + }; + } + check!(threads::channel); + #[cfg(feature = "async")] + check!(tasks::channel); + }, + ); } #[test] fn close_waits_for_publication_and_rejects_late_senders() { - let (sender, receiver) = threads::channel(); - let held = sender.admission.enter().unwrap(); - let (done, completion) = mpsc::channel(); - let owner = std::thread::spawn(move || { - drop(receiver); - done.send(()).unwrap(); - }); - let deadline = std::time::Instant::now() + Duration::from_secs(5); - while !sender.admission.gate.is_closed() { - assert!(std::time::Instant::now() < deadline, "close did not start"); - std::thread::yield_now(); - } - assert!(sender.send(1).is_err()); - assert!(completion.try_recv().is_err()); - // Model a producer paused between reservation and raw publication. - sender.inner.send(2).unwrap(); - drop(held); - completion.recv_timeout(Duration::from_secs(5)).unwrap(); - owner.join().unwrap(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::close_waits_for_publication_and_rejects_late_senders" + ), + move || { + let (sender, receiver) = threads::channel(); + let held = sender.admission.enter().unwrap(); + let (done, completion) = mpsc::channel(); + let owner = std::thread::spawn(move || { + drop(receiver); + done.send(()).unwrap(); + }); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !sender.admission.gate.is_closed() { + assert!(std::time::Instant::now() < deadline, "close did not start"); + std::thread::yield_now(); + } + assert!(sender.send(1).is_err()); + assert!(completion.try_recv().is_err()); + // Model a producer paused between reservation and raw publication. + sender.inner.send(2).unwrap(); + drop(held); + completion.recv_timeout(Duration::from_secs(5)).unwrap(); + owner.join().unwrap(); + }, + ); } #[cfg(feature = "async")] #[tokio::test] async fn compact_async_senders_close_after_their_last_owner() { - let (send, mut receive) = tasks::channel(); - let other = send.clone(); - send.send(1).unwrap(); - drop(send); - assert_eq!(receive.recv().await, Some(1)); - other.send(2).unwrap(); - drop(other); - assert_eq!(receive.recv().await, Some(2)); - assert_eq!( - tokio::time::timeout(Duration::from_secs(5), receive.recv()) - .await - .unwrap(), - None - ); + crate::test_wait::within( + concat!( + module_path!(), + "::compact_async_senders_close_after_their_last_owner" + ), + async move { + let (send, mut receive) = tasks::channel(); + let other = send.clone(); + send.send(1).unwrap(); + drop(send); + assert_eq!(receive.recv().await, Some(1)); + other.send(2).unwrap(); + drop(other); + assert_eq!(receive.recv().await, Some(2)); + assert_eq!( + tokio::time::timeout(Duration::from_secs(5), receive.recv()) + .await + .unwrap(), + None + ); + }, + ) + .await; } #[cfg(feature = "async")] #[tokio::test] async fn unpolled_tracked_unwind_drains_before_releasing_lease() { - struct State; - impl Drop for State { - fn drop(&mut self) { - panic!("state destructor"); - } - } - struct Bomb(Arc); - impl Drop for Bomb { - fn drop(&mut self) { - self.0.fetch_add(1, Ordering::SeqCst); - panic!("queued destructor"); - } - } - let system = crate::system::ActorSystem::new().unwrap(); - let group = system.async_group("unpolled-tracked-drop", 1).unwrap(); - let (scope, lease) = group.reserve(None).unwrap(); - let (sender, receiver) = tasks::channel(); - let dropped = Arc::new(AtomicUsize::new(0)); - for _ in 0..2 { - assert!(sender.send(Bomb(dropped.clone())).is_ok()); - } - let payload = (State, receiver); - let future = async move { - drop(payload); - }; - let tracked = crate::system::Tracked { future, lease }; - assert!(catch_unwind(AssertUnwindSafe(|| drop(tracked))).is_err()); - assert_eq!(dropped.load(Ordering::SeqCst), 2); - assert_eq!( - sender.failures().first().as_deref(), - Some("queued destructor") - ); - tokio::time::timeout(Duration::from_secs(5), scope.joined()) - .await - .unwrap(); - assert!(system.shutdown(Duration::from_secs(5)).await.complete); + crate::test_wait::within( + concat!( + module_path!(), + "::unpolled_tracked_unwind_drains_before_releasing_lease" + ), + async move { + struct State; + impl Drop for State { + fn drop(&mut self) { + panic!("state destructor"); + } + } + struct Bomb(Arc); + impl Drop for Bomb { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + panic!("queued destructor"); + } + } + let system = crate::system::ActorSystem::new().unwrap(); + let group = system.async_group("unpolled-tracked-drop", 1).unwrap(); + let (scope, lease) = group.reserve(None).unwrap(); + let (sender, receiver) = tasks::channel(); + let dropped = Arc::new(AtomicUsize::new(0)); + for _ in 0..2 { + assert!(sender.send(Bomb(dropped.clone())).is_ok()); + } + let payload = (State, receiver); + let future = async move { + drop(payload); + }; + let tracked = crate::system::Tracked { future, lease }; + assert!(catch_unwind(AssertUnwindSafe(|| drop(tracked))).is_err()); + assert_eq!(dropped.load(Ordering::SeqCst), 2); + assert_eq!( + sender.failures().first().as_deref(), + Some("queued destructor") + ); + tokio::time::timeout(Duration::from_secs(5), scope.joined()) + .await + .unwrap(); + assert!(system.shutdown(Duration::from_secs(5)).await.complete); + }, + ) + .await; } } diff --git a/concurrency/src/native_link_regression_tests.rs b/concurrency/src/native_link_regression_tests.rs index 07890261..6fbc625d 100644 --- a/concurrency/src/native_link_regression_tests.rs +++ b/concurrency/src/native_link_regression_tests.rs @@ -2,38 +2,46 @@ use crate::{link, ActorId, ExitReason}; use std::time::Duration; #[test] fn start_linked_registers_before_threads_started() { - use crate::threads::{ - Actor as ThreadActor, ActorStart as ThreadStart, Context as ThreadContext, - }; - struct Parent; - impl ThreadActor for Parent {} - struct Child { - parent: ActorId, - seen: std::sync::mpsc::Sender, - } - impl ThreadActor for Child { - fn started(&mut self, ctx: &ThreadContext) { - let h = ctx.actor_ref().child_handle(); - self.seen - .send(link::has_link(h.links(), self.parent)) - .unwrap(); - ctx.stop(); - } - } - let parent = Parent.start(); - for _ in 0..32 { - let (tx, rx) = std::sync::mpsc::channel(); - let child = Child { - parent: parent.id(), - seen: tx, - } - .start_linked(&parent.context()); - assert!(rx.recv_timeout(Duration::from_secs(1)).unwrap()); - assert_eq!( - child.child_handle().wait_exit_blocking(), - ExitReason::Normal - ); - } - parent.child_handle().stop(); - parent.child_handle().wait_exit_blocking(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::start_linked_registers_before_threads_started" + ), + move || { + use crate::threads::{ + Actor as ThreadActor, ActorStart as ThreadStart, Context as ThreadContext, + }; + struct Parent; + impl ThreadActor for Parent {} + struct Child { + parent: ActorId, + seen: std::sync::mpsc::Sender, + } + impl ThreadActor for Child { + fn started(&mut self, ctx: &ThreadContext) { + let h = ctx.actor_ref().child_handle(); + self.seen + .send(link::has_link(h.links(), self.parent)) + .unwrap(); + ctx.stop(); + } + } + let parent = Parent.start(); + for _ in 0..32 { + let (tx, rx) = std::sync::mpsc::channel(); + let child = Child { + parent: parent.id(), + seen: tx, + } + .start_linked(&parent.context()); + assert!(rx.recv_timeout(Duration::from_secs(1)).unwrap()); + assert_eq!( + child.child_handle().wait_exit_blocking(), + ExitReason::Normal + ); + } + parent.child_handle().stop(); + parent.child_handle().wait_exit_blocking(); + }, + ); } diff --git a/concurrency/src/reply/pending.rs b/concurrency/src/reply/pending.rs index 94b70c36..1cb1c9c4 100644 --- a/concurrency/src/reply/pending.rs +++ b/concurrency/src/reply/pending.rs @@ -155,38 +155,43 @@ mod tests { #[test] fn bounded_with_duplicates_and_purge() { - let mut table = PendingReplies::with_capacity(2); - let (a, ra) = token(); - let (b, rb) = token(); - let (c, rc) = token(); - let (dup, rdup) = token(); - table.insert(1, a).unwrap(); - table.insert(2, b).unwrap(); - let full = table.insert(3, c).unwrap_err(); - assert!(matches!(full, InsertError::Full(_))); - full.into_inner().overloaded(); - assert!(matches!(rc.recv(), Err(ActorError::Overloaded))); - let duplicate = table.insert(1, dup).unwrap_err(); - assert!(matches!(duplicate, InsertError::Duplicate(_))); - drop(duplicate); - assert!(matches!(rdup.recv(), Err(ActorError::ReplyDropped))); - // A caller that left frees its entry on the next insert. - drop(rb); - assert_eq!( - table.tokens.len(), - 2, - "terminal tokens stay until an insert" + crate::test_wait::bounded( + concat!(module_path!(), "::bounded_with_duplicates_and_purge"), + move || { + let mut table = PendingReplies::with_capacity(2); + let (a, ra) = token(); + let (b, rb) = token(); + let (c, rc) = token(); + let (dup, rdup) = token(); + table.insert(1, a).unwrap(); + table.insert(2, b).unwrap(); + let full = table.insert(3, c).unwrap_err(); + assert!(matches!(full, InsertError::Full(_))); + full.into_inner().overloaded(); + assert!(matches!(rc.recv(), Err(ActorError::Overloaded))); + let duplicate = table.insert(1, dup).unwrap_err(); + assert!(matches!(duplicate, InsertError::Duplicate(_))); + drop(duplicate); + assert!(matches!(rdup.recv(), Err(ActorError::ReplyDropped))); + // A caller that left frees its entry on the next insert. + drop(rb); + assert_eq!( + table.tokens.len(), + 2, + "terminal tokens stay until an insert" + ); + assert_eq!(table.len(), 1); + let (d, rd) = token(); + table.insert(4, d).unwrap(); + assert_eq!(table.tokens.len(), 2); + assert_eq!(table.len(), 2); + assert!(table.take(&2).is_none()); + table.take(&1).unwrap().reply(10); + table.take(&4).unwrap().reply(40); + assert_eq!(ra.recv().unwrap(), 10); + assert_eq!(rd.recv().unwrap(), 40); + }, ); - assert_eq!(table.len(), 1); - let (d, rd) = token(); - table.insert(4, d).unwrap(); - assert_eq!(table.tokens.len(), 2); - assert_eq!(table.len(), 2); - assert!(table.take(&2).is_none()); - table.take(&1).unwrap().reply(10); - table.take(&4).unwrap().reply(40); - assert_eq!(ra.recv().unwrap(), 10); - assert_eq!(rd.recv().unwrap(), 40); } /// #261: draining follows insertion order, so two tables built the /// same way (in the old `HashMap`, two different `RandomState` seeds) @@ -237,23 +242,28 @@ mod tests { /// caller can answer the token for any reason, known or not. #[test] fn a_refusal_gives_its_kind_and_its_token() { - let mut table = PendingReplies::with_capacity(1); - let (a, ra) = token(); - let (dup, rdup) = token(); - let (full, rfull) = token(); - table.insert(1, a).unwrap(); - let refused = table.insert(1, dup).unwrap_err(); - assert_eq!(refused.kind(), InsertErrorKind::Duplicate); - let (kind, token) = refused.into_parts(); - assert_eq!(kind, InsertErrorKind::Duplicate); - token.overloaded(); - assert!(matches!(rdup.recv(), Err(ActorError::Overloaded))); - let refused = table.insert(2, full).unwrap_err(); - let (kind, token) = refused.into_parts(); - assert_eq!(kind, InsertErrorKind::Full); - token.reply(9); - assert_eq!(rfull.recv().unwrap(), 9); - table.take(&1).unwrap().reply(1); - assert_eq!(ra.recv().unwrap(), 1); + crate::test_wait::bounded( + concat!(module_path!(), "::a_refusal_gives_its_kind_and_its_token"), + move || { + let mut table = PendingReplies::with_capacity(1); + let (a, ra) = token(); + let (dup, rdup) = token(); + let (full, rfull) = token(); + table.insert(1, a).unwrap(); + let refused = table.insert(1, dup).unwrap_err(); + assert_eq!(refused.kind(), InsertErrorKind::Duplicate); + let (kind, token) = refused.into_parts(); + assert_eq!(kind, InsertErrorKind::Duplicate); + token.overloaded(); + assert!(matches!(rdup.recv(), Err(ActorError::Overloaded))); + let refused = table.insert(2, full).unwrap_err(); + let (kind, token) = refused.into_parts(); + assert_eq!(kind, InsertErrorKind::Full); + token.reply(9); + assert_eq!(rfull.recv().unwrap(), 9); + table.take(&1).unwrap().reply(1); + assert_eq!(ra.recv().unwrap(), 1); + }, + ); } } diff --git a/concurrency/src/response.rs b/concurrency/src/response.rs index 1960b25c..f3732bbe 100644 --- a/concurrency/src/response.rs +++ b/concurrency/src/response.rs @@ -284,25 +284,35 @@ mod tests { #[test] #[cfg(feature = "async")] fn future_resolves_from_receiver() { - let rt = spawned_rt::tasks::Runtime::new().unwrap(); - rt.block_on(async { - let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); - let resp: Response = Response::from(Ok(rx)); - reply.reply(99); - assert_eq!(resp.await.unwrap(), 99); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::future_resolves_from_receiver"), + move || { + let rt = spawned_rt::tasks::Runtime::new().unwrap(); + rt.block_on(async { + let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); + let resp: Response = Response::from(Ok(rx)); + reply.reply(99); + assert_eq!(resp.await.unwrap(), 99); + }); + }, + ); } #[test] #[cfg(feature = "async")] fn future_err_on_dropped_reply() { - let rt = spawned_rt::tasks::Runtime::new().unwrap(); - rt.block_on(async { - let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); - let resp: Response = Response::from(Ok(rx)); - drop(reply); - assert!(matches!(resp.await, Err(ActorError::ReplyDropped))); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::future_err_on_dropped_reply"), + move || { + let rt = spawned_rt::tasks::Runtime::new().unwrap(); + rt.block_on(async { + let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); + let resp: Response = Response::from(Ok(rx)); + drop(reply); + assert!(matches!(resp.await, Err(ActorError::ReplyDropped))); + }); + }, + ); } /// Review footgun (tasks): `actor.query().unwrap()` used to panic on a @@ -312,34 +322,47 @@ mod tests { #[test] #[cfg(all(feature = "async", not(spawned_sim)))] fn pending_response_waits_off_a_runtime() { - let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); - let answer = std::thread::spawn(move || { - std::thread::sleep(Duration::from_millis(20)); - reply.reply(7); - }); - let resp: Response = Response::from(Ok(rx)); - assert_eq!(resp.unwrap(), 7); - answer.join().unwrap(); - - let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); - drop(reply); - let resp: Response = Response::from(Ok(rx)); - assert!(matches!(resp.into_result(), Err(ActorError::ReplyDropped))); - - let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); - reply.reply(2); - assert_eq!(Response::from(Ok(rx)).map(|x| x * 5).unwrap(), 10); + crate::test_wait::bounded( + concat!(module_path!(), "::pending_response_waits_off_a_runtime"), + move || { + let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); + let answer = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(20)); + reply.reply(7); + }); + let resp: Response = Response::from(Ok(rx)); + assert_eq!(resp.unwrap(), 7); + answer.join().unwrap(); + + let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); + drop(reply); + let resp: Response = Response::from(Ok(rx)); + assert!(matches!(resp.into_result(), Err(ActorError::ReplyDropped))); + + let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); + reply.reply(2); + assert_eq!(Response::from(Ok(rx)).map(|x| x * 5).unwrap(), 10); + }, + ); } #[test] #[cfg(all(feature = "async", not(spawned_sim)))] #[should_panic(expected = "inside an async runtime")] fn pending_response_unwrap_panics_inside_a_runtime() { - let rt = spawned_rt::tasks::Runtime::new().unwrap(); - rt.block_on(async { - let (_reply, rx) = crate::reply::reply_pair::(Deadline::Never); - Response::from(Ok(rx)).unwrap(); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::pending_response_unwrap_panics_inside_a_runtime" + ), + move || { + let rt = spawned_rt::tasks::Runtime::new().unwrap(); + rt.block_on(async { + let (_reply, rx) = crate::reply::reply_pair::(Deadline::Never); + Response::from(Ok(rx)).unwrap(); + }); + }, + ); } #[test] @@ -360,13 +383,18 @@ mod tests { #[test] #[cfg(feature = "async")] fn receiver_times_out_at_its_send_deadline() { - let rt = spawned_rt::tasks::Runtime::new().unwrap(); - rt.block_on(async { - let (reply, rx) = - crate::reply::reply_pair::(Deadline::after(Duration::from_millis(50))); - let resp: Response = Response::from(Ok(rx)); - assert!(matches!(resp.await, Err(ActorError::RequestTimeout))); - assert_eq!(reply.reply(1), crate::reply::Delivery::Expired); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::receiver_times_out_at_its_send_deadline"), + move || { + let rt = spawned_rt::tasks::Runtime::new().unwrap(); + rt.block_on(async { + let (reply, rx) = + crate::reply::reply_pair::(Deadline::after(Duration::from_millis(50))); + let resp: Response = Response::from(Ok(rx)); + assert!(matches!(resp.await, Err(ActorError::RequestTimeout))); + assert_eq!(reply.reply(1), crate::reply::Delivery::Expired); + }); + }, + ); } } diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs index 81666d84..da460669 100644 --- a/concurrency/src/supervision.rs +++ b/concurrency/src/supervision.rs @@ -2639,74 +2639,92 @@ mod policy_tests { } #[tokio::test] async fn failure_classification_is_typed_and_terminal_failure_is_sticky() { - let system = ActorSystem::new().unwrap(); - let group = system.async_group("terminal-order", 1).unwrap(); - let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); - let first = tree - .add(ChildSpec::new("first", &group, |_| async { - Ok::<_, String>(CoordinatorWorker) - })) - .unwrap(); - let second = tree - .add(ChildSpec::new("second", &group, |_| async { - Ok::<_, String>(CoordinatorWorker) - })) - .unwrap(); - first.view.finish(true); - first.view.finish_unstarted(false); // stale non-failed sweep races last - second.view.finish_unstarted(false); - second.view.finish(true); // failed controller races last - assert_eq!(first.snapshot().state, ChildState::Failed); - assert_eq!(second.snapshot().state, ChildState::Failed); - tree.core - .fail("restart budget exhausted by a user-provided message".into()); - assert_eq!( - system::lock(&tree.core.failure).as_ref().unwrap().kind, - FailureKind::Other - ); - drop(tree); - assert!(system.shutdown(Duration::from_secs(10)).await.complete); + crate::test_wait::within( + concat!( + module_path!(), + "::failure_classification_is_typed_and_terminal_failure_is_sticky" + ), + async move { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("terminal-order", 1).unwrap(); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let first = tree + .add(ChildSpec::new("first", &group, |_| async { + Ok::<_, String>(CoordinatorWorker) + })) + .unwrap(); + let second = tree + .add(ChildSpec::new("second", &group, |_| async { + Ok::<_, String>(CoordinatorWorker) + })) + .unwrap(); + first.view.finish(true); + first.view.finish_unstarted(false); // stale non-failed sweep races last + second.view.finish_unstarted(false); + second.view.finish(true); // failed controller races last + assert_eq!(first.snapshot().state, ChildState::Failed); + assert_eq!(second.snapshot().state, ChildState::Failed); + tree.core + .fail("restart budget exhausted by a user-provided message".into()); + assert_eq!( + system::lock(&tree.core.failure).as_ref().unwrap().kind, + FailureKind::Other + ); + drop(tree); + assert!(system.shutdown(Duration::from_secs(10)).await.complete); + }, + ) + .await; } #[tokio::test] async fn initial_placement_failure_has_deterministic_failed_slots() { - use crate::tasks::ActorStart; - for _ in 0..8 { - let system = ActorSystem::new().unwrap(); - let group = system.thread_group("placement-failure", 1).unwrap(); - let occupied = CoordinatorWorker.start_on(&group).unwrap(); - rt::timeout(Duration::from_secs(10), occupied.ready()) - .await - .unwrap() - .unwrap(); - let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); - let first = tree - .add(ChildSpec::new("first", &group, |_| async { - Ok::<_, String>(CoordinatorWorker) - })) - .unwrap(); - let second = tree - .add(ChildSpec::new("second", &group, |_| async { - Ok::<_, String>(CoordinatorWorker) - })) - .unwrap(); - let core = tree.core.clone(); - assert!(tree.start().await.is_err()); - assert!(rt::timeout( - Duration::from_secs(10), - system::wait_true(core.done.subscribe()) - ) - .await - .unwrap()); - for child in [first, second] { - assert_eq!(child.snapshot().generation, 0); - assert_eq!(child.snapshot().state, ChildState::Failed); - assert!(matches!( - child.current(), - Err(ChildUnavailable(ChildState::Failed)) - )); - } - assert!(system.shutdown(Duration::from_secs(10)).await.complete); - } + crate::test_wait::within( + concat!( + module_path!(), + "::initial_placement_failure_has_deterministic_failed_slots" + ), + async move { + use crate::tasks::ActorStart; + for _ in 0..8 { + let system = ActorSystem::new().unwrap(); + let group = system.thread_group("placement-failure", 1).unwrap(); + let occupied = CoordinatorWorker.start_on(&group).unwrap(); + rt::timeout(Duration::from_secs(10), occupied.ready()) + .await + .unwrap() + .unwrap(); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let first = tree + .add(ChildSpec::new("first", &group, |_| async { + Ok::<_, String>(CoordinatorWorker) + })) + .unwrap(); + let second = tree + .add(ChildSpec::new("second", &group, |_| async { + Ok::<_, String>(CoordinatorWorker) + })) + .unwrap(); + let core = tree.core.clone(); + assert!(tree.start().await.is_err()); + assert!(rt::timeout( + Duration::from_secs(10), + system::wait_true(core.done.subscribe()) + ) + .await + .unwrap()); + for child in [first, second] { + assert_eq!(child.snapshot().generation, 0); + assert_eq!(child.snapshot().state, ChildState::Failed); + assert!(matches!( + child.current(), + Err(ChildUnavailable(ChildState::Failed)) + )); + } + assert!(system.shutdown(Duration::from_secs(10)).await.complete); + } + }, + ) + .await; } struct PanicRunner; impl Runner for PanicRunner { @@ -2718,6 +2736,7 @@ mod policy_tests { impl Actor for CoordinatorWorker {} #[tokio::test] async fn coordinator_fault_joins_launched_controllers_and_helpers() { + crate::test_wait::within(concat!(module_path!(), "::coordinator_fault_joins_launched_controllers_and_helpers"), async move { let system = ActorSystem::new().unwrap(); let group = system .async_group("supervisor-coordinator-fault", 1) @@ -2773,186 +2792,136 @@ mod policy_tests { .supervisor_failures .iter() .any(|e| e.contains("injected supervisor coordinator fault"))); + }).await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn stop_wins_over_late_child_failure_without_trace_serialization() { - let system = ActorSystem::new().unwrap(); - let tree = system.supervisor(SupervisorConfig::default()).unwrap(); - tree.core.close(); - assert!(!tree - .core - .fail_if_open_as("late child failure".into(), || None)); - assert!(system::lock(&tree.core.failure).is_none()); - assert!(tree - .core - .gate - .if_open(|| tree.core.ready.send_replace(true)) - .is_none()); - assert!(!*tree.core.ready.borrow()); - // Cleanup failures remain observable after ordinary admission closure. - tree.core.fail_cleanup("late cleanup failure".into()); - assert_eq!( - system::lock(&tree.core.failure).as_deref(), - Some("late cleanup failure") - ); - drop(tree); - assert!(system.shutdown(Duration::from_secs(5)).await.complete); + crate::test_wait::within( + concat!( + module_path!(), + "::stop_wins_over_late_child_failure_without_trace_serialization" + ), + async move { + let system = ActorSystem::new().unwrap(); + let tree = system.supervisor(SupervisorConfig::default()).unwrap(); + tree.core.close(); + assert!(!tree + .core + .fail_if_open_as("late child failure".into(), || None)); + assert!(system::lock(&tree.core.failure).is_none()); + assert!(tree + .core + .gate + .if_open(|| tree.core.ready.send_replace(true)) + .is_none()); + assert!(!*tree.core.ready.borrow()); + // Cleanup failures remain observable after ordinary admission closure. + tree.core.fail_cleanup("late cleanup failure".into()); + assert_eq!( + system::lock(&tree.core.failure).as_deref(), + Some("late cleanup failure") + ); + drop(tree); + assert!(system.shutdown(Duration::from_secs(5)).await.complete); + }, + ) + .await; } #[test] fn delayed_failure_notifications_are_absorbed_by_the_already_assigned_round() { - let mut queue = RecoveryQueue { - pending: vec![false; 3], - covered: vec![0; 3], - next_id: 1, - }; - assert!(queue.request(1, 0)); - queue.pending.fill(false); - // The coordinator publishes coverage before per-child assignment. - queue.covered[1..].fill(1); - assert!(!queue.request(2, 0)); - assert!(!queue.pending.iter().any(|value| *value)); - // A new failure in a reconstructed generation must still be admitted. - assert!(queue.request(2, 1)); - assert!(queue.request(0, 0)); // unaffected upstream in RestForOne + crate::test_wait::bounded( + concat!( + module_path!(), + "::delayed_failure_notifications_are_absorbed_by_the_already_assigned_round" + ), + move || { + let mut queue = RecoveryQueue { + pending: vec![false; 3], + covered: vec![0; 3], + next_id: 1, + }; + assert!(queue.request(1, 0)); + queue.pending.fill(false); + // The coordinator publishes coverage before per-child assignment. + queue.covered[1..].fill(1); + assert!(!queue.request(2, 0)); + assert!(!queue.pending.iter().any(|value| *value)); + // A new failure in a reconstructed generation must still be admitted. + assert!(queue.request(2, 1)); + assert!(queue.request(0, 0)); // unaffected upstream in RestForOne + }, + ); } #[tokio::test] async fn admission_closure_after_readiness_preserves_fixed_system_order() { - use crate::tasks::ActorStart; - struct Ordered { - id: u32, - log: Arc>>, - entered: Option>, - release: Option>, - } - impl Actor for Ordered { - async fn started(&mut self, ctx: &crate::tasks::Context) { - // v0.8: stopped() records the order only for a trapping - // child (E3, decision 2). - ctx.trap_exit(true); - } - async fn stopped(&mut self, _: &crate::tasks::Context) { - system::lock(&self.log).push(self.id); - if let Some(entered) = self.entered.take() { - let _ = entered.send(()); + crate::test_wait::within( + concat!( + module_path!(), + "::admission_closure_after_readiness_preserves_fixed_system_order" + ), + async move { + use crate::tasks::ActorStart; + struct Ordered { + id: u32, + log: Arc>>, + entered: Option>, + release: Option>, } - if let Some(release) = self.release.take() { - let _ = release.await; + impl Actor for Ordered { + async fn started(&mut self, ctx: &crate::tasks::Context) { + // v0.8: stopped() records the order only for a trapping + // child (E3, decision 2). + ctx.trap_exit(true); + } + async fn stopped(&mut self, _: &crate::tasks::Context) { + system::lock(&self.log).push(self.id); + if let Some(entered) = self.entered.take() { + let _ = entered.send(()); + } + if let Some(release) = self.release.take() { + let _ = release.await; + } + } } - } - } - let limit = Duration::from_secs(30); - let system = ActorSystem::new().unwrap(); - let group = system.thread_group("fixed-order-2-1-2", 5).unwrap(); - let log = Arc::new(Mutex::new(Vec::new())); - for id in 0..2 { - Ordered { - id, - log: log.clone(), - entered: None, - release: None, - } - .start_on(&group) - .unwrap() - .ready() - .await - .unwrap(); - } - let mut builder = system.supervisor(SupervisorConfig::default()).unwrap(); - let child_log = log.clone(); - builder - .add(ChildSpec::new("middle", &group, move |_| { - let log = child_log.clone(); - async move { - Ok::<_, String>(Ordered { - id: 100, - log, + let limit = Duration::from_secs(30); + let system = ActorSystem::new().unwrap(); + let group = system.thread_group("fixed-order-2-1-2", 5).unwrap(); + let log = Arc::new(Mutex::new(Vec::new())); + for id in 0..2 { + Ordered { + id, + log: log.clone(), entered: None, release: None, - }) - } - })) - .unwrap(); - let resume = CancellationToken::new(); - let (passed, after_check) = watch::channel(false); - *system::lock(&builder.core.initial_ready_pause) = Some((resume.clone(), passed)); - let tree = builder.start().await.unwrap(); - Ordered { - id: 200, - log: log.clone(), - entered: None, - release: None, - } - .start_on(&group) - .unwrap() - .ready() - .await - .unwrap(); - let (entered, arrival) = rt::oneshot::channel(); - let (release, gate) = rt::oneshot::channel(); - Ordered { - id: 201, - log: log.clone(), - entered: Some(entered), - release: Some(gate), - } - .start_on(&group) - .unwrap() - .ready() - .await - .unwrap(); - let shutdown = system.shutdown(limit); - tokio::pin!(shutdown); - assert!(futures::poll!(&mut shutdown).is_pending()); - rt::timeout(limit, arrival).await.unwrap().unwrap(); - // The later standalone actor owns the current stop turn. Resume the - // supervisor exactly between readiness and its cancellation wait. - resume.cancel(); - assert!(rt::timeout(limit, system::wait_true(after_check)) - .await - .unwrap()); - assert!( - !tree.core.cancel.is_cancelled(), - "admission closure stole the supervisor's stop turn" - ); - release.send(()).unwrap(); - assert!(rt::timeout(limit, shutdown).await.unwrap().complete); - assert_eq!(*system::lock(&log), vec![201, 200, 100, 1, 0]); - } - - #[tokio::test] - async fn startup_completion_or_drop_after_readiness_preserves_system_order() { - use crate::tasks::ActorStart; - struct Ordered { - id: u32, - log: Arc>>, - entered: Option>, - release: Option>, - } - impl Actor for Ordered { - async fn started(&mut self, ctx: &crate::tasks::Context) { - // v0.8: stopped() records the order only for a trapping - // child (E3, decision 2). - ctx.trap_exit(true); - } - async fn stopped(&mut self, _: &crate::tasks::Context) { - system::lock(&self.log).push(self.id); - if let Some(entered) = self.entered.take() { - let _ = entered.send(()); - } - if let Some(release) = self.release.take() { - let _ = release.await; + } + .start_on(&group) + .unwrap() + .ready() + .await + .unwrap(); } - } - } - for complete_start in [true, false] { - let limit = Duration::from_secs(30); - let system = ActorSystem::new().unwrap(); - let group = system.thread_group("fixed-order-2-1-2", 5).unwrap(); - let log = Arc::new(Mutex::new(Vec::new())); - for id in 0..2 { + let mut builder = system.supervisor(SupervisorConfig::default()).unwrap(); + let child_log = log.clone(); + builder + .add(ChildSpec::new("middle", &group, move |_| { + let log = child_log.clone(); + async move { + Ok::<_, String>(Ordered { + id: 100, + log, + entered: None, + release: None, + }) + } + })) + .unwrap(); + let resume = CancellationToken::new(); + let (passed, after_check) = watch::channel(false); + *system::lock(&builder.core.initial_ready_pause) = Some((resume.clone(), passed)); + let tree = builder.start().await.unwrap(); Ordered { - id, + id: 200, log: log.clone(), entered: None, release: None, @@ -2962,260 +2931,373 @@ mod policy_tests { .ready() .await .unwrap(); - } - let mut builder = system.supervisor(SupervisorConfig::default()).unwrap(); - let child_log = log.clone(); - // Hold construction until start() has been polled once, so the - // first poll is pending however fast the controller runs. - let (construct, gate) = rt::oneshot::channel::<()>(); - let gate = Arc::new(Mutex::new(Some(gate))); - builder - .add(ChildSpec::new("middle", &group, move |_| { - let log = child_log.clone(); - let gate = system::lock(&gate).take(); - async move { - if let Some(gate) = gate { - let _ = gate.await; + let (entered, arrival) = rt::oneshot::channel(); + let (release, gate) = rt::oneshot::channel(); + Ordered { + id: 201, + log: log.clone(), + entered: Some(entered), + release: Some(gate), + } + .start_on(&group) + .unwrap() + .ready() + .await + .unwrap(); + let shutdown = system.shutdown(limit); + tokio::pin!(shutdown); + assert!(futures::poll!(&mut shutdown).is_pending()); + rt::timeout(limit, arrival).await.unwrap().unwrap(); + // The later standalone actor owns the current stop turn. Resume the + // supervisor exactly between readiness and its cancellation wait. + resume.cancel(); + assert!(rt::timeout(limit, system::wait_true(after_check)) + .await + .unwrap()); + assert!( + !tree.core.cancel.is_cancelled(), + "admission closure stole the supervisor's stop turn" + ); + release.send(()).unwrap(); + assert!(rt::timeout(limit, shutdown).await.unwrap().complete); + assert_eq!(*system::lock(&log), vec![201, 200, 100, 1, 0]); + }, + ) + .await; + } + + #[tokio::test] + async fn startup_completion_or_drop_after_readiness_preserves_system_order() { + crate::test_wait::within( + concat!( + module_path!(), + "::startup_completion_or_drop_after_readiness_preserves_system_order" + ), + async move { + use crate::tasks::ActorStart; + struct Ordered { + id: u32, + log: Arc>>, + entered: Option>, + release: Option>, + } + impl Actor for Ordered { + async fn started(&mut self, ctx: &crate::tasks::Context) { + // v0.8: stopped() records the order only for a trapping + // child (E3, decision 2). + ctx.trap_exit(true); + } + async fn stopped(&mut self, _: &crate::tasks::Context) { + system::lock(&self.log).push(self.id); + if let Some(entered) = self.entered.take() { + let _ = entered.send(()); + } + if let Some(release) = self.release.take() { + let _ = release.await; } - Ok::<_, String>(Ordered { - id: 100, - log, + } + } + for complete_start in [true, false] { + let limit = Duration::from_secs(30); + let system = ActorSystem::new().unwrap(); + let group = system.thread_group("fixed-order-2-1-2", 5).unwrap(); + let log = Arc::new(Mutex::new(Vec::new())); + for id in 0..2 { + Ordered { + id, + log: log.clone(), entered: None, release: None, - }) + } + .start_on(&group) + .unwrap() + .ready() + .await + .unwrap(); } - })) - .unwrap(); - let core = builder.core.clone(); - let mut startup = Box::pin(builder.start()); - assert!(futures::poll!(startup.as_mut()).is_pending()); - construct.send(()).unwrap(); - assert!( - rt::timeout(limit, system::wait_true(core.ready.subscribe())) + let mut builder = system.supervisor(SupervisorConfig::default()).unwrap(); + let child_log = log.clone(); + // Hold construction until start() has been polled once, so the + // first poll is pending however fast the controller runs. + let (construct, gate) = rt::oneshot::channel::<()>(); + let gate = Arc::new(Mutex::new(Some(gate))); + builder + .add(ChildSpec::new("middle", &group, move |_| { + let log = child_log.clone(); + let gate = system::lock(&gate).take(); + async move { + if let Some(gate) = gate { + let _ = gate.await; + } + Ok::<_, String>(Ordered { + id: 100, + log, + entered: None, + release: None, + }) + } + })) + .unwrap(); + let core = builder.core.clone(); + let mut startup = Box::pin(builder.start()); + assert!(futures::poll!(startup.as_mut()).is_pending()); + construct.send(()).unwrap(); + assert!( + rt::timeout(limit, system::wait_true(core.ready.subscribe())) + .await + .unwrap() + ); + Ordered { + id: 200, + log: log.clone(), + entered: None, + release: None, + } + .start_on(&group) + .unwrap() + .ready() .await + .unwrap(); + let (entered, arrival) = rt::oneshot::channel(); + let (release, gate) = rt::oneshot::channel(); + Ordered { + id: 201, + log: log.clone(), + entered: Some(entered), + release: Some(gate), + } + .start_on(&group) .unwrap() - ); - Ordered { - id: 200, - log: log.clone(), - entered: None, - release: None, - } - .start_on(&group) - .unwrap() - .ready() - .await - .unwrap(); - let (entered, arrival) = rt::oneshot::channel(); - let (release, gate) = rt::oneshot::channel(); - Ordered { - id: 201, - log: log.clone(), - entered: Some(entered), - release: Some(gate), - } - .start_on(&group) - .unwrap() - .ready() - .await - .unwrap(); - let shutdown = system.shutdown(limit); - tokio::pin!(shutdown); - assert!(futures::poll!(&mut shutdown).is_pending()); - rt::timeout(limit, arrival).await.unwrap().unwrap(); - // Readiness committed, but the caller has not observed start's result. - // Admission now closes while the later actor still owns the stop turn. - if complete_start { - assert!(rt::timeout(limit, startup.as_mut()).await.unwrap().is_ok()); - } else { - drop(startup); - } - assert!( - !core.cancel.is_cancelled(), - "completed readiness was rolled back" - ); - release.send(()).unwrap(); - assert!(rt::timeout(limit, shutdown).await.unwrap().complete); - assert_eq!(*system::lock(&log), vec![201, 200, 100, 1, 0]); - } + .ready() + .await + .unwrap(); + let shutdown = system.shutdown(limit); + tokio::pin!(shutdown); + assert!(futures::poll!(&mut shutdown).is_pending()); + rt::timeout(limit, arrival).await.unwrap().unwrap(); + // Readiness committed, but the caller has not observed start's result. + // Admission now closes while the later actor still owns the stop turn. + if complete_start { + assert!(rt::timeout(limit, startup.as_mut()).await.unwrap().is_ok()); + } else { + drop(startup); + } + assert!( + !core.cancel.is_cancelled(), + "completed readiness was rolled back" + ); + release.send(()).unwrap(); + assert!(rt::timeout(limit, shutdown).await.unwrap().complete); + assert_eq!(*system::lock(&log), vec![201, 200, 100, 1, 0]); + } + }, + ) + .await; } #[tokio::test] async fn startup_success_remains_historical_after_unobserved_ready_then_failure() { - struct Idle; - impl Actor for Idle {} - let limit = Duration::from_secs(30); - let system = ActorSystem::new().unwrap(); - let group = system.async_group("historical-ready-failure", 1).unwrap(); - let mut builder = system - .supervisor(SupervisorConfig { - max_restarts: 0, - ..SupervisorConfig::default() - }) - .unwrap(); - // Hold construction until start() has been polled once, so the - // first poll is pending however fast the controller runs. - let (construct, gate) = rt::oneshot::channel::<()>(); - let gate = Arc::new(Mutex::new(Some(gate))); - let child = builder - .add(ChildSpec::new("ready-then-failed", &group, move |_| { - let gate = system::lock(&gate).take(); - async move { - if let Some(gate) = gate { - let _ = gate.await; - } - Ok::<_, String>(Idle) - } - })) - .unwrap(); - let core = builder.core.clone(); - let mut startup = Box::pin(builder.start()); - assert!(futures::poll!(startup.as_mut()).is_pending()); - construct.send(()).unwrap(); - assert!( - rt::timeout(limit, system::wait_true(core.ready.subscribe())) - .await - .unwrap() - ); - child - .current() - .unwrap() - .context() - .stop_with_reason(ExitReason::Kill); - assert!(rt::timeout(limit, system::wait_true(core.done.subscribe())) - .await - .unwrap()); - let tree = rt::timeout(limit, startup).await.unwrap().unwrap(); - assert!(tree.failure().unwrap().contains("budget exhausted")); - assert!(system.shutdown(limit).await.complete); + crate::test_wait::within( + concat!( + module_path!(), + "::startup_success_remains_historical_after_unobserved_ready_then_failure" + ), + async move { + struct Idle; + impl Actor for Idle {} + let limit = Duration::from_secs(30); + let system = ActorSystem::new().unwrap(); + let group = system.async_group("historical-ready-failure", 1).unwrap(); + let mut builder = system + .supervisor(SupervisorConfig { + max_restarts: 0, + ..SupervisorConfig::default() + }) + .unwrap(); + // Hold construction until start() has been polled once, so the + // first poll is pending however fast the controller runs. + let (construct, gate) = rt::oneshot::channel::<()>(); + let gate = Arc::new(Mutex::new(Some(gate))); + let child = builder + .add(ChildSpec::new("ready-then-failed", &group, move |_| { + let gate = system::lock(&gate).take(); + async move { + if let Some(gate) = gate { + let _ = gate.await; + } + Ok::<_, String>(Idle) + } + })) + .unwrap(); + let core = builder.core.clone(); + let mut startup = Box::pin(builder.start()); + assert!(futures::poll!(startup.as_mut()).is_pending()); + construct.send(()).unwrap(); + assert!( + rt::timeout(limit, system::wait_true(core.ready.subscribe())) + .await + .unwrap() + ); + child + .current() + .unwrap() + .context() + .stop_with_reason(ExitReason::Kill); + assert!(rt::timeout(limit, system::wait_true(core.done.subscribe())) + .await + .unwrap()); + let tree = rt::timeout(limit, startup).await.unwrap().unwrap(); + assert!(tree.failure().unwrap().contains("budget exhausted")); + assert!(system.shutdown(limit).await.complete); + }, + ) + .await; } #[tokio::test] async fn started_failures_allow_n_restarts_and_escalate_the_next_failure() { - use std::sync::atomic::{AtomicUsize, Ordering}; - // v0.8 (S1): a first start is not retried, so the first generation - // is healthy and every replacement panics in started(). - struct FailsStarted(Arc, bool); - impl Actor for FailsStarted { - async fn started(&mut self, _: &crate::tasks::Context) { - if self.1 { - self.0.fetch_add(1, Ordering::SeqCst); - panic!("injected started failure"); + crate::test_wait::within( + concat!( + module_path!(), + "::started_failures_allow_n_restarts_and_escalate_the_next_failure" + ), + async move { + use std::sync::atomic::{AtomicUsize, Ordering}; + // v0.8 (S1): a first start is not retried, so the first generation + // is healthy and every replacement panics in started(). + struct FailsStarted(Arc, bool); + impl Actor for FailsStarted { + async fn started(&mut self, _: &crate::tasks::Context) { + if self.1 { + self.0.fetch_add(1, Ordering::SeqCst); + panic!("injected started failure"); + } + } } - } - } - for allowed in [0, 1, 3] { - let system = ActorSystem::new().unwrap(); - let group = system.async_group("started-budget", 1).unwrap(); - let count = Arc::new(AtomicUsize::new(0)); - let attempts = count.clone(); - let mut builder = system - .supervisor(SupervisorConfig { - max_restarts: allowed, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) - .unwrap(); - let child = builder - .add(ChildSpec::new("startup-failure", &group, move |ctx| { - let attempts = attempts.clone(); - let fail = ctx.generation() > 1; - async move { Ok::<_, String>(FailsStarted(attempts, fail)) } - })) - .unwrap(); - let core = builder.core.clone(); - rt::timeout(Duration::from_secs(30), builder.start()) - .await - .unwrap() - .unwrap(); - child.current().unwrap().child_handle().kill(); - assert!(rt::timeout( - Duration::from_secs(30), - system::wait_true(core.done.subscribe()) - ) - .await - .unwrap()); - assert_eq!(count.load(Ordering::SeqCst), allowed); - assert_eq!(child.snapshot().generation as usize, allowed + 1); - assert_eq!( - system::lock(&core.failure).as_ref().unwrap().kind, - FailureKind::RestartBudget - ); - assert_eq!(child.snapshot().state, ChildState::Failed); - assert!(system.shutdown(Duration::from_secs(30)).await.complete); - } + for allowed in [0, 1, 3] { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("started-budget", 1).unwrap(); + let count = Arc::new(AtomicUsize::new(0)); + let attempts = count.clone(); + let mut builder = system + .supervisor(SupervisorConfig { + max_restarts: allowed, + backoff: Backoff { + initial: Duration::ZERO, + maximum: Duration::ZERO, + jitter: false, + ..Backoff::default() + }, + ..SupervisorConfig::default() + }) + .unwrap(); + let child = builder + .add(ChildSpec::new("startup-failure", &group, move |ctx| { + let attempts = attempts.clone(); + let fail = ctx.generation() > 1; + async move { Ok::<_, String>(FailsStarted(attempts, fail)) } + })) + .unwrap(); + let core = builder.core.clone(); + rt::timeout(Duration::from_secs(30), builder.start()) + .await + .unwrap() + .unwrap(); + child.current().unwrap().child_handle().kill(); + assert!(rt::timeout( + Duration::from_secs(30), + system::wait_true(core.done.subscribe()) + ) + .await + .unwrap()); + assert_eq!(count.load(Ordering::SeqCst), allowed); + assert_eq!(child.snapshot().generation as usize, allowed + 1); + assert_eq!( + system::lock(&core.failure).as_ref().unwrap().kind, + FailureKind::RestartBudget + ); + assert_eq!(child.snapshot().state, ChildState::Failed); + assert!(system.shutdown(Duration::from_secs(30)).await.complete); + } + }, + ) + .await; } #[tokio::test] async fn sliding_window_expires_only_old_grants_before_the_next_retry() { - let system = ActorSystem::new().unwrap(); - let group = system.async_group("sliding-budget", 1).unwrap(); - let window = Duration::from_secs(3600); - let mut builder = system - .supervisor(SupervisorConfig { - max_restarts: 2, - restart_window: window, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) - .unwrap(); - let child = builder - .add(ChildSpec::new("worker", &group, |_| async { - Ok::<_, String>(CoordinatorWorker) - })) - .unwrap(); - let core = builder.core.clone(); - let tree = builder.start().await.unwrap(); - let mut actor = child.current().unwrap(); - for generation in 1..=2 { - actor - .child_handle() - .stop_with_reason(ExitReason::Panic("retry".into())); - actor = rt::timeout( - Duration::from_secs(30), - child.wait_for_generation(generation), - ) - .await - .unwrap() - .unwrap(); - } - { - let mut budget = system::lock(&core.budget); - assert_eq!(budget.attempts.len(), 2); - // Control just the oldest timestamp; the real controller still owns - // admission and retry. The second grant remains inside the window. - budget.attempts[0] = Instant::now().checked_sub(window).unwrap(); - } - actor - .child_handle() - .stop_with_reason(ExitReason::Panic("expired oldest".into())); - let fourth = rt::timeout(Duration::from_secs(30), child.wait_for_generation(3)) - .await - .unwrap() - .unwrap(); - assert_eq!(system::lock(&core.budget).attempts.len(), 2); - fourth - .child_handle() - .stop_with_reason(ExitReason::Panic("now exhausted".into())); - rt::timeout(Duration::from_secs(30), tree.join()) - .await - .unwrap(); - assert_eq!(child.snapshot().generation, 4); - assert_eq!( - system::lock(&core.failure).as_ref().unwrap().kind, - FailureKind::RestartBudget - ); - assert!(system.shutdown(Duration::from_secs(30)).await.complete); + crate::test_wait::within( + concat!( + module_path!(), + "::sliding_window_expires_only_old_grants_before_the_next_retry" + ), + async move { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("sliding-budget", 1).unwrap(); + let window = Duration::from_secs(3600); + let mut builder = system + .supervisor(SupervisorConfig { + max_restarts: 2, + restart_window: window, + backoff: Backoff { + initial: Duration::ZERO, + maximum: Duration::ZERO, + jitter: false, + ..Backoff::default() + }, + ..SupervisorConfig::default() + }) + .unwrap(); + let child = builder + .add(ChildSpec::new("worker", &group, |_| async { + Ok::<_, String>(CoordinatorWorker) + })) + .unwrap(); + let core = builder.core.clone(); + let tree = builder.start().await.unwrap(); + let mut actor = child.current().unwrap(); + for generation in 1..=2 { + actor + .child_handle() + .stop_with_reason(ExitReason::Panic("retry".into())); + actor = rt::timeout( + Duration::from_secs(30), + child.wait_for_generation(generation), + ) + .await + .unwrap() + .unwrap(); + } + { + let mut budget = system::lock(&core.budget); + assert_eq!(budget.attempts.len(), 2); + // Control just the oldest timestamp; the real controller still owns + // admission and retry. The second grant remains inside the window. + budget.attempts[0] = Instant::now().checked_sub(window).unwrap(); + } + actor + .child_handle() + .stop_with_reason(ExitReason::Panic("expired oldest".into())); + let fourth = rt::timeout(Duration::from_secs(30), child.wait_for_generation(3)) + .await + .unwrap() + .unwrap(); + assert_eq!(system::lock(&core.budget).attempts.len(), 2); + fourth + .child_handle() + .stop_with_reason(ExitReason::Panic("now exhausted".into())); + rt::timeout(Duration::from_secs(30), tree.join()) + .await + .unwrap(); + assert_eq!(child.snapshot().generation, 4); + assert_eq!( + system::lock(&core.failure).as_ref().unwrap().kind, + FailureKind::RestartBudget + ); + assert!(system.shutdown(Duration::from_secs(30)).await.complete); + }, + ) + .await; } /// OTP charges `add_restart` when it decides a restart, before the @@ -3223,177 +3305,204 @@ mod policy_tests { /// superseding round is charged once more. #[tokio::test] async fn supersession_charges_each_selected_round_once() { - let limit = Duration::from_secs(30); - let system = ActorSystem::new().unwrap(); - let group = system - .async_group("refund-superseded-admission", 2) - .unwrap(); - let mut builder = system - .supervisor(SupervisorConfig { - strategy: SupervisorStrategy::RestForOne, - max_restarts: 3, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) - .unwrap(); - let mut children = Vec::new(); - for name in ["upstream", "middle", "tail"] { - children.push( - builder - .add(ChildSpec::new(name, &group, |_| async { - Ok::<_, String>(CoordinatorWorker) - })) - .unwrap(), - ); - } - let core = builder.core.clone(); - let tree = builder.start().await.unwrap(); - let resume = CancellationToken::new(); - let (entered, arrival) = watch::channel(false); - *system::lock(&core.retry_admission_pause) = Some((resume.clone(), entered)); - children[1] - .current() - .unwrap() - .child_handle() - .stop_with_reason(ExitReason::Panic("middle fails".into())); - assert!(rt::timeout(limit, system::wait_true(arrival)) - .await - .unwrap()); - let old_round = system::lock(&children[1].view.data) - .recovery - .as_ref() - .unwrap() - .meta - .id; - { - let budget = system::lock(&core.budget); - // The round was charged once, at selection; its rebuilds hold - // no reservation of their own. - assert_eq!(budget.attempts.len(), 1); - assert_eq!(budget.reserved, 0); - } - children[0] - .current() - .unwrap() - .child_handle() - .stop_with_reason(ExitReason::Panic("upstream supersedes".into())); - let _new_round = rt::timeout(limit, children[1].view.next_recovery(old_round)) - .await - .unwrap(); - resume.cancel(); - for child in &children { - rt::timeout(limit, child.wait_for_generation(1)) - .await - .unwrap() - .unwrap(); - assert_eq!(child.snapshot().generation, 2); - } - assert!(tree.failure().is_none()); - { - let budget = system::lock(&core.budget); - // Two restart actions: the superseded round and its successor, - // however many children each rebuilt. - assert_eq!(budget.attempts.len(), 2); - assert_eq!(budget.reserved, 0); - } - assert!(system.shutdown(limit).await.complete); + crate::test_wait::within( + concat!( + module_path!(), + "::supersession_charges_each_selected_round_once" + ), + async move { + let limit = Duration::from_secs(30); + let system = ActorSystem::new().unwrap(); + let group = system + .async_group("refund-superseded-admission", 2) + .unwrap(); + let mut builder = system + .supervisor(SupervisorConfig { + strategy: SupervisorStrategy::RestForOne, + max_restarts: 3, + backoff: Backoff { + initial: Duration::ZERO, + maximum: Duration::ZERO, + jitter: false, + ..Backoff::default() + }, + ..SupervisorConfig::default() + }) + .unwrap(); + let mut children = Vec::new(); + for name in ["upstream", "middle", "tail"] { + children.push( + builder + .add(ChildSpec::new(name, &group, |_| async { + Ok::<_, String>(CoordinatorWorker) + })) + .unwrap(), + ); + } + let core = builder.core.clone(); + let tree = builder.start().await.unwrap(); + let resume = CancellationToken::new(); + let (entered, arrival) = watch::channel(false); + *system::lock(&core.retry_admission_pause) = Some((resume.clone(), entered)); + children[1] + .current() + .unwrap() + .child_handle() + .stop_with_reason(ExitReason::Panic("middle fails".into())); + assert!(rt::timeout(limit, system::wait_true(arrival)) + .await + .unwrap()); + let old_round = system::lock(&children[1].view.data) + .recovery + .as_ref() + .unwrap() + .meta + .id; + { + let budget = system::lock(&core.budget); + // The round was charged once, at selection; its rebuilds hold + // no reservation of their own. + assert_eq!(budget.attempts.len(), 1); + assert_eq!(budget.reserved, 0); + } + children[0] + .current() + .unwrap() + .child_handle() + .stop_with_reason(ExitReason::Panic("upstream supersedes".into())); + let _new_round = rt::timeout(limit, children[1].view.next_recovery(old_round)) + .await + .unwrap(); + resume.cancel(); + for child in &children { + rt::timeout(limit, child.wait_for_generation(1)) + .await + .unwrap() + .unwrap(); + assert_eq!(child.snapshot().generation, 2); + } + assert!(tree.failure().is_none()); + { + let budget = system::lock(&core.budget); + // Two restart actions: the superseded round and its successor, + // however many children each rebuilt. + assert_eq!(budget.attempts.len(), 2); + assert_eq!(budget.reserved, 0); + } + assert!(system.shutdown(limit).await.complete); + }, + ) + .await; } #[tokio::test] async fn cohort_round_takes_one_grant_for_all_rebuilt_children() { - let limit = Duration::from_secs(30); - let system = ActorSystem::new().unwrap(); - let group = system.async_group("cohort-budget-contract", 2).unwrap(); - let mut builder = system - .supervisor(SupervisorConfig { - strategy: SupervisorStrategy::OneForAll, - max_restarts: 1, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) - .unwrap(); - let mut children = Vec::new(); - for index in 0..6 { - children.push( - builder - .add(ChildSpec::new( - format!("worker-{index}"), - &group, - |_| async { Ok::<_, String>(CoordinatorWorker) }, - )) - .unwrap(), - ); - } - let core = builder.core.clone(); - let tree = builder.start().await.unwrap(); - children[5] - .current() - .unwrap() - .child_handle() - .stop_with_reason(ExitReason::Panic("last child fails".into())); - // One grant rebuilds all six children (one restart action). - for child in &children { - rt::timeout(limit, child.wait_for_generation(1)) - .await - .unwrap() - .unwrap(); - } - assert!(tree.failure().is_none()); - assert_eq!(system::lock(&core.budget).attempts.len(), 1); - children[5] - .current() - .unwrap() - .child_handle() - .stop_with_reason(ExitReason::Panic("second failure".into())); - rt::timeout(limit, tree.join()).await.unwrap(); - assert!(tree - .failure() - .unwrap() - .contains("budget exhausted by worker-5")); - assert_eq!( - children - .iter() - .map(|child| child.snapshot().generation) - .collect::>(), - vec![2; 6] - ); - assert!(system.shutdown(limit).await.complete); + crate::test_wait::within( + concat!( + module_path!(), + "::cohort_round_takes_one_grant_for_all_rebuilt_children" + ), + async move { + let limit = Duration::from_secs(30); + let system = ActorSystem::new().unwrap(); + let group = system.async_group("cohort-budget-contract", 2).unwrap(); + let mut builder = system + .supervisor(SupervisorConfig { + strategy: SupervisorStrategy::OneForAll, + max_restarts: 1, + backoff: Backoff { + initial: Duration::ZERO, + maximum: Duration::ZERO, + jitter: false, + ..Backoff::default() + }, + ..SupervisorConfig::default() + }) + .unwrap(); + let mut children = Vec::new(); + for index in 0..6 { + children.push( + builder + .add(ChildSpec::new( + format!("worker-{index}"), + &group, + |_| async { Ok::<_, String>(CoordinatorWorker) }, + )) + .unwrap(), + ); + } + let core = builder.core.clone(); + let tree = builder.start().await.unwrap(); + children[5] + .current() + .unwrap() + .child_handle() + .stop_with_reason(ExitReason::Panic("last child fails".into())); + // One grant rebuilds all six children (one restart action). + for child in &children { + rt::timeout(limit, child.wait_for_generation(1)) + .await + .unwrap() + .unwrap(); + } + assert!(tree.failure().is_none()); + assert_eq!(system::lock(&core.budget).attempts.len(), 1); + children[5] + .current() + .unwrap() + .child_handle() + .stop_with_reason(ExitReason::Panic("second failure".into())); + rt::timeout(limit, tree.join()).await.unwrap(); + assert!(tree + .failure() + .unwrap() + .contains("budget exhausted by worker-5")); + assert_eq!( + children + .iter() + .map(|child| child.snapshot().generation) + .collect::>(), + vec![2; 6] + ); + assert!(system.shutdown(limit).await.complete); + }, + ) + .await; } #[tokio::test] async fn retry_reservations_exclude_siblings_and_refund_interrupted_backoff() { - let system = ActorSystem::new().unwrap(); - let tree = system - .supervisor(SupervisorConfig { - max_restarts: 1, - ..SupervisorConfig::default() - }) - .unwrap(); - let reservation = tree.core.reserve_retry(1).unwrap(); - assert!(tree.core.reserve_retry(2).is_none()); - assert_eq!(system::lock(&tree.core.budget).attempts.len(), 0); - drop(reservation); - let mut replacement = tree.core.reserve_retry(2).unwrap(); - tree.core.close(); - assert!(!trace::atomic(|| tree.core.retry(&mut replacement))); - drop(replacement); - { - let budget = system::lock(&tree.core.budget); - assert!(budget.attempts.is_empty()); - assert_eq!(budget.reserved, 0); - } - drop(tree); - assert!(system.shutdown(Duration::from_secs(10)).await.complete); + crate::test_wait::within( + concat!( + module_path!(), + "::retry_reservations_exclude_siblings_and_refund_interrupted_backoff" + ), + async move { + let system = ActorSystem::new().unwrap(); + let tree = system + .supervisor(SupervisorConfig { + max_restarts: 1, + ..SupervisorConfig::default() + }) + .unwrap(); + let reservation = tree.core.reserve_retry(1).unwrap(); + assert!(tree.core.reserve_retry(2).is_none()); + assert_eq!(system::lock(&tree.core.budget).attempts.len(), 0); + drop(reservation); + let mut replacement = tree.core.reserve_retry(2).unwrap(); + tree.core.close(); + assert!(!trace::atomic(|| tree.core.retry(&mut replacement))); + drop(replacement); + { + let budget = system::lock(&tree.core.budget); + assert!(budget.attempts.is_empty()); + assert_eq!(budget.reserved, 0); + } + drop(tree); + assert!(system.shutdown(Duration::from_secs(10)).await.complete); + }, + ) + .await; } #[test] diff --git a/concurrency/src/supervision/nested.rs b/concurrency/src/supervision/nested.rs index f35fd1fc..f78e2118 100644 --- a/concurrency/src/supervision/nested.rs +++ b/concurrency/src/supervision/nested.rs @@ -286,35 +286,44 @@ mod cancellation_tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn first_hook_poll_after_admission_closes_is_normal_cancellation() { - let system = ActorSystem::new().unwrap(); - let group = system.async_group("closed-nested-hook", 1).unwrap(); - let (entered_tx, entered) = tokio::sync::oneshot::channel(); - let (release, release_rx) = std::sync::mpsc::channel(); - let (scope, lease) = group.reserve(None).unwrap(); - let blocker = scope.handle.spawn(async move { - let _ = entered_tx.send(()); - release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); - }); - entered.await.unwrap(); - let context_owner = ActorRef::spawn_managed( - adapter(system.supervisor(SupervisorConfig::default()).unwrap()), - scope.clone(), - lease, + crate::test_wait::within( + concat!( + module_path!(), + "::first_hook_poll_after_admission_closes_is_normal_cancellation" + ), + async move { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("closed-nested-hook", 1).unwrap(); + let (entered_tx, entered) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel(); + let (scope, lease) = group.reserve(None).unwrap(); + let blocker = scope.handle.spawn(async move { + let _ = entered_tx.send(()); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + }); + entered.await.unwrap(); + let context_owner = ActorRef::spawn_managed( + adapter(system.supervisor(SupervisorConfig::default()).unwrap()), + scope.clone(), + lease, + ) + .unwrap(); + let mut hook = adapter(system.supervisor(SupervisorConfig::default()).unwrap()); + let cleanup = hook.builder.as_ref().unwrap().core.clone(); + scope.close_admission(); + // Exercise the narrow gap between the actor's can_start check and the + // first poll of the nested started hook, with real closed scope state. + hook.started(&context_owner.context()).await; + assert!(*cleanup.done.borrow()); + assert!(cleanup.failure.lock().unwrap().is_none()); + scope.stop(ExitReason::Shutdown); + release.send(()).unwrap(); + blocker.await.unwrap(); + context_owner.child_handle().wait_exit_async().await; + assert!(system.shutdown(Duration::from_secs(5)).await.complete); + }, ) - .unwrap(); - let mut hook = adapter(system.supervisor(SupervisorConfig::default()).unwrap()); - let cleanup = hook.builder.as_ref().unwrap().core.clone(); - scope.close_admission(); - // Exercise the narrow gap between the actor's can_start check and the - // first poll of the nested started hook, with real closed scope state. - hook.started(&context_owner.context()).await; - assert!(*cleanup.done.borrow()); - assert!(cleanup.failure.lock().unwrap().is_none()); - scope.stop(ExitReason::Shutdown); - release.send(()).unwrap(); - blocker.await.unwrap(); - context_owner.child_handle().wait_exit_async().await; - assert!(system.shutdown(Duration::from_secs(5)).await.complete); + .await; } } @@ -347,45 +356,63 @@ mod resolve_and_drop_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn resolve_requires_the_current_id_and_an_open_tree() { - let limit = Duration::from_secs(5); - let system = ActorSystem::new().unwrap(); - let tree = running_tree(&system).await; - let (current, other) = (ActorId::next(), ActorId::next()); - let ready: Ready = Arc::new(Mutex::new(Some((current, tree.clone())))); - assert!(resolve(&ready, current).is_ok()); - // An open tree published for another adapter instance never resolves. - assert!(resolve(&ready, other).is_err()); - tree.stop(); - tree.join().await; - // The current adapter's tree, once closed, no longer resolves either. - assert!(resolve(&ready, current).is_err()); - assert!(system.shutdown(limit).await.complete); + crate::test_wait::within( + concat!( + module_path!(), + "::resolve_requires_the_current_id_and_an_open_tree" + ), + async move { + let limit = Duration::from_secs(5); + let system = ActorSystem::new().unwrap(); + let tree = running_tree(&system).await; + let (current, other) = (ActorId::next(), ActorId::next()); + let ready: Ready = Arc::new(Mutex::new(Some((current, tree.clone())))); + assert!(resolve(&ready, current).is_ok()); + // An open tree published for another adapter instance never resolves. + assert!(resolve(&ready, other).is_err()); + tree.stop(); + tree.join().await; + // The current adapter's tree, once closed, no longer resolves either. + assert!(resolve(&ready, current).is_err()); + assert!(system.shutdown(limit).await.complete); + }, + ) + .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn dropped_adapter_stops_its_tree_and_clears_only_its_own_entry() { - let limit = Duration::from_secs(5); - let system = ActorSystem::new().unwrap(); - let (mine, newer) = (ActorId::next(), ActorId::next()); + crate::test_wait::within( + concat!( + module_path!(), + "::dropped_adapter_stops_its_tree_and_clears_only_its_own_entry" + ), + async move { + let limit = Duration::from_secs(5); + let system = ActorSystem::new().unwrap(); + let (mine, newer) = (ActorId::next(), ActorId::next()); - // Its own published entry is cleared and its subtree stopped. - let tree = running_tree(&system).await; - let ready: Ready = Arc::new(Mutex::new(Some((mine, tree.clone())))); - drop(adapter(Some(tree.clone()), &ready, Some(mine))); - assert!(system::lock(&ready).is_none()); - assert!(!tree.core.gate.is_open()); + // Its own published entry is cleared and its subtree stopped. + let tree = running_tree(&system).await; + let ready: Ready = Arc::new(Mutex::new(Some((mine, tree.clone())))); + drop(adapter(Some(tree.clone()), &ready, Some(mine))); + assert!(system::lock(&ready).is_none()); + assert!(!tree.core.gate.is_open()); - // A newer generation's entry survives an older adapter's drop. - let replacement = running_tree(&system).await; - let ready: Ready = Arc::new(Mutex::new(Some((newer, replacement.clone())))); - drop(adapter(None, &ready, Some(mine))); - assert_eq!( - system::lock(&ready).as_ref().map(|(id, _)| *id), - Some(newer) - ); - assert!(replacement.core.gate.is_open()); - replacement.stop(); - assert!(system.shutdown(limit).await.complete); + // A newer generation's entry survives an older adapter's drop. + let replacement = running_tree(&system).await; + let ready: Ready = Arc::new(Mutex::new(Some((newer, replacement.clone())))); + drop(adapter(None, &ready, Some(mine))); + assert_eq!( + system::lock(&ready).as_ref().map(|(id, _)| *id), + Some(newer) + ); + assert!(replacement.core.gate.is_open()); + replacement.stop(); + assert!(system.shutdown(limit).await.complete); + }, + ) + .await; } } @@ -417,56 +444,62 @@ mod stop_panic_tests { /// the adapter's lifecycle helper races it to close the subtree first. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn stopped_hook_stops_the_running_subtree() { - use futures::FutureExt; - let limit = Duration::from_secs(10); - let system = ActorSystem::new().unwrap(); - let group = system.async_group("nested-stop-panic", 1).unwrap(); - let published: Arc>>> = Arc::default(); - let expose = published.clone(); - let placement = group.clone(); - let mut root = system.supervisor(SupervisorConfig::default()).unwrap(); - let subtree = root - .add_supervisor(SupervisorSpec::new( - "subtree", - &group, - SupervisorConfig::default(), - move |tree| { - let leaf = tree.add(ChildSpec::new("leaf", &placement, |_| async { - Ok::<_, String>(Leaf) - }))?; - *system::lock(&expose) = Some(leaf); - Ok::<_, SupervisorError>(()) - }, - )) - .unwrap(); - let root = root.start().await.unwrap(); - let leaf = system::lock(&published).clone().unwrap(); - let mut waiter = Box::pin(leaf.wait_for_generation(leaf.snapshot().generation)); - let waker = std::task::Waker::from(Arc::new(PanicOnWake(AtomicBool::new(false)))); - let mut cx = std::task::Context::from_waker(&waker); - assert!(waiter.as_mut().poll(&mut cx).is_pending()); + crate::test_wait::within( + concat!(module_path!(), "::stopped_hook_stops_the_running_subtree"), + async move { + use futures::FutureExt; + let limit = Duration::from_secs(10); + let system = ActorSystem::new().unwrap(); + let group = system.async_group("nested-stop-panic", 1).unwrap(); + let published: Arc>>> = Arc::default(); + let expose = published.clone(); + let placement = group.clone(); + let mut root = system.supervisor(SupervisorConfig::default()).unwrap(); + let subtree = root + .add_supervisor(SupervisorSpec::new( + "subtree", + &group, + SupervisorConfig::default(), + move |tree| { + let leaf = tree.add(ChildSpec::new("leaf", &placement, |_| async { + Ok::<_, String>(Leaf) + }))?; + *system::lock(&expose) = Some(leaf); + Ok::<_, SupervisorError>(()) + }, + )) + .unwrap(); + let root = root.start().await.unwrap(); + let leaf = system::lock(&published).clone().unwrap(); + let mut waiter = Box::pin(leaf.wait_for_generation(leaf.snapshot().generation)); + let waker = std::task::Waker::from(Arc::new(PanicOnWake(AtomicBool::new(false)))); + let mut cx = std::task::Context::from_waker(&waker); + assert!(waiter.as_mut().poll(&mut cx).is_pending()); - let inner = subtree.current().unwrap(); - let ctx = subtree.child.current().unwrap().context(); - let mut hook = SupervisorActor { - builder: None, - running: Some(inner.clone()), - ready: Arc::default(), - id: None, - shutdown_grace: limit, - parent_core: std::sync::Weak::new(), - }; - let unwound = AssertUnwindSafe(hook.stopped(&ctx)).catch_unwind().await; - let payload = unwound.expect_err("stopped() did not stop the running subtree"); - assert_eq!( - crate::error::panic_message(&*payload), - "waiter waker panicked" - ); - // The backstop stop in Drop is idempotent once the waiter has fired. - drop(hook); - assert!(!inner.core.gate.is_open()); - drop(waiter); - drop(root); - assert!(system.shutdown(limit).await.complete); + let inner = subtree.current().unwrap(); + let ctx = subtree.child.current().unwrap().context(); + let mut hook = SupervisorActor { + builder: None, + running: Some(inner.clone()), + ready: Arc::default(), + id: None, + shutdown_grace: limit, + parent_core: std::sync::Weak::new(), + }; + let unwound = AssertUnwindSafe(hook.stopped(&ctx)).catch_unwind().await; + let payload = unwound.expect_err("stopped() did not stop the running subtree"); + assert_eq!( + crate::error::panic_message(&*payload), + "waiter waker panicked" + ); + // The backstop stop in Drop is idempotent once the waiter has fired. + drop(hook); + assert!(!inner.core.gate.is_open()); + drop(waiter); + drop(root); + assert!(system.shutdown(limit).await.complete); + }, + ) + .await; } } diff --git a/concurrency/src/system.rs b/concurrency/src/system.rs index f2f38f4e..a2c545c6 100644 --- a/concurrency/src/system.rs +++ b/concurrency/src/system.rs @@ -1415,70 +1415,88 @@ mod coordinator_tests { } #[tokio::test] async fn coordinator_fault_stops_all_targets_and_retains_owned_cleanup() { - use crate::tasks::ActorStart; - let system = ActorSystem::new().unwrap(); - let group = system.async_group("coordinator-fault", 1).unwrap(); - let stopped = Arc::new(AtomicUsize::new(0)); - let first = Worker(stopped.clone()).start_on(&group).unwrap(); - let second = Worker(stopped.clone()).start_on(&group).unwrap(); - first.ready().await.unwrap(); - second.ready().await.unwrap(); - let (done, _) = watch::channel(false); - let fault = Arc::new(FaultUnit { - allow_emergency: AtomicBool::new(false), - done, - }); - system.inner.register_unit(fault.clone()).unwrap(); - let mut report = system.shutdown(Duration::from_millis(30)).await; - assert!(!report.complete); - // The report is a snapshot, not an acknowledgement that the coordinator - // ran. Under full-suite load its 30ms deadline can precede the fault. - // Refresh with bounded, yielding waits until that diagnostic is visible. - report = rt::timeout(LIMIT, async move { - while !report - .coordinator_failures - .iter() - .any(|e| e.contains("injected system coordinator fault")) - { - // Yield even if a regressed coordinator closes its done watch: - // wait() can then return immediately, and LIMIT must still fire. - rt::sleep(Duration::from_millis(1)).await; - report = report.wait(Duration::ZERO).await; - } - report - }) - .await - .expect("coordinator did not report its injected failure"); - assert!(report - .coordinator_failures - .iter() - .any(|e| e.contains("injected system coordinator fault"))); - tokio::time::timeout(LIMIT, first.join()).await.unwrap(); - tokio::time::timeout(LIMIT, second.join()).await.unwrap(); - assert_eq!(stopped.load(Ordering::SeqCst), 2); - assert!(!*system.inner.control.done.borrow()); - drop(system); // pending report + coordinator retain runtime ownership - fault.allow_emergency.store(true, Ordering::SeqCst); - let report = report.wait(LIMIT).await; - assert!(report.complete); - assert!(!report.coordinator_failures.is_empty()); - assert!(report.coordinator_failures.len() <= 16); + crate::test_wait::within( + concat!( + module_path!(), + "::coordinator_fault_stops_all_targets_and_retains_owned_cleanup" + ), + async move { + use crate::tasks::ActorStart; + let system = ActorSystem::new().unwrap(); + let group = system.async_group("coordinator-fault", 1).unwrap(); + let stopped = Arc::new(AtomicUsize::new(0)); + let first = Worker(stopped.clone()).start_on(&group).unwrap(); + let second = Worker(stopped.clone()).start_on(&group).unwrap(); + first.ready().await.unwrap(); + second.ready().await.unwrap(); + let (done, _) = watch::channel(false); + let fault = Arc::new(FaultUnit { + allow_emergency: AtomicBool::new(false), + done, + }); + system.inner.register_unit(fault.clone()).unwrap(); + let mut report = system.shutdown(Duration::from_millis(30)).await; + assert!(!report.complete); + // The report is a snapshot, not an acknowledgement that the coordinator + // ran. Under full-suite load its 30ms deadline can precede the fault. + // Refresh with bounded, yielding waits until that diagnostic is visible. + report = rt::timeout(LIMIT, async move { + while !report + .coordinator_failures + .iter() + .any(|e| e.contains("injected system coordinator fault")) + { + // Yield even if a regressed coordinator closes its done watch: + // wait() can then return immediately, and LIMIT must still fire. + rt::sleep(Duration::from_millis(1)).await; + report = report.wait(Duration::ZERO).await; + } + report + }) + .await + .expect("coordinator did not report its injected failure"); + assert!(report + .coordinator_failures + .iter() + .any(|e| e.contains("injected system coordinator fault"))); + tokio::time::timeout(LIMIT, first.join()).await.unwrap(); + tokio::time::timeout(LIMIT, second.join()).await.unwrap(); + assert_eq!(stopped.load(Ordering::SeqCst), 2); + assert!(!*system.inner.control.done.borrow()); + drop(system); // pending report + coordinator retain runtime ownership + fault.allow_emergency.store(true, Ordering::SeqCst); + let report = report.wait(LIMIT).await; + assert!(report.complete); + assert!(!report.coordinator_failures.is_empty()); + assert!(report.coordinator_failures.len() <= 16); + }, + ) + .await; } #[tokio::test] async fn emergency_cleanup_does_not_resignal_an_already_finished_faulting_unit() { - let system = ActorSystem::new().unwrap(); - let (done, _) = watch::channel(false); - let fault = Arc::new(FaultUnit { - allow_emergency: AtomicBool::new(false), - done, - }); - system.inner.register_unit(fault.clone()).unwrap(); - let report = system.shutdown(Duration::from_millis(30)).await; - assert!(!report.complete); - // Another owner finishes it; its stop method remains permanently broken. - fault.done.send_replace(true); - let report = report.wait(LIMIT).await; - assert!(report.complete); - assert!(!report.coordinator_failures.is_empty()); + crate::test_wait::within( + concat!( + module_path!(), + "::emergency_cleanup_does_not_resignal_an_already_finished_faulting_unit" + ), + async move { + let system = ActorSystem::new().unwrap(); + let (done, _) = watch::channel(false); + let fault = Arc::new(FaultUnit { + allow_emergency: AtomicBool::new(false), + done, + }); + system.inner.register_unit(fault.clone()).unwrap(); + let report = system.shutdown(Duration::from_millis(30)).await; + assert!(!report.complete); + // Another owner finishes it; its stop method remains permanently broken. + fault.done.send_replace(true); + let report = report.wait(LIMIT).await; + assert!(report.complete); + assert!(!report.coordinator_failures.is_empty()); + }, + ) + .await; } } diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index afd96b44..3f7de171 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -2422,149 +2422,180 @@ mod tests { #[test] pub fn backend_async_handles_send_and_request() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let counter = Counter { count: 0 }.start(); + crate::test_wait::bounded( + concat!(module_path!(), "::backend_async_handles_send_and_request"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let counter = Counter { count: 0 }.start(); - let result = counter.request(GetCount).await.unwrap(); - assert_eq!(result, 0); + let result = counter.request(GetCount).await.unwrap(); + assert_eq!(result, 0); - let result = counter.request(Increment).await.unwrap(); - assert_eq!(result, 1); + let result = counter.request(Increment).await.unwrap(); + assert_eq!(result, 1); - // fire-and-forget send; the request below shares its mailbox, so - // it is handled after the send without any extra wait. - counter.send(Increment).unwrap(); + // fire-and-forget send; the request below shares its mailbox, so + // it is handled after the send without any extra wait. + counter.send(Increment).unwrap(); - let result = counter.request(GetCount).await.unwrap(); - assert_eq!(result, 2); + let result = counter.request(GetCount).await.unwrap(); + assert_eq!(result, 2); - let final_count = counter.request(StopCounter).await.unwrap(); - assert_eq!(final_count, 2); - }); + let final_count = counter.request(StopCounter).await.unwrap(); + assert_eq!(final_count, 2); + }); + }, + ); } #[test] pub fn backend_blocking_handles_send_and_request() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let counter = Counter { count: 0 }.start_with_backend(Backend::Blocking); + crate::test_wait::bounded( + concat!( + module_path!(), + "::backend_blocking_handles_send_and_request" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let counter = Counter { count: 0 }.start_with_backend(Backend::Blocking); - let result = counter.request(GetCount).await.unwrap(); - assert_eq!(result, 0); + let result = counter.request(GetCount).await.unwrap(); + assert_eq!(result, 0); - let result = counter.request(Increment).await.unwrap(); - assert_eq!(result, 1); + let result = counter.request(Increment).await.unwrap(); + assert_eq!(result, 1); - counter.send(Increment).unwrap(); + counter.send(Increment).unwrap(); - let result = counter.request(GetCount).await.unwrap(); - assert_eq!(result, 2); + let result = counter.request(GetCount).await.unwrap(); + assert_eq!(result, 2); - let final_count = counter.request(StopCounter).await.unwrap(); - assert_eq!(final_count, 2); - }); + let final_count = counter.request(StopCounter).await.unwrap(); + assert_eq!(final_count, 2); + }); + }, + ); } #[test] pub fn backend_thread_handles_send_and_request() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let counter = Counter { count: 0 }.start_with_backend(Backend::Thread); + crate::test_wait::bounded( + concat!(module_path!(), "::backend_thread_handles_send_and_request"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let counter = Counter { count: 0 }.start_with_backend(Backend::Thread); - let result = counter.request(GetCount).await.unwrap(); - assert_eq!(result, 0); + let result = counter.request(GetCount).await.unwrap(); + assert_eq!(result, 0); - let result = counter.request(Increment).await.unwrap(); - assert_eq!(result, 1); + let result = counter.request(Increment).await.unwrap(); + assert_eq!(result, 1); - counter.send(Increment).unwrap(); + counter.send(Increment).unwrap(); - let result = counter.request(GetCount).await.unwrap(); - assert_eq!(result, 2); + let result = counter.request(GetCount).await.unwrap(); + assert_eq!(result, 2); - let final_count = counter.request(StopCounter).await.unwrap(); - assert_eq!(final_count, 2); - }); + let final_count = counter.request(StopCounter).await.unwrap(); + assert_eq!(final_count, 2); + }); + }, + ); } #[test] pub fn multiple_backends_concurrent() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let async_counter = Counter { count: 0 }.start(); - let blocking_counter = Counter { count: 100 }.start_with_backend(Backend::Blocking); - let thread_counter = Counter { count: 200 }.start_with_backend(Backend::Thread); - - async_counter.request(Increment).await.unwrap(); - blocking_counter.request(Increment).await.unwrap(); - thread_counter.request(Increment).await.unwrap(); - - let async_val = async_counter.request(GetCount).await.unwrap(); - let blocking_val = blocking_counter.request(GetCount).await.unwrap(); - let thread_val = thread_counter.request(GetCount).await.unwrap(); - - assert_eq!(async_val, 1); - assert_eq!(blocking_val, 101); - assert_eq!(thread_val, 201); - - async_counter.request(StopCounter).await.unwrap(); - blocking_counter.request(StopCounter).await.unwrap(); - thread_counter.request(StopCounter).await.unwrap(); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::multiple_backends_concurrent"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let async_counter = Counter { count: 0 }.start(); + let blocking_counter = + Counter { count: 100 }.start_with_backend(Backend::Blocking); + let thread_counter = Counter { count: 200 }.start_with_backend(Backend::Thread); + + async_counter.request(Increment).await.unwrap(); + blocking_counter.request(Increment).await.unwrap(); + thread_counter.request(Increment).await.unwrap(); + + let async_val = async_counter.request(GetCount).await.unwrap(); + let blocking_val = blocking_counter.request(GetCount).await.unwrap(); + let thread_val = thread_counter.request(GetCount).await.unwrap(); + + assert_eq!(async_val, 1); + assert_eq!(blocking_val, 101); + assert_eq!(thread_val, 201); + + async_counter.request(StopCounter).await.unwrap(); + blocking_counter.request(StopCounter).await.unwrap(); + thread_counter.request(StopCounter).await.unwrap(); + }); + }, + ); } #[test] pub fn request_timeout() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - struct SlowActor; - struct SlowOp; - impl Message for SlowOp { - type Result = (); - } - impl Actor for SlowActor {} - impl Handler for SlowActor { - async fn handle( - &mut self, - _msg: SlowOp, - reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - // Never replies, so the timeout is the only possible outcome - // however late the runtime polls it. - std::future::pending::<()>().await; - reply.reply(()); + crate::test_wait::bounded(concat!(module_path!(), "::request_timeout"), move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + struct SlowActor; + struct SlowOp; + impl Message for SlowOp { + type Result = (); + } + impl Actor for SlowActor {} + impl Handler for SlowActor { + async fn handle( + &mut self, + _msg: SlowOp, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + // Never replies, so the timeout is the only possible outcome + // however late the runtime polls it. + std::future::pending::<()>().await; + reply.reply(()); + } } - } - let actor = SlowActor.start(); - let result = actor - .request_with_timeout(SlowOp, Duration::from_millis(50)) - .await; - assert!(matches!(result, Err(ActorError::RequestTimeout))); + let actor = SlowActor.start(); + let result = actor + .request_with_timeout(SlowOp, Duration::from_millis(50)) + .await; + assert!(matches!(result, Err(ActorError::RequestTimeout))); + }); }); } #[test] pub fn recipient_type_erasure() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let counter = Counter { count: 42 }.start(); - let recipient: Recipient = counter.recipient(); - - let rx = recipient - .send_request(GetCount, Deadline::after(DEFAULT_REQUEST_TIMEOUT)) - .unwrap(); - let result = rx.await.unwrap(); - assert_eq!(result, 42); - - // Also test request helper - let result = request(&*recipient, GetCount, Duration::from_secs(5)) - .await - .unwrap(); - assert_eq!(result, 42); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::recipient_type_erasure"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let counter = Counter { count: 42 }.start(); + let recipient: Recipient = counter.recipient(); + + let rx = recipient + .send_request(GetCount, Deadline::after(DEFAULT_REQUEST_TIMEOUT)) + .unwrap(); + let result = rx.await.unwrap(); + assert_eq!(result, 42); + + // Also test request helper + let result = request(&*recipient, GetCount, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(result, 42); + }); + }, + ); } // --- SlowShutdownActor for join tests --- @@ -2655,59 +2686,72 @@ mod tests { #[test] pub fn thread_backend_join_does_not_block_runtime() { - // stopped() blocks its OS thread until a task on this current-thread - // runtime releases it. If join() blocked the runtime, that task could - // never run and stopped() would give up after the timeout instead. - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::thread_backend_join_does_not_block_runtime" + ), + move || { + // stopped() blocks its OS thread until a task on this current-thread + // runtime releases it. If join() blocked the runtime, that task could + // never run and stopped() would give up after the timeout instead. + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); - runtime.block_on(async move { - let (held, release) = HeldThread::new(); - let released = held.released.clone(); - let actor = held.start_with_backend(Backend::Thread); - actor.send(StopSlow).unwrap(); - let _releaser = rt::spawn(async move { - let _ = release.send(()); - }); + runtime.block_on(async move { + let (held, release) = HeldThread::new(); + let released = held.released.clone(); + let actor = held.start_with_backend(Backend::Thread); + actor.send(StopSlow).unwrap(); + let _releaser = rt::spawn(async move { + let _ = release.send(()); + }); - actor.join().await; + actor.join().await; - assert!( - released.load(atomic::Ordering::SeqCst), - "join() blocked the runtime: the releasing task never ran" - ); - }); + assert!( + released.load(atomic::Ordering::SeqCst), + "join() blocked the runtime: the releasing task never ran" + ); + }); + }, + ); } #[test] pub fn multiple_join_callers_all_notified() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let actor = SlowShutdownActor.start(); - let actor_clone1 = actor.clone(); - let actor_clone2 = actor.clone(); - - let join1 = rt::spawn(async move { - actor_clone1.join().await; - 1u32 - }); - let join2 = rt::spawn(async move { - actor_clone2.join().await; - 2u32 - }); + crate::test_wait::bounded( + concat!(module_path!(), "::multiple_join_callers_all_notified"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let actor = SlowShutdownActor.start(); + let actor_clone1 = actor.clone(); + let actor_clone2 = actor.clone(); + + let join1 = rt::spawn(async move { + actor_clone1.join().await; + 1u32 + }); + let join2 = rt::spawn(async move { + actor_clone2.join().await; + 2u32 + }); - rt::sleep(Duration::from_millis(10)).await; + rt::sleep(Duration::from_millis(10)).await; - actor.send(StopSlow).unwrap(); + actor.send(StopSlow).unwrap(); - let (r1, r2) = tokio::join!(join1, join2); - assert_eq!(r1.unwrap(), 1); - assert_eq!(r2.unwrap(), 2); + let (r1, r2) = tokio::join!(join1, join2); + assert_eq!(r1.unwrap(), 1); + assert_eq!(r2.unwrap(), 2); - actor.join().await; - }); + actor.join().await; + }); + }, + ); } // --- Badly behaved actors for blocking tests --- @@ -2788,25 +2832,30 @@ mod tests { #[test] pub fn badly_behaved_thread_non_blocking() { - // An async actor that blocks its thread starves everything else on - // it. With one runtime thread this is deterministic; on a multi-thread - // runtime the well-behaved actor may run on another worker. - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - runtime.block_on(async move { - let badboy = BadlyBehavedTask.start(); - badboy.send(DoBlock).unwrap(); - let goodboy = WellBehavedTask { count: 0 }.start(); - goodboy.send(IncrementWell).unwrap(); - rt::sleep(Duration::from_secs(1)).await; - let count = goodboy.request(GetCount).await.unwrap(); - // The blocking handler held the only thread for 2 s of the 1 s - // window, so the 100 ms chain could not get far. - assert!(count < 10, "blocked runtime still made progress: {count}"); - goodboy.request(StopCounter).await.unwrap(); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::badly_behaved_thread_non_blocking"), + move || { + // An async actor that blocks its thread starves everything else on + // it. With one runtime thread this is deterministic; on a multi-thread + // runtime the well-behaved actor may run on another worker. + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async move { + let badboy = BadlyBehavedTask.start(); + badboy.send(DoBlock).unwrap(); + let goodboy = WellBehavedTask { count: 0 }.start(); + goodboy.send(IncrementWell).unwrap(); + rt::sleep(Duration::from_secs(1)).await; + let count = goodboy.request(GetCount).await.unwrap(); + // The blocking handler held the only thread for 2 s of the 1 s + // window, so the 100 ms chain could not get far. + assert!(count < 10, "blocked runtime still made progress: {count}"); + goodboy.request(StopCounter).await.unwrap(); + }); + }, + ); } /// An async actor on a single-threaded runtime keeps making progress while @@ -2859,79 +2908,89 @@ mod tests { #[test] pub fn panic_in_started_stops_actor() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - struct PanicOnStart; - struct Ping; - impl Message for Ping { - type Result = (); - } - impl Actor for PanicOnStart { - async fn started(&mut self, _ctx: &Context) { - panic!("boom in started"); - } - } - impl Handler for PanicOnStart { - async fn handle( - &mut self, - _msg: Ping, - reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - reply.reply(()); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::panic_in_started_stops_actor"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + struct PanicOnStart; + struct Ping; + impl Message for Ping { + type Result = (); + } + impl Actor for PanicOnStart { + async fn started(&mut self, _ctx: &Context) { + panic!("boom in started"); + } + } + impl Handler for PanicOnStart { + async fn handle( + &mut self, + _msg: Ping, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + reply.reply(()); + } + } - let actor = PanicOnStart.start(); - actor.join().await; - let result = actor.send(Ping); - assert!(result.is_err()); - }); + let actor = PanicOnStart.start(); + actor.join().await; + let result = actor.send(Ping); + assert!(result.is_err()); + }); + }, + ); } #[test] pub fn panic_in_handler_stops_actor() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - struct PanicOnMsg; - struct Explode; - impl Message for Explode { - type Result = (); - } - struct Check; - impl Message for Check { - type Result = u32; - } - impl Actor for PanicOnMsg {} - impl Handler for PanicOnMsg { - async fn handle( - &mut self, - _msg: Explode, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom in handler"); + crate::test_wait::bounded( + concat!(module_path!(), "::panic_in_handler_stops_actor"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + struct PanicOnMsg; + struct Explode; + impl Message for Explode { + type Result = (); + } + struct Check; + impl Message for Check { + type Result = u32; + } + impl Actor for PanicOnMsg {} + impl Handler for PanicOnMsg { + async fn handle( + &mut self, + _msg: Explode, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom in handler"); + } + } + } + impl Handler for PanicOnMsg { + async fn handle( + &mut self, + _msg: Check, + reply: crate::reply::PendingReply<'_, u32>, + _ctx: &Context, + ) { + reply.reply(42); + } } - } - } - impl Handler for PanicOnMsg { - async fn handle( - &mut self, - _msg: Check, - reply: crate::reply::PendingReply<'_, u32>, - _ctx: &Context, - ) { - reply.reply(42); - } - } - let actor = PanicOnMsg.start(); - actor.send(Explode).unwrap(); - actor.join().await; - let result = actor.request(Check).await; - assert!(result.is_err()); - }); + let actor = PanicOnMsg.start(); + actor.send(Explode).unwrap(); + actor.join().await; + let result = actor.request(Check).await; + assert!(result.is_err()); + }); + }, + ); } /// Review P1 (Codex 9): a manual `Handler` may be a plain `fn` that @@ -2942,182 +3001,218 @@ mod tests { /// lifecycle (no `stopped()`, an "actor lifecycle unwound" reason). #[test] pub fn panic_building_the_handler_future_is_a_handler_panic() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - struct Eager(Arc); - struct Explode; - impl Message for Explode { - type Result = (); - } - impl Actor for Eager { - async fn stopped(&mut self, _ctx: &Context) { - self.0.store(true, Ordering::SeqCst); - } - } - impl Handler for Eager { - #[allow(unreachable_code, clippy::diverging_sub_expression)] - fn handle( - &mut self, - _msg: Explode, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) -> impl Future + Send { - panic!("boom building the handler future"); - async {} - } - } + crate::test_wait::bounded( + concat!( + module_path!(), + "::panic_building_the_handler_future_is_a_handler_panic" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + struct Eager(Arc); + struct Explode; + impl Message for Explode { + type Result = (); + } + impl Actor for Eager { + async fn stopped(&mut self, _ctx: &Context) { + self.0.store(true, Ordering::SeqCst); + } + } + impl Handler for Eager { + #[allow(unreachable_code, clippy::diverging_sub_expression)] + fn handle( + &mut self, + _msg: Explode, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) -> impl Future + Send { + panic!("boom building the handler future"); + async {} + } + } - let stopped = Arc::new(atomic::AtomicBool::new(false)); - let actor = Eager(stopped.clone()).start(); - let reply = actor.request(Explode).await; - let reason = actor.wait_exit().await; - let ExitReason::Panic(message) = &reason else { - panic!("unexpected reason {reason:?}"); - }; - assert!( - message.starts_with("panic in handler: boom building"), - "not a handler panic: {message}" - ); - assert!( - matches!(&reply, Err(ActorError::ActorStopped(r)) if *r == reason), - "caller saw {reply:?}" - ); - assert!(stopped.load(Ordering::SeqCst), "stopped() was skipped"); - }); + let stopped = Arc::new(atomic::AtomicBool::new(false)); + let actor = Eager(stopped.clone()).start(); + let reply = actor.request(Explode).await; + let reason = actor.wait_exit().await; + let ExitReason::Panic(message) = &reason else { + panic!("unexpected reason {reason:?}"); + }; + assert!( + message.starts_with("panic in handler: boom building"), + "not a handler panic: {message}" + ); + assert!( + matches!(&reply, Err(ActorError::ActorStopped(r)) if *r == reason), + "caller saw {reply:?}" + ); + assert!(stopped.load(Ordering::SeqCst), "stopped() was skipped"); + }); + }, + ); } #[test] pub fn panic_in_stopped_still_completes() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - struct PanicOnStop; - struct StopMe; - impl Message for StopMe { - type Result = (); - } - impl Actor for PanicOnStop { - async fn stopped(&mut self, _ctx: &Context) { - panic!("boom in stopped"); - } - } - impl Handler for PanicOnStop { - async fn handle( - &mut self, - _msg: StopMe, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - ctx.stop(); - reply.reply(()); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::panic_in_stopped_still_completes"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + struct PanicOnStop; + struct StopMe; + impl Message for StopMe { + type Result = (); + } + impl Actor for PanicOnStop { + async fn stopped(&mut self, _ctx: &Context) { + panic!("boom in stopped"); + } + } + impl Handler for PanicOnStop { + async fn handle( + &mut self, + _msg: StopMe, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.stop(); + reply.reply(()); + } + } - let actor = PanicOnStop.start(); - actor.send(StopMe).unwrap(); - actor.join().await; - }); + let actor = PanicOnStop.start(); + actor.send(StopMe).unwrap(); + actor.join().await; + }); + }, + ); } #[test] pub fn send_message_on_delivers() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let counter = Counter { count: 0 }.start(); - let ctx = counter.context(); - send_message_on(ctx, rt::sleep(Duration::from_millis(10)), Increment); - let count = crate::test_wait::wait_for_async( - || counter.request(GetCount), - |count| matches!(count, Ok(1)), - ) - .await - .unwrap(); - assert_eq!(count, 1); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::send_message_on_delivers"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let counter = Counter { count: 0 }.start(); + let ctx = counter.context(); + send_message_on(ctx, rt::sleep(Duration::from_millis(10)), Increment); + let count = crate::test_wait::wait_for_async( + || counter.request(GetCount), + |count| matches!(count, Ok(1)), + ) + .await + .unwrap(); + assert_eq!(count, 1); + }); + }, + ); } #[test] pub fn send_message_on_cancelled() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let counter = Counter { count: 0 }.start(); - let ctx = counter.context(); - send_message_on(ctx, rt::sleep(Duration::from_millis(200)), Increment); - // Stop actor before the future resolves - let final_count = counter.request(StopCounter).await.unwrap(); - assert_eq!(final_count, 0, "message should not have been delivered"); - counter.join().await; - }); + crate::test_wait::bounded( + concat!(module_path!(), "::send_message_on_cancelled"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let counter = Counter { count: 0 }.start(); + let ctx = counter.context(); + send_message_on(ctx, rt::sleep(Duration::from_millis(200)), Increment); + // Stop actor before the future resolves + let final_count = counter.request(StopCounter).await.unwrap(); + assert_eq!(final_count, 0, "message should not have been delivered"); + counter.join().await; + }); + }, + ); } // --- ExitReason tests --- #[test] pub fn exit_reason_normal_on_clean_stop() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let actor = Counter { count: 0 }.start(); - actor.request(StopCounter).await.unwrap(); - let reason = actor.wait_exit().await; - assert!(matches!(reason, ExitReason::Normal)); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::exit_reason_normal_on_clean_stop"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let actor = Counter { count: 0 }.start(); + actor.request(StopCounter).await.unwrap(); + let reason = actor.wait_exit().await; + assert!(matches!(reason, ExitReason::Normal)); + }); + }, + ); } #[test] pub fn exit_reason_panic_in_started() { - struct PanicStart; - struct Ping; - impl Message for Ping { - type Result = (); - } - impl Actor for PanicStart { - async fn started(&mut self, _ctx: &Context) { - panic!("boom in started"); - } - } - impl Handler for PanicStart { - async fn handle( - &mut self, - _msg: Ping, - reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - reply.reply(()); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::exit_reason_panic_in_started"), + move || { + struct PanicStart; + struct Ping; + impl Message for Ping { + type Result = (); + } + impl Actor for PanicStart { + async fn started(&mut self, _ctx: &Context) { + panic!("boom in started"); + } + } + impl Handler for PanicStart { + async fn handle( + &mut self, + _msg: Ping, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + reply.reply(()); + } + } - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { let actor = PanicStart.start(); let reason = actor.wait_exit().await; assert!( matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in started")) ); }); + }, + ); } #[test] pub fn exit_reason_panic_in_handler() { - struct PanicHandler; - struct Explode; - impl Message for Explode { - type Result = (); - } - impl Actor for PanicHandler {} - impl Handler for PanicHandler { - async fn handle( - &mut self, - _msg: Explode, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom in handler"); + crate::test_wait::bounded( + concat!(module_path!(), "::exit_reason_panic_in_handler"), + move || { + struct PanicHandler; + struct Explode; + impl Message for Explode { + type Result = (); + } + impl Actor for PanicHandler {} + impl Handler for PanicHandler { + async fn handle( + &mut self, + _msg: Explode, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom in handler"); + } + } } - } - } - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { let actor = PanicHandler.start(); let _ = actor.send(Explode); let reason = actor.wait_exit().await; @@ -3125,18 +3220,28 @@ mod tests { matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in handler")) ); }); + }, + ); } #[test] pub fn exit_reason_poll_returns_none_while_running() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let actor = Counter { count: 0 }.start(); - assert!(actor.exit_reason().is_none()); - actor.request(StopCounter).await.unwrap(); - actor.join().await; - assert!(actor.exit_reason().is_some()); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::exit_reason_poll_returns_none_while_running" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let actor = Counter { count: 0 }.start(); + assert!(actor.exit_reason().is_none()); + actor.request(StopCounter).await.unwrap(); + actor.join().await; + assert!(actor.exit_reason().is_some()); + }); + }, + ); } // --- Monitor tests --- @@ -3229,32 +3334,40 @@ mod tests { #[test] pub fn monitor_running_actor_delivers_down_on_exit() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::monitor_running_actor_delivers_down_on_exit" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); - let monitor_ref = watcher.request(StartMonitor(target_handle)).await.unwrap(); + let monitor_ref = watcher.request(StartMonitor(target_handle)).await.unwrap(); - // Stop the target — Down should be delivered - target.request(StopCounter).await.unwrap(); - target.join().await; + // Stop the target — Down should be delivered + target.request(StopCounter).await.unwrap(); + target.join().await; - let downs = crate::test_wait::wait_for_async( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), - ) - .await - .unwrap(); - assert_eq!(downs.len(), 1); - assert_eq!(downs[0].monitor_ref, monitor_ref); - assert!(matches!(downs[0].reason, ExitReason::Normal)); - }); + let downs = crate::test_wait::wait_for_async( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), + ) + .await + .unwrap(); + assert_eq!(downs.len(), 1); + assert_eq!(downs[0].monitor_ref, monitor_ref); + assert!(matches!(downs[0].reason, ExitReason::Normal)); + }); + }, + ); } /// OTP `demonitor(Ref, [flush])`: a `Down` already in the mailbox @@ -3262,240 +3375,290 @@ mod tests { /// never handled once `demonitor` returned. #[test] pub fn demonitor_flushes_a_queued_down() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - target.request(StopCounter).await.unwrap(); - target.join().await; - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); - watcher - .request(MonitorThenDemonitor(target.child_handle())) - .await - .unwrap(); - // FIFO: the queued Down is dispatched before this request. - let downs = watcher.request(GetDowns).await.unwrap(); - assert!(downs.is_empty(), "a flushed Down was handled: {downs:?}"); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::demonitor_flushes_a_queued_down"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + target.request(StopCounter).await.unwrap(); + target.join().await; + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); + watcher + .request(MonitorThenDemonitor(target.child_handle())) + .await + .unwrap(); + // FIFO: the queued Down is dispatched before this request. + let downs = watcher.request(GetDowns).await.unwrap(); + assert!(downs.is_empty(), "a flushed Down was handled: {downs:?}"); + }); + }, + ); } #[test] pub fn monitor_already_dead_actor_delivers_down_immediately() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - target.request(StopCounter).await.unwrap(); - target.join().await; - let target_handle = target.child_handle(); - - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::monitor_already_dead_actor_delivers_down_immediately" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + target.request(StopCounter).await.unwrap(); + target.join().await; + let target_handle = target.child_handle(); + + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); - let _ = watcher.request(StartMonitor(target_handle)).await.unwrap(); + let _ = watcher.request(StartMonitor(target_handle)).await.unwrap(); - let downs = crate::test_wait::wait_for_async( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), - ) - .await - .unwrap(); - assert_eq!(downs.len(), 1); - }); + let downs = crate::test_wait::wait_for_async( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), + ) + .await + .unwrap(); + assert_eq!(downs.len(), 1); + }); + }, + ); } #[test] pub fn demonitor_before_target_dies_suppresses_down() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::demonitor_before_target_dies_suppresses_down" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); - let monitor_ref = watcher.request(StartMonitor(target_handle)).await.unwrap(); - watcher.request(CallDemonitor(monitor_ref)).await.unwrap(); + let monitor_ref = watcher.request(StartMonitor(target_handle)).await.unwrap(); + watcher.request(CallDemonitor(monitor_ref)).await.unwrap(); - // Now stop the target - target.request(StopCounter).await.unwrap(); - target.join().await; - rt::sleep(Duration::from_millis(50)).await; + // Now stop the target + target.request(StopCounter).await.unwrap(); + target.join().await; + rt::sleep(Duration::from_millis(50)).await; - let downs = watcher.request(GetDowns).await.unwrap(); - assert!(downs.is_empty(), "expected no Down, got {:?}", downs.len()); - }); + let downs = watcher.request(GetDowns).await.unwrap(); + assert!(downs.is_empty(), "expected no Down, got {:?}", downs.len()); + }); + }, + ); } #[test] pub fn multiple_monitors_each_get_own_ref_and_down() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); - - let r1 = watcher - .request(StartMonitor(target_handle.clone())) - .await - .unwrap(); - let r2 = watcher.request(StartMonitor(target_handle)).await.unwrap(); - assert_ne!(r1, r2); - - target.request(StopCounter).await.unwrap(); - target.join().await; - let downs = crate::test_wait::wait_for_async( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| d.len() >= 2), - ) - .await - .unwrap(); - assert_eq!(downs.len(), 2); - let refs: Vec<_> = downs.iter().map(|d| d.monitor_ref).collect(); - assert!(refs.contains(&r1)); - assert!(refs.contains(&r2)); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::multiple_monitors_each_get_own_ref_and_down" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); + + let r1 = watcher + .request(StartMonitor(target_handle.clone())) + .await + .unwrap(); + let r2 = watcher.request(StartMonitor(target_handle)).await.unwrap(); + assert_ne!(r1, r2); + + target.request(StopCounter).await.unwrap(); + target.join().await; + let downs = crate::test_wait::wait_for_async( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| d.len() >= 2), + ) + .await + .unwrap(); + assert_eq!(downs.len(), 2); + let refs: Vec<_> = downs.iter().map(|d| d.monitor_ref).collect(); + assert!(refs.contains(&r1)); + assert!(refs.contains(&r2)); + }); + }, + ); } #[test] pub fn monitor_table_is_cleaned_up_after_target_dies() { - // Watcher should remove its entry from the monitor table after the - // target dies, so the table doesn't accumulate stale entries. - struct Inspect; - impl Message for Inspect { - type Result = usize; - } - impl Handler for Watcher { - async fn handle( - &mut self, - _msg: Inspect, - reply: crate::reply::PendingReply<'_, usize>, - ctx: &Context, - ) { - reply.reply( - ctx.shared - .monitors - .lock() - .unwrap() - .as_ref() - .map_or(0, HashMap::len), - ); - } - } + crate::test_wait::bounded( + concat!( + module_path!(), + "::monitor_table_is_cleaned_up_after_target_dies" + ), + move || { + // Watcher should remove its entry from the monitor table after the + // target dies, so the table doesn't accumulate stale entries. + struct Inspect; + impl Message for Inspect { + type Result = usize; + } + impl Handler for Watcher { + async fn handle( + &mut self, + _msg: Inspect, + reply: crate::reply::PendingReply<'_, usize>, + ctx: &Context, + ) { + reply.reply( + ctx.shared + .monitors + .lock() + .unwrap() + .as_ref() + .map_or(0, HashMap::len), + ); + } + } - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); - - let _ = watcher - .request(StartMonitor(target.child_handle())) - .await - .unwrap(); - assert_eq!(watcher.request(Inspect).await.unwrap(), 1); - - target.request(StopCounter).await.unwrap(); - target.join().await; - let entries = crate::test_wait::wait_for_async( - || watcher.request(Inspect), - |entries| !matches!(entries, Ok(n) if *n > 0), - ) - .await; - assert_eq!( - entries.unwrap(), - 0, - "monitor table should be empty after target died" - ); - }); + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); + + let _ = watcher + .request(StartMonitor(target.child_handle())) + .await + .unwrap(); + assert_eq!(watcher.request(Inspect).await.unwrap(), 1); + + target.request(StopCounter).await.unwrap(); + target.join().await; + let entries = crate::test_wait::wait_for_async( + || watcher.request(Inspect), + |entries| !matches!(entries, Ok(n) if *n > 0), + ) + .await; + assert_eq!( + entries.unwrap(), + 0, + "monitor table should be empty after target died" + ); + }); + }, + ); } #[test] pub fn monitor_observes_panic_reason() { - struct PanicMsg; - impl Message for PanicMsg { - type Result = (); - } - struct PanicMe; - impl Actor for PanicMe {} - impl Handler for PanicMe { - async fn handle( - &mut self, - _msg: PanicMsg, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("intentional panic"); + crate::test_wait::bounded( + concat!(module_path!(), "::monitor_observes_panic_reason"), + move || { + struct PanicMsg; + impl Message for PanicMsg { + type Result = (); + } + struct PanicMe; + impl Actor for PanicMe {} + impl Handler for PanicMe { + async fn handle( + &mut self, + _msg: PanicMsg, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("intentional panic"); + } + } } - } - } - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = PanicMe.start(); - let target_handle = target.child_handle(); + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = PanicMe.start(); + let target_handle = target.child_handle(); - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); - let _ = watcher.request(StartMonitor(target_handle)).await.unwrap(); - let _ = target.send(PanicMsg); + let _ = watcher.request(StartMonitor(target_handle)).await.unwrap(); + let _ = target.send(PanicMsg); - let downs = crate::test_wait::wait_for_async( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), - ) - .await - .unwrap(); - assert_eq!(downs.len(), 1); - assert!(matches!(downs[0].reason, ExitReason::Panic(_))); - }); + let downs = crate::test_wait::wait_for_async( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), + ) + .await + .unwrap(); + assert_eq!(downs.len(), 1); + assert!(matches!(downs[0].reason, ExitReason::Panic(_))); + }); + }, + ); } #[test] pub fn monitoring_actor_stops_before_target_does_not_panic() { - // Regression: if the monitoring actor stops while the target is still - // alive, the watcher must not crash when the target eventually dies. - // The watcher's send() will fail silently (mailbox closed), and the - // watcher exits cleanly. - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::monitoring_actor_stops_before_target_does_not_panic" + ), + move || { + // Regression: if the monitoring actor stops while the target is still + // alive, the watcher must not crash when the target eventually dies. + // The watcher's send() will fail silently (mailbox closed), and the + // watcher exits cleanly. + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); - let _ = watcher.request(StartMonitor(target_handle)).await.unwrap(); + let _ = watcher.request(StartMonitor(target_handle)).await.unwrap(); - // Stop the monitoring actor first - let watcher_handle = watcher.child_handle(); - watcher_handle.stop(); - watcher_handle.wait_exit_async().await; + // Stop the monitoring actor first + let watcher_handle = watcher.child_handle(); + watcher_handle.stop(); + watcher_handle.wait_exit_async().await; - // Now stop the target — the watcher should clean up without panicking - target.request(StopCounter).await.unwrap(); - target.join().await; - // Give the orphaned watcher task time to attempt delivery - rt::sleep(Duration::from_millis(50)).await; - // If we got here without panicking, the test passes. - }); + // Now stop the target — the watcher should clean up without panicking + target.request(StopCounter).await.unwrap(); + target.join().await; + // Give the orphaned watcher task time to attempt delivery + rt::sleep(Duration::from_millis(50)).await; + // If we got here without panicking, the test passes. + }); + }, + ); } // --- Link tests --- @@ -3557,401 +3720,461 @@ mod tests { #[test] pub fn link_propagates_panic_to_non_trapping_peer() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - // Two non-trapping actors. When A panics, B should be terminated. - let a = make_trapper(false); - let b = make_trapper(false); - a.request(LinkTo(b.child_handle())).await.unwrap(); - - // Make A panic by dropping its only ActorRef while it's mid-handler... easier: - // make A panic via a panic message - struct Boom; - impl Message for Boom { - type Result = (); - } - impl Handler for TrapActor { - async fn handle( - &mut self, - _msg: Boom, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom"); + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_propagates_panic_to_non_trapping_peer" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + // Two non-trapping actors. When A panics, B should be terminated. + let a = make_trapper(false); + let b = make_trapper(false); + a.request(LinkTo(b.child_handle())).await.unwrap(); + + // Make A panic by dropping its only ActorRef while it's mid-handler... easier: + // make A panic via a panic message + struct Boom; + impl Message for Boom { + type Result = (); + } + impl Handler for TrapActor { + async fn handle( + &mut self, + _msg: Boom, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom"); + } + } } - } - } - let _ = a.send(Boom); - // Both should die - let reason_a = a.wait_exit().await; - let reason_b = b.wait_exit().await; - assert!(matches!(reason_a, ExitReason::Panic(_))); - // B should propagate A's reason (transitive propagation) - assert!(matches!(reason_b, ExitReason::Panic(_))); - }); + let _ = a.send(Boom); + // Both should die + let reason_a = a.wait_exit().await; + let reason_b = b.wait_exit().await; + assert!(matches!(reason_a, ExitReason::Panic(_))); + // B should propagate A's reason (transitive propagation) + assert!(matches!(reason_b, ExitReason::Panic(_))); + }); + }, + ); } #[test] pub fn link_delivers_exit_to_trapping_peer() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let a = make_trapper(false); - let b = make_trapper(true); // B traps - b.request(LinkTo(a.child_handle())).await.unwrap(); - - struct Boom2; - impl Message for Boom2 { - type Result = (); - } - impl Handler for TrapActor { - async fn handle( - &mut self, - _msg: Boom2, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom2"); + crate::test_wait::bounded( + concat!(module_path!(), "::link_delivers_exit_to_trapping_peer"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let a = make_trapper(false); + let b = make_trapper(true); // B traps + b.request(LinkTo(a.child_handle())).await.unwrap(); + + struct Boom2; + impl Message for Boom2 { + type Result = (); + } + impl Handler for TrapActor { + async fn handle( + &mut self, + _msg: Boom2, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom2"); + } + } } - } - } - let _ = a.send(Boom2); - a.wait_exit().await; - let exits = crate::test_wait::wait_for_async( - || b.request(GetExits), - |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), - ) - .await - .unwrap(); + let _ = a.send(Boom2); + a.wait_exit().await; + let exits = crate::test_wait::wait_for_async( + || b.request(GetExits), + |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), + ) + .await + .unwrap(); - // B should still be alive and have received an Exit - assert!(b.exit_reason().is_none()); - assert_eq!(exits.len(), 1); - assert_eq!(exits[0].from, a.id()); - assert!(matches!(exits[0].reason, ExitReason::Panic(_))); + // B should still be alive and have received an Exit + assert!(b.exit_reason().is_none()); + assert_eq!(exits.len(), 1); + assert_eq!(exits[0].from, a.id()); + assert!(matches!(exits[0].reason, ExitReason::Panic(_))); - // Clean up - let bh = b.child_handle(); - bh.stop(); - bh.wait_exit_async().await; - }); + // Clean up + let bh = b.child_handle(); + bh.stop(); + bh.wait_exit_async().await; + }); + }, + ); } #[test] pub fn link_normal_exit_not_propagated_to_non_trapping_peer() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let a = Counter { count: 0 }.start(); - let b = make_trapper(false); - b.request(LinkTo(a.child_handle())).await.unwrap(); - - // A stops cleanly - a.request(StopCounter).await.unwrap(); - a.join().await; - rt::sleep(Duration::from_millis(50)).await; - - // B should still be alive - assert!(b.exit_reason().is_none()); - - let bh = b.child_handle(); - bh.stop(); - bh.wait_exit_async().await; - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_normal_exit_not_propagated_to_non_trapping_peer" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let a = Counter { count: 0 }.start(); + let b = make_trapper(false); + b.request(LinkTo(a.child_handle())).await.unwrap(); + + // A stops cleanly + a.request(StopCounter).await.unwrap(); + a.join().await; + rt::sleep(Duration::from_millis(50)).await; + + // B should still be alive + assert!(b.exit_reason().is_none()); + + let bh = b.child_handle(); + bh.stop(); + bh.wait_exit_async().await; + }); + }, + ); } #[test] pub fn link_to_already_dead_actor_delivers_noproc() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - // Start and panic a target - struct Boom3; - impl Message for Boom3 { - type Result = (); - } - impl Handler for TrapActor { - async fn handle( - &mut self, - _msg: Boom3, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom3"); + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_to_already_dead_actor_delivers_noproc" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + // Start and panic a target + struct Boom3; + impl Message for Boom3 { + type Result = (); } - } - } - let target = make_trapper(false); - let _ = target.send(Boom3); - target.wait_exit().await; - - // Now link from a trapping observer: the target is past its - // death point, so the Exit carries NoProc (E4), not its reason. - let observer = make_trapper(true); - observer - .request(LinkTo(target.child_handle())) - .await - .unwrap(); - - let exits = crate::test_wait::wait_for_async( - || observer.request(GetExits), - |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), - ) - .await - .unwrap(); - assert_eq!(exits.len(), 1); - assert_eq!(exits[0].reason, ExitReason::NoProc); - assert!(matches!(target.exit_reason(), Some(ExitReason::Panic(_)))); + impl Handler for TrapActor { + async fn handle( + &mut self, + _msg: Boom3, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom3"); + } + } + } + let target = make_trapper(false); + let _ = target.send(Boom3); + target.wait_exit().await; + + // Now link from a trapping observer: the target is past its + // death point, so the Exit carries NoProc (E4), not its reason. + let observer = make_trapper(true); + observer + .request(LinkTo(target.child_handle())) + .await + .unwrap(); + + let exits = crate::test_wait::wait_for_async( + || observer.request(GetExits), + |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), + ) + .await + .unwrap(); + assert_eq!(exits.len(), 1); + assert_eq!(exits[0].reason, ExitReason::NoProc); + assert!(matches!(target.exit_reason(), Some(ExitReason::Panic(_)))); - let oh = observer.child_handle(); - oh.stop(); - oh.wait_exit_async().await; - }); + let oh = observer.child_handle(); + oh.stop(); + oh.wait_exit_async().await; + }); + }, + ); } #[test] pub fn unlink_prevents_signal_delivery() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let a = Counter { count: 0 }.start(); - let b = make_trapper(true); - b.request(LinkTo(a.child_handle())).await.unwrap(); - - // Now unlink - struct UnlinkFrom(crate::ChildHandle); - impl Message for UnlinkFrom { - type Result = (); - } - impl Handler for TrapActor { - async fn handle( - &mut self, - msg: UnlinkFrom, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - ctx.unlink(&msg.0); - reply.reply(()); - } - } - b.request(UnlinkFrom(a.child_handle())).await.unwrap(); + crate::test_wait::bounded( + concat!(module_path!(), "::unlink_prevents_signal_delivery"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let a = Counter { count: 0 }.start(); + let b = make_trapper(true); + b.request(LinkTo(a.child_handle())).await.unwrap(); + + // Now unlink + struct UnlinkFrom(crate::ChildHandle); + impl Message for UnlinkFrom { + type Result = (); + } + impl Handler for TrapActor { + async fn handle( + &mut self, + msg: UnlinkFrom, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.unlink(&msg.0); + reply.reply(()); + } + } + b.request(UnlinkFrom(a.child_handle())).await.unwrap(); - // A panics - struct Boom4; - impl Message for Boom4 { - type Result = u64; - } - impl Handler for Counter { - async fn handle( - &mut self, - _msg: Boom4, - _reply: crate::reply::PendingReply<'_, u64>, - _ctx: &Context, - ) { - panic!("boom4"); - } - } - // Counter's Boom4 returns u64; just use send with no expectation of reply - let _ = a.send(Boom4); - a.wait_exit().await; - rt::sleep(Duration::from_millis(50)).await; - - // B should NOT have received an Exit - let exits = b.request(GetExits).await.unwrap(); - assert!(exits.is_empty()); - - let bh = b.child_handle(); - bh.stop(); - bh.wait_exit_async().await; - }); + // A panics + struct Boom4; + impl Message for Boom4 { + type Result = u64; + } + impl Handler for Counter { + async fn handle( + &mut self, + _msg: Boom4, + _reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + panic!("boom4"); + } + } + // Counter's Boom4 returns u64; just use send with no expectation of reply + let _ = a.send(Boom4); + a.wait_exit().await; + rt::sleep(Duration::from_millis(50)).await; + + // B should NOT have received an Exit + let exits = b.request(GetExits).await.unwrap(); + assert!(exits.is_empty()); + + let bh = b.child_handle(); + bh.stop(); + bh.wait_exit_async().await; + }); + }, + ); } #[test] pub fn duplicate_link_is_idempotent() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let a = Counter { count: 0 }.start(); - let b = make_trapper(true); - b.request(LinkTo(a.child_handle())).await.unwrap(); - b.request(LinkTo(a.child_handle())).await.unwrap(); // duplicate - - struct Boom5; - impl Message for Boom5 { - type Result = u64; - } - impl Handler for Counter { - async fn handle( - &mut self, - _msg: Boom5, - _reply: crate::reply::PendingReply<'_, u64>, - _ctx: &Context, - ) { - panic!("boom5"); - } - } - let _ = a.send(Boom5); - a.wait_exit().await; - crate::test_wait::wait_for_async( - || b.request(GetExits), - |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), - ) - .await - .unwrap(); - // Leave room for a duplicate before checking there is none. - rt::sleep(Duration::from_millis(50)).await; + crate::test_wait::bounded( + concat!(module_path!(), "::duplicate_link_is_idempotent"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let a = Counter { count: 0 }.start(); + let b = make_trapper(true); + b.request(LinkTo(a.child_handle())).await.unwrap(); + b.request(LinkTo(a.child_handle())).await.unwrap(); // duplicate + + struct Boom5; + impl Message for Boom5 { + type Result = u64; + } + impl Handler for Counter { + async fn handle( + &mut self, + _msg: Boom5, + _reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + panic!("boom5"); + } + } + let _ = a.send(Boom5); + a.wait_exit().await; + crate::test_wait::wait_for_async( + || b.request(GetExits), + |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), + ) + .await + .unwrap(); + // Leave room for a duplicate before checking there is none. + rt::sleep(Duration::from_millis(50)).await; - // Should receive only ONE Exit, not two - let exits = b.request(GetExits).await.unwrap(); - assert_eq!(exits.len(), 1); + // Should receive only ONE Exit, not two + let exits = b.request(GetExits).await.unwrap(); + assert_eq!(exits.len(), 1); - let bh = b.child_handle(); - bh.stop(); - bh.wait_exit_async().await; - }); + let bh = b.child_handle(); + bh.stop(); + bh.wait_exit_async().await; + }); + }, + ); } #[test] pub fn chain_propagation_through_non_trapping_middle() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - // A linked to B linked to C. A panics → B dies (not trapping) → - // B's death propagates to C. - let a = make_trapper(false); - let b = make_trapper(false); - let c = make_trapper(true); // C traps - - a.request(LinkTo(b.child_handle())).await.unwrap(); - c.request(LinkTo(b.child_handle())).await.unwrap(); - - struct Boom6; - impl Message for Boom6 { - type Result = (); - } - impl Handler for TrapActor { - async fn handle( - &mut self, - _msg: Boom6, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom6"); + crate::test_wait::bounded( + concat!( + module_path!(), + "::chain_propagation_through_non_trapping_middle" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + // A linked to B linked to C. A panics → B dies (not trapping) → + // B's death propagates to C. + let a = make_trapper(false); + let b = make_trapper(false); + let c = make_trapper(true); // C traps + + a.request(LinkTo(b.child_handle())).await.unwrap(); + c.request(LinkTo(b.child_handle())).await.unwrap(); + + struct Boom6; + impl Message for Boom6 { + type Result = (); } - } - } - let _ = a.send(Boom6); - a.wait_exit().await; - b.wait_exit().await; - let exits = crate::test_wait::wait_for_async( - || c.request(GetExits), - |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), - ) - .await - .unwrap(); - assert_eq!(exits.len(), 1, "expected C to receive Exit from B's death"); + impl Handler for TrapActor { + async fn handle( + &mut self, + _msg: Boom6, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom6"); + } + } + } + let _ = a.send(Boom6); + a.wait_exit().await; + b.wait_exit().await; + let exits = crate::test_wait::wait_for_async( + || c.request(GetExits), + |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), + ) + .await + .unwrap(); + assert_eq!(exits.len(), 1, "expected C to receive Exit from B's death"); - let ch = c.child_handle(); - ch.stop(); - ch.wait_exit_async().await; - }); + let ch = c.child_handle(); + ch.stop(); + ch.wait_exit_async().await; + }); + }, + ); } #[test] pub fn link_to_already_dead_delivers_exactly_once_to_trapping_peer() { - // Regression: previously, `ctx.link()` would deliver a duplicate Exit - // to trapping actors when the target died concurrently — once from - // propagate_exit (since register_link inserted self in target's table - // before target's drain ran) and once from the fallback exit_reason() - // check. - struct Boom7; - impl Message for Boom7 { - type Result = (); - } - impl Handler for TrapActor { - async fn handle( - &mut self, - _msg: Boom7, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom7"); + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_to_already_dead_delivers_exactly_once_to_trapping_peer" + ), + move || { + // Regression: previously, `ctx.link()` would deliver a duplicate Exit + // to trapping actors when the target died concurrently — once from + // propagate_exit (since register_link inserted self in target's table + // before target's drain ran) and once from the fallback exit_reason() + // check. + struct Boom7; + impl Message for Boom7 { + type Result = (); + } + impl Handler for TrapActor { + async fn handle( + &mut self, + _msg: Boom7, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom7"); + } + } } - } - } - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = make_trapper(false); - // Kick off the panic - let _ = target.send(Boom7); - // Wait for target to fully die so exit_reason() is Some - target.wait_exit().await; - - // Now link from a trapping observer — should receive Exit EXACTLY ONCE - let observer = make_trapper(true); - observer - .request(LinkTo(target.child_handle())) - .await - .unwrap(); - - crate::test_wait::wait_for_async( - || observer.request(GetExits), - |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), - ) - .await - .unwrap(); - // Leave room for a duplicate before checking there is none. - rt::sleep(Duration::from_millis(50)).await; - let exits = observer.request(GetExits).await.unwrap(); - assert_eq!(exits.len(), 1, "expected exactly one Exit, got {:?}", exits); - - let oh = observer.child_handle(); - oh.stop(); - oh.wait_exit_async().await; - }); + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = make_trapper(false); + // Kick off the panic + let _ = target.send(Boom7); + // Wait for target to fully die so exit_reason() is Some + target.wait_exit().await; + + // Now link from a trapping observer — should receive Exit EXACTLY ONCE + let observer = make_trapper(true); + observer + .request(LinkTo(target.child_handle())) + .await + .unwrap(); + + crate::test_wait::wait_for_async( + || observer.request(GetExits), + |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), + ) + .await + .unwrap(); + // Leave room for a duplicate before checking there is none. + rt::sleep(Duration::from_millis(50)).await; + let exits = observer.request(GetExits).await.unwrap(); + assert_eq!(exits.len(), 1, "expected exactly one Exit, got {:?}", exits); + + let oh = observer.child_handle(); + oh.stop(); + oh.wait_exit_async().await; + }); + }, + ); } #[test] pub fn start_linked_links_atomically() { - // Parent + child via start_linked. When parent stops via panic, - // child (not trapping) should die too. - struct Parent; - struct PanicParent; - impl Message for PanicParent { - type Result = (); - } - impl Actor for Parent {} - impl Handler for Parent { - async fn handle( - &mut self, - _msg: PanicParent, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("parent boom"); + crate::test_wait::bounded( + concat!(module_path!(), "::start_linked_links_atomically"), + move || { + // Parent + child via start_linked. When parent stops via panic, + // child (not trapping) should die too. + struct Parent; + struct PanicParent; + impl Message for PanicParent { + type Result = (); + } + impl Actor for Parent {} + impl Handler for Parent { + async fn handle( + &mut self, + _msg: PanicParent, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("parent boom"); + } + } } - } - } - - struct Child; - impl Actor for Child {} - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let parent = Parent.start(); - let child = Child.start_linked(&parent.context()); - - // Parent panics — child should die too - let _ = parent.send(PanicParent); - parent.wait_exit().await; - // Child should also die (link propagation) - let reason = child.wait_exit().await; - // Reason should be the parent's panic - assert!(matches!(reason, ExitReason::Panic(_))); - }); + struct Child; + impl Actor for Child {} + + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let parent = Parent.start(); + let child = Child.start_linked(&parent.context()); + + // Parent panics — child should die too + let _ = parent.send(PanicParent); + parent.wait_exit().await; + // Child should also die (link propagation) + let reason = child.wait_exit().await; + // Reason should be the parent's panic + assert!(matches!(reason, ExitReason::Panic(_))); + }); + }, + ); } } @@ -3972,22 +4195,30 @@ mod monitor_cleanup_regressions { } #[test] fn closed_monitor_table_rejects_late_registration() { - let rt = rt::Runtime::new().unwrap(); - let _entered = rt.enter(); - let observer = Idle.start(); - let target = Idle.start(); - // Model cleanup during an unexpected unwind before token cancellation. - drop(MonitorCleanup(observer.shared.monitors.clone())); - let child = target.child_handle(); - let waiter = child.lifetime_probe(); - observer.context().monitor(&child); - drop(child); - assert_eq!(waiter.strong_count(), 0); - assert!(observer.shared.monitors.lock().unwrap().is_none()); - observer.child_handle().stop(); - target.child_handle().stop(); - rt.block_on(observer.join()); - rt.block_on(target.join()); + crate::test_wait::bounded( + concat!( + module_path!(), + "::closed_monitor_table_rejects_late_registration" + ), + move || { + let rt = rt::Runtime::new().unwrap(); + let _entered = rt.enter(); + let observer = Idle.start(); + let target = Idle.start(); + // Model cleanup during an unexpected unwind before token cancellation. + drop(MonitorCleanup(observer.shared.monitors.clone())); + let child = target.child_handle(); + let waiter = child.lifetime_probe(); + observer.context().monitor(&child); + drop(child); + assert_eq!(waiter.strong_count(), 0); + assert!(observer.shared.monitors.lock().unwrap().is_none()); + observer.child_handle().stop(); + target.child_handle().stop(); + rt.block_on(observer.join()); + rt.block_on(target.join()); + }, + ); } fn check_cleanup(stop_owner: bool) { let rt = rt::Runtime::new().unwrap(); @@ -4043,11 +4274,16 @@ mod early_stop_regression { } #[test] fn panic_during_early_stop_is_reported() { - let rt = rt::Runtime::new().unwrap(); - let _enter = rt.enter(); - let actor = EarlyStop.start(); - assert!( - matches!(rt.block_on(actor.wait_exit()), ExitReason::Panic(message) if message.contains("cleanup failed")) + crate::test_wait::bounded( + concat!(module_path!(), "::panic_during_early_stop_is_reported"), + move || { + let rt = rt::Runtime::new().unwrap(); + let _enter = rt.enter(); + let actor = EarlyStop.start(); + assert!( + matches!(rt.block_on(actor.wait_exit()), ExitReason::Panic(message) if message.contains("cleanup failed")) + ); + }, ); } } @@ -4058,105 +4294,130 @@ mod turn_regressions { use std::sync::atomic::{AtomicUsize, Ordering}; #[test] fn idle_time_is_excluded_and_an_empty_mailbox_needs_no_yield() { - let mut turn = MessageTurn { - limit: 16, - adaptive: false, - slice: Some(Duration::from_micros(50)), - completed: 0, - began: Some(std::time::Instant::now()), - }; - turn.parked(); - assert!(turn.began.is_none()); - let wake = std::time::Instant::now(); - turn.resumed(); - assert!(turn.began.unwrap() >= wake); - futures::executor::block_on(turn.after_message(true)); - assert!(turn.began.is_none()); - assert_eq!(turn.completed, 0); + crate::test_wait::bounded( + concat!( + module_path!(), + "::idle_time_is_excluded_and_an_empty_mailbox_needs_no_yield" + ), + move || { + let mut turn = MessageTurn { + limit: 16, + adaptive: false, + slice: Some(Duration::from_micros(50)), + completed: 0, + began: Some(std::time::Instant::now()), + }; + turn.parked(); + assert!(turn.began.is_none()); + let wake = std::time::Instant::now(); + turn.resumed(); + assert!(turn.began.unwrap() >= wake); + futures::executor::block_on(turn.after_message(true)); + assert!(turn.began.is_none()); + assert_eq!(turn.completed, 0); + }, + ); } #[test] fn count_only_turn_never_starts_a_clock_and_yields_at_its_count() { - rt::Runtime::new().unwrap().block_on(async { - let mut turn = MessageTurn { - limit: 4, - adaptive: false, - slice: None, - completed: 0, - began: None, - }; - turn.resumed(); - for _ in 0..3 { - let mut message = Box::pin(turn.after_message(false)); - assert!(futures::poll!(message.as_mut()).is_ready()); - } - assert!(turn.began.is_none()); - let mut fourth = Box::pin(turn.after_message(false)); - assert!(futures::poll!(fourth.as_mut()).is_pending()); - fourth.await; - assert_eq!(turn.completed, 0); - assert!(turn.began.is_none()); - turn.parked(); - turn.resumed(); - assert!(turn.began.is_none()); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::count_only_turn_never_starts_a_clock_and_yields_at_its_count" + ), + move || { + rt::Runtime::new().unwrap().block_on(async { + let mut turn = MessageTurn { + limit: 4, + adaptive: false, + slice: None, + completed: 0, + began: None, + }; + turn.resumed(); + for _ in 0..3 { + let mut message = Box::pin(turn.after_message(false)); + assert!(futures::poll!(message.as_mut()).is_ready()); + } + assert!(turn.began.is_none()); + let mut fourth = Box::pin(turn.after_message(false)); + assert!(futures::poll!(fourth.as_mut()).is_pending()); + fourth.await; + assert_eq!(turn.completed, 0); + assert!(turn.began.is_none()); + turn.parked(); + turn.resumed(); + assert!(turn.began.is_none()); + }); + }, + ); } #[test] fn stop_during_batch_yield_does_not_execute_the_next_queued_message() { - struct Count(Arc); - impl Actor for Count {} - struct Inc; - impl Message for Inc { - type Result = (); - } - impl Handler for Count { - async fn handle( - &mut self, - _: Inc, - reply: crate::reply::PendingReply<'_, ()>, - _: &Context, - ) { - self.0.fetch_add(1, Ordering::SeqCst); - reply.reply(()); - } - } - rt::Runtime::new().unwrap().block_on(async { - let (tx, rx) = mpsc::channel(); - let token = CancellationToken::new(); - let (ctx, mut cleanup) = test_context(&tx, &token); - let count = Arc::new(AtomicUsize::new(0)); - for _ in 0..2 { - tx.send(Box::new(MessageEnvelope { - msg: Inc, - slot: SlotHandle::detached(), - #[cfg(feature = "metrics")] - observation: None, - #[cfg(feature = "lifecycle-trace")] - from: 0, - }) as Box + Send>) - .unwrap(); - } - let turn = MessageTurn { - limit: 1, - adaptive: false, - slice: Some(Duration::from_micros(50)), - completed: 0, - began: None, - }; - let mut job = Box::pin(run_actor( - Count(count.clone()), - ctx, - rx, - token.clone(), - &mut cleanup, - Some(turn), - )); - assert!(futures::poll!(job.as_mut()).is_pending()); - assert_eq!(count.load(Ordering::SeqCst), 1); - token.cancel(); - job.await; - assert_eq!(cleanup.die(), ExitReason::Normal); - assert_eq!(count.load(Ordering::SeqCst), 1); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::stop_during_batch_yield_does_not_execute_the_next_queued_message" + ), + move || { + struct Count(Arc); + impl Actor for Count {} + struct Inc; + impl Message for Inc { + type Result = (); + } + impl Handler for Count { + async fn handle( + &mut self, + _: Inc, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { + self.0.fetch_add(1, Ordering::SeqCst); + reply.reply(()); + } + } + rt::Runtime::new().unwrap().block_on(async { + let (tx, rx) = mpsc::channel(); + let token = CancellationToken::new(); + let (ctx, mut cleanup) = test_context(&tx, &token); + let count = Arc::new(AtomicUsize::new(0)); + for _ in 0..2 { + tx.send(Box::new(MessageEnvelope { + msg: Inc, + slot: SlotHandle::detached(), + #[cfg(feature = "metrics")] + observation: None, + #[cfg(feature = "lifecycle-trace")] + from: 0, + }) + as Box + Send>) + .unwrap(); + } + let turn = MessageTurn { + limit: 1, + adaptive: false, + slice: Some(Duration::from_micros(50)), + completed: 0, + began: None, + }; + let mut job = Box::pin(run_actor( + Count(count.clone()), + ctx, + rx, + token.clone(), + &mut cleanup, + Some(turn), + )); + assert!(futures::poll!(job.as_mut()).is_pending()); + assert_eq!(count.load(Ordering::SeqCst), 1); + token.cancel(); + job.await; + assert_eq!(cleanup.die(), ExitReason::Normal); + assert_eq!(count.load(Ordering::SeqCst), 1); + }); + }, + ); } } @@ -4183,43 +4444,51 @@ mod idle_stop_regression { } #[test] fn stop_while_idle_does_not_dispatch_a_queued_request_on_resume() { - rt::Runtime::new().unwrap().block_on(async { - let (tx, rx) = mpsc::channel(); - let token = CancellationToken::new(); - let (ctx, mut cleanup) = test_context(&tx, &token); - let count = Arc::new(AtomicUsize::new(0)); - let mut actor = Box::pin(run_actor( - Counter(count.clone()), - ctx, - rx, - token.clone(), - &mut cleanup, - #[cfg(feature = "backend-experiments")] - None, - )); - // Start and park the actual actor loop, without allowing another - // executor task to run it between admission and stop below. - assert!(futures::poll!(actor.as_mut()).is_pending()); - let (slot, response) = reply_slot(crate::reply::Deadline::Never, None); - tx.send(Box::new(MessageEnvelope { - msg: Increment, - slot, - #[cfg(feature = "metrics")] - observation: None, - #[cfg(feature = "lifecycle-trace")] - from: 0, - }) as Box + Send>) - .unwrap(); - token.cancel(); - actor.await; - assert_eq!(cleanup.die(), ExitReason::Normal); - assert_eq!( - count.load(Ordering::SeqCst), - 0, - "idle actor dispatched queued work after stop" - ); - assert!(response.await.is_err(), "queued request unexpectedly ran"); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::stop_while_idle_does_not_dispatch_a_queued_request_on_resume" + ), + move || { + rt::Runtime::new().unwrap().block_on(async { + let (tx, rx) = mpsc::channel(); + let token = CancellationToken::new(); + let (ctx, mut cleanup) = test_context(&tx, &token); + let count = Arc::new(AtomicUsize::new(0)); + let mut actor = Box::pin(run_actor( + Counter(count.clone()), + ctx, + rx, + token.clone(), + &mut cleanup, + #[cfg(feature = "backend-experiments")] + None, + )); + // Start and park the actual actor loop, without allowing another + // executor task to run it between admission and stop below. + assert!(futures::poll!(actor.as_mut()).is_pending()); + let (slot, response) = reply_slot(crate::reply::Deadline::Never, None); + tx.send(Box::new(MessageEnvelope { + msg: Increment, + slot, + #[cfg(feature = "metrics")] + observation: None, + #[cfg(feature = "lifecycle-trace")] + from: 0, + }) as Box + Send>) + .unwrap(); + token.cancel(); + actor.await; + assert_eq!(cleanup.die(), ExitReason::Normal); + assert_eq!( + count.load(Ordering::SeqCst), + 0, + "idle actor dispatched queued work after stop" + ); + assert!(response.await.is_err(), "queued request unexpectedly ran"); + }); + }, + ); } } @@ -4296,121 +4565,160 @@ mod api_coverage { #[test] fn context_recipient_reaches_its_actor() { - rt::Runtime::new().unwrap().block_on(async { - let actor = Echo.start(); - let (recipient, id) = actor.request(OwnRecipient).await.unwrap(); - assert_eq!(id, actor.id()); - assert!(matches!(request(&*recipient, Ask(7), WAIT).await, Ok(7))); - recipient.send(Ask(1)).unwrap(); - actor.child_handle().stop(); - actor.join().await; - // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). - assert!(matches!(recipient.send(Ask(1)), Err(ActorError::NoProc))); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::context_recipient_reaches_its_actor"), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Echo.start(); + let (recipient, id) = actor.request(OwnRecipient).await.unwrap(); + assert_eq!(id, actor.id()); + assert!(matches!(request(&*recipient, Ask(7), WAIT).await, Ok(7))); + recipient.send(Ask(1)).unwrap(); + actor.child_handle().stop(); + actor.join().await; + // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). + assert!(matches!(recipient.send(Ask(1)), Err(ActorError::NoProc))); + }); + }, + ); } #[test] fn requests_to_a_stopped_actor_fail_without_waiting() { - rt::Runtime::new().unwrap().block_on(async { - let actor = Echo.start(); - let recipient = actor.recipient::(); - actor.child_handle().stop(); - actor.join().await; - // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). - assert!(matches!( - actor.request_with_timeout(Ask(1), WAIT).await, - Err(ActorError::NoProc) - )); - assert!(matches!( - request(&*recipient, Ask(1), WAIT).await, - Err(ActorError::NoProc) - )); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::requests_to_a_stopped_actor_fail_without_waiting" + ), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Echo.start(); + let recipient = actor.recipient::(); + actor.child_handle().stop(); + actor.join().await; + // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). + assert!(matches!( + actor.request_with_timeout(Ask(1), WAIT).await, + Err(ActorError::NoProc) + )); + assert!(matches!( + request(&*recipient, Ask(1), WAIT).await, + Err(ActorError::NoProc) + )); + }); + }, + ); } #[test] fn admitted_requests_fail_when_the_actor_stops_before_replying() { - rt::Runtime::new().unwrap().block_on(async { - let actor = Echo.start(); - let recipient = actor.recipient::(); - let (release, held) = rt::oneshot::channel(); - actor.send(StopAfter(held)).unwrap(); - let mut direct = Box::pin(actor.request_with_timeout(Ask(1), WAIT)); - let mut erased = Box::pin(request(&*recipient, Ask(2), WAIT)); - assert!(futures::poll!(direct.as_mut()).is_pending()); - assert!(futures::poll!(erased.as_mut()).is_pending()); - release.send(()).unwrap(); - assert!(matches!( - direct.await, - Err(ActorError::ActorStopped(ExitReason::Normal)) - )); - assert!(matches!( - erased.await, - Err(ActorError::ActorStopped(ExitReason::Normal)) - )); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::admitted_requests_fail_when_the_actor_stops_before_replying" + ), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Echo.start(); + let recipient = actor.recipient::(); + let (release, held) = rt::oneshot::channel(); + actor.send(StopAfter(held)).unwrap(); + let mut direct = Box::pin(actor.request_with_timeout(Ask(1), WAIT)); + let mut erased = Box::pin(request(&*recipient, Ask(2), WAIT)); + assert!(futures::poll!(direct.as_mut()).is_pending()); + assert!(futures::poll!(erased.as_mut()).is_pending()); + release.send(()).unwrap(); + assert!(matches!( + direct.await, + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); + assert!(matches!( + erased.await, + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); + }); + }, + ); } #[test] fn erased_request_times_out() { - rt::Runtime::new().unwrap().block_on(async { - let actor = Echo.start(); - let recipient = actor.recipient::(); - let (release, held) = rt::oneshot::channel(); - actor.send(StopAfter(held)).unwrap(); - assert!(matches!( - request(&*recipient, Ask(1), Duration::from_millis(1)).await, - Err(ActorError::RequestTimeout) - )); - release.send(()).unwrap(); - actor.join().await; - }); + crate::test_wait::bounded( + concat!(module_path!(), "::erased_request_times_out"), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Echo.start(); + let recipient = actor.recipient::(); + let (release, held) = rt::oneshot::channel(); + actor.send(StopAfter(held)).unwrap(); + assert!(matches!( + request(&*recipient, Ask(1), Duration::from_millis(1)).await, + Err(ActorError::RequestTimeout) + )); + release.send(()).unwrap(); + actor.join().await; + }); + }, + ); } #[test] fn default_exit_received_keeps_a_trapping_actor_alive() { - struct Trapper; - impl Actor for Trapper { - async fn started(&mut self, ctx: &Context) { - ctx.trap_exit(true); - } - } - impl Handler for Trapper { - async fn handle( - &mut self, - msg: Ask, - reply: crate::reply::PendingReply<'_, u32>, - _: &Context, - ) { - reply.reply(msg.0); - } - } - rt::Runtime::new().unwrap().block_on(async { - let trapper = Trapper.start(); - assert!(matches!(trapper.request(Ask(0)).await, Ok(0))); - let peer = Echo.start(); - trapper.context().link(&peer.child_handle()); - peer.send(Boom).unwrap(); - assert!(matches!(peer.wait_exit().await, ExitReason::Panic(_))); - // The Exit is handled by the default `exit_received`, which ignores it. - assert!(matches!(trapper.request(Ask(3)).await, Ok(3))); - assert!(trapper.exit_reason().is_none()); - trapper - .child_handle() - .stop_with_reason(crate::ExitReason::Shutdown); - assert_eq!(trapper.wait_exit().await, ExitReason::Shutdown); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::default_exit_received_keeps_a_trapping_actor_alive" + ), + move || { + struct Trapper; + impl Actor for Trapper { + async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); + } + } + impl Handler for Trapper { + async fn handle( + &mut self, + msg: Ask, + reply: crate::reply::PendingReply<'_, u32>, + _: &Context, + ) { + reply.reply(msg.0); + } + } + rt::Runtime::new().unwrap().block_on(async { + let trapper = Trapper.start(); + assert!(matches!(trapper.request(Ask(0)).await, Ok(0))); + let peer = Echo.start(); + trapper.context().link(&peer.child_handle()); + peer.send(Boom).unwrap(); + assert!(matches!(peer.wait_exit().await, ExitReason::Panic(_))); + // The Exit is handled by the default `exit_received`, which ignores it. + assert!(matches!(trapper.request(Ask(3)).await, Ok(3))); + assert!(trapper.exit_reason().is_none()); + trapper + .child_handle() + .stop_with_reason(crate::ExitReason::Shutdown); + assert_eq!(trapper.wait_exit().await, ExitReason::Shutdown); + }); + }, + ); } #[test] fn debug_output_names_the_handle() { - rt::Runtime::new().unwrap().block_on(async { - let actor = Echo.start(); - assert!(format!("{actor:?}").starts_with("ActorRef")); - assert!(format!("{:?}", actor.context()).starts_with("Context")); - actor.child_handle().stop(); - actor.join().await; - }); + crate::test_wait::bounded( + concat!(module_path!(), "::debug_output_names_the_handle"), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Echo.start(); + assert!(format!("{actor:?}").starts_with("ActorRef")); + assert!(format!("{:?}", actor.context()).starts_with("Context")); + actor.child_handle().stop(); + actor.join().await; + }); + }, + ); } } @@ -4455,57 +4763,75 @@ mod shared_state_release { #[test] fn freed_after_stop_and_last_handle_dropped() { - rt::Runtime::new().unwrap().block_on(async { - let actor = Plain.start(); - let weak = actor.shared_weak(); - let extra = (actor.clone(), actor.context(), actor.child_handle()); - actor.context().stop(); - actor.join().await; - drop(extra); - drop(actor); - assert!(freed(&weak).await, "ActorShared outlived every handle"); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::freed_after_stop_and_last_handle_dropped"), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Plain.start(); + let weak = actor.shared_weak(); + let extra = (actor.clone(), actor.context(), actor.child_handle()); + actor.context().stop(); + actor.join().await; + drop(extra); + drop(actor); + assert!(freed(&weak).await, "ActorShared outlived every handle"); + }); + }, + ); } #[test] fn freed_when_context_was_stored_in_actor_state() { - rt::Runtime::new().unwrap().block_on(async { - let actor = Holder(None).start(); - let weak = actor.shared_weak(); - actor.context().stop(); - actor.join().await; - drop(actor); - assert!(freed(&weak).await, "Context kept in actor state leaked"); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::freed_when_context_was_stored_in_actor_state" + ), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Holder(None).start(); + let weak = actor.shared_weak(); + actor.context().stop(); + actor.join().await; + drop(actor); + assert!(freed(&weak).await, "Context kept in actor state leaked"); + }); + }, + ); } #[test] fn freed_with_links_and_monitors() { - rt::Runtime::new().unwrap().block_on(async { - let a = Plain.start(); - let b = Plain.start(); - let watcher = Plain.start(); - a.context().trap_exit(true); - a.context().link(&b.child_handle()); - let _monitor = watcher.context().monitor(&a.child_handle()); - watcher.context().monitor(&b.child_handle()); - let child = Plain.start_linked(&a.context()); - let weaks = [ - a.shared_weak(), - b.shared_weak(), - watcher.shared_weak(), - child.shared_weak(), - ]; - for actor in [&b, &a, &watcher] { - actor.context().stop(); - actor.join().await; - } - child.context().stop(); - child.join().await; - drop((a, b, watcher, child)); - for weak in &weaks { - assert!(freed(weak).await, "linked or monitored actor leaked"); - } - }); + crate::test_wait::bounded( + concat!(module_path!(), "::freed_with_links_and_monitors"), + move || { + rt::Runtime::new().unwrap().block_on(async { + let a = Plain.start(); + let b = Plain.start(); + let watcher = Plain.start(); + a.context().trap_exit(true); + a.context().link(&b.child_handle()); + let _monitor = watcher.context().monitor(&a.child_handle()); + watcher.context().monitor(&b.child_handle()); + let child = Plain.start_linked(&a.context()); + let weaks = [ + a.shared_weak(), + b.shared_weak(), + watcher.shared_weak(), + child.shared_weak(), + ]; + for actor in [&b, &a, &watcher] { + actor.context().stop(); + actor.join().await; + } + child.context().stop(); + child.join().await; + drop((a, b, watcher, child)); + for weak in &weaks { + assert!(freed(weak).await, "linked or monitored actor leaked"); + } + }); + }, + ); } } diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index f0dfb4fc..6f6706a4 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -1053,20 +1053,26 @@ mod tests { /// `u16::MAX` credits overflowed). #[tokio::test] async fn a_refused_emit_returns_its_credit_once() { - let owner = Owner.start(); - for credits in [1, u16::MAX] { - let state = JobState::new(owner.id(), credits); - let job = JobCtx { - state: state.clone(), - ctx: owner.context(), - }; - state.acquire_blocking().unwrap(); - state.cancel(); - assert_eq!(job.enqueue(Note), Err(JobGone)); - assert_eq!(state.emit_credits(), credits, "credit returned twice"); - } - owner.context().stop(); - owner.join().await; + crate::test_wait::within( + concat!(module_path!(), "::a_refused_emit_returns_its_credit_once"), + async move { + let owner = Owner.start(); + for credits in [1, u16::MAX] { + let state = JobState::new(owner.id(), credits); + let job = JobCtx { + state: state.clone(), + ctx: owner.context(), + }; + state.acquire_blocking().unwrap(); + state.cancel(); + assert_eq!(job.enqueue(Note), Err(JobGone)); + assert_eq!(state.emit_credits(), credits, "credit returned twice"); + } + owner.context().stop(); + owner.join().await; + }, + ) + .await; } /// `cancel_on_caller_gone` applies to reply jobs only: a plain job has no @@ -1074,16 +1080,25 @@ mod tests { #[cfg(debug_assertions)] #[tokio::test] async fn cancel_on_caller_gone_on_a_plain_job_is_a_bug() { - let owner = Owner.start(); - let ctx = owner.context(); - let refused = std::panic::catch_unwind(AssertUnwindSafe(|| { - ctx.reserve_blocking_job(JobOptions::new().cancel_on_caller_gone()) - .map(drop) - })); - assert!(refused.is_err(), "the option was accepted on a plain job"); - let plain = ctx.reserve_blocking_job(JobOptions::new()).map(drop); - assert!(plain.is_ok()); - ctx.stop(); - owner.join().await; + crate::test_wait::within( + concat!( + module_path!(), + "::cancel_on_caller_gone_on_a_plain_job_is_a_bug" + ), + async move { + let owner = Owner.start(); + let ctx = owner.context(); + let refused = std::panic::catch_unwind(AssertUnwindSafe(|| { + ctx.reserve_blocking_job(JobOptions::new().cancel_on_caller_gone()) + .map(drop) + })); + assert!(refused.is_err(), "the option was accepted on a plain job"); + let plain = ctx.reserve_blocking_job(JobOptions::new()).map(drop); + assert!(plain.is_ok()); + ctx.stop(); + owner.join().await; + }, + ) + .await; } } diff --git a/concurrency/src/tasks/stream_tests.rs b/concurrency/src/tasks/stream_tests.rs index 3a08af6c..98b411dd 100644 --- a/concurrency/src/tasks/stream_tests.rs +++ b/concurrency/src/tasks/stream_tests.rs @@ -82,172 +82,204 @@ impl Handler for Summatory { #[test] pub fn test_sum_numbers_from_stream() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let summatory = Summatory::new(0).start(); - let stream = stream::iter(vec![1u16, 2, 3, 4, 5].into_iter().map(Ok::)); - - let ctx = summatory.context(); - spawn_listener( - ctx, - stream.filter_map(|result| async move { result.ok().map(StreamMsg::Add) }), - ); - - let val = wait_for_async( - || summatory.request(GetValue), - |val| matches!(val, Ok(n) if *n >= 15), - ) - .await - .unwrap(); - assert_eq!(val, 15); - }) + crate::test_wait::bounded( + concat!(module_path!(), "::test_sum_numbers_from_stream"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let summatory = Summatory::new(0).start(); + let stream = stream::iter(vec![1u16, 2, 3, 4, 5].into_iter().map(Ok::)); + + let ctx = summatory.context(); + spawn_listener( + ctx, + stream.filter_map(|result| async move { result.ok().map(StreamMsg::Add) }), + ); + + let val = wait_for_async( + || summatory.request(GetValue), + |val| matches!(val, Ok(n) if *n >= 15), + ) + .await + .unwrap(); + assert_eq!(val, 15); + }) + }, + ); } #[test] pub fn test_sum_numbers_from_channel() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let summatory = Summatory::new(0).start(); - let (tx, rx) = spawned_rt::tasks::mpsc::channel::>(); - - spawned_rt::tasks::spawn(async move { - for i in 1..=5 { - tx.send(Ok(i)).unwrap(); - } - }); - - let ctx = summatory.context(); - spawn_listener( - ctx, - ReceiverStream::new(rx) - .filter_map(|result| async move { result.ok().map(StreamMsg::Add) }), - ); - - let val = wait_for_async( - || summatory.request(GetValue), - |val| matches!(val, Ok(n) if *n >= 15), - ) - .await - .unwrap(); - assert_eq!(val, 15); - }) + crate::test_wait::bounded( + concat!(module_path!(), "::test_sum_numbers_from_channel"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let summatory = Summatory::new(0).start(); + let (tx, rx) = spawned_rt::tasks::mpsc::channel::>(); + + spawned_rt::tasks::spawn(async move { + for i in 1..=5 { + tx.send(Ok(i)).unwrap(); + } + }); + + let ctx = summatory.context(); + spawn_listener( + ctx, + ReceiverStream::new(rx) + .filter_map(|result| async move { result.ok().map(StreamMsg::Add) }), + ); + + let val = wait_for_async( + || summatory.request(GetValue), + |val| matches!(val, Ok(n) if *n >= 15), + ) + .await + .unwrap(); + assert_eq!(val, 15); + }) + }, + ); } #[test] pub fn test_sum_numbers_from_broadcast_channel() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let summatory = Summatory::new(0).start(); - let (tx, rx) = tokio::sync::broadcast::channel::(5); - - spawned_rt::tasks::spawn(async move { - for i in 1u16..=5 { - tx.send(i).unwrap(); - } - }); - - let ctx = summatory.context(); - spawn_listener( - ctx, - BroadcastStream::new(rx) - .filter_map(|result| async move { result.ok().map(StreamMsg::Add) }), - ); - - let val = wait_for_async( - || summatory.request(GetValue), - |val| matches!(val, Ok(n) if *n >= 15), - ) - .await - .unwrap(); - assert_eq!(val, 15); - }) + crate::test_wait::bounded( + concat!(module_path!(), "::test_sum_numbers_from_broadcast_channel"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let summatory = Summatory::new(0).start(); + let (tx, rx) = tokio::sync::broadcast::channel::(5); + + spawned_rt::tasks::spawn(async move { + for i in 1u16..=5 { + tx.send(i).unwrap(); + } + }); + + let ctx = summatory.context(); + spawn_listener( + ctx, + BroadcastStream::new(rx) + .filter_map(|result| async move { result.ok().map(StreamMsg::Add) }), + ); + + let val = wait_for_async( + || summatory.request(GetValue), + |val| matches!(val, Ok(n) if *n >= 15), + ) + .await + .unwrap(); + assert_eq!(val, 15); + }) + }, + ); } #[test] pub fn test_stream_cancellation() { - const MESSAGE_INTERVAL: u64 = 250; - const READ_TIME: u64 = 850; - const STOP_TIME: u64 = 1100; - - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let summatory = Summatory::new(0).start(); - let (tx, rx) = spawned_rt::tasks::mpsc::channel::>(); - - spawned_rt::tasks::spawn(async move { - for i in 1..=5 { - tx.send(Ok(i)).unwrap(); - rt::sleep(Duration::from_millis(MESSAGE_INTERVAL)).await; - } - }); - - let ctx = summatory.context(); - let listener_handle = spawn_listener( - ctx.clone(), - ReceiverStream::new(rx) - .filter_map(|result| async move { result.ok().map(StreamMsg::Add) }), - ); - - let _ = send_after(Duration::from_millis(STOP_TIME), ctx, StopSum); - - rt::sleep(Duration::from_millis(READ_TIME)).await; - let val = summatory.request(GetValue).await.unwrap(); - - assert!((1..=15).contains(&val)); - - assert!(listener_handle.await.is_ok()); - - summatory.join().await; - assert!(summatory.request(GetValue).await.is_err()); - }) + crate::test_wait::bounded( + concat!(module_path!(), "::test_stream_cancellation"), + move || { + const MESSAGE_INTERVAL: u64 = 250; + const READ_TIME: u64 = 850; + const STOP_TIME: u64 = 1100; + + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let summatory = Summatory::new(0).start(); + let (tx, rx) = spawned_rt::tasks::mpsc::channel::>(); + + spawned_rt::tasks::spawn(async move { + for i in 1..=5 { + tx.send(Ok(i)).unwrap(); + rt::sleep(Duration::from_millis(MESSAGE_INTERVAL)).await; + } + }); + + let ctx = summatory.context(); + let listener_handle = spawn_listener( + ctx.clone(), + ReceiverStream::new(rx) + .filter_map(|result| async move { result.ok().map(StreamMsg::Add) }), + ); + + let _ = send_after(Duration::from_millis(STOP_TIME), ctx, StopSum); + + rt::sleep(Duration::from_millis(READ_TIME)).await; + let val = summatory.request(GetValue).await.unwrap(); + + assert!((1..=15).contains(&val)); + + assert!(listener_handle.await.is_ok()); + + summatory.join().await; + assert!(summatory.request(GetValue).await.is_err()); + }) + }, + ); } #[test] pub fn test_halting_on_stream_error() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let summatory = Summatory::new(0).start(); - let stream = tokio_stream::iter(vec![Ok(1u16), Ok(2), Ok(3), Err(()), Ok(4), Ok(5)]); - let msg_stream = stream.filter_map(|value| async move { - match value { - Ok(number) => Some(StreamMsg::Add(number)), - Err(_) => Some(StreamMsg::Error), - } - }); - - let ctx = summatory.context(); - spawn_listener(ctx, msg_stream); - - // The error message stops the actor. - summatory.join().await; - - let result = summatory.request(GetValue).await; - assert!(result.is_err()); - }) + crate::test_wait::bounded( + concat!(module_path!(), "::test_halting_on_stream_error"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let summatory = Summatory::new(0).start(); + let stream = + tokio_stream::iter(vec![Ok(1u16), Ok(2), Ok(3), Err(()), Ok(4), Ok(5)]); + let msg_stream = stream.filter_map(|value| async move { + match value { + Ok(number) => Some(StreamMsg::Add(number)), + Err(_) => Some(StreamMsg::Error), + } + }); + + let ctx = summatory.context(); + spawn_listener(ctx, msg_stream); + + // The error message stops the actor. + summatory.join().await; + + let result = summatory.request(GetValue).await; + assert!(result.is_err()); + }) + }, + ); } #[test] pub fn test_skipping_on_stream_error() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let summatory = Summatory::new(0).start(); - let stream = tokio_stream::iter(vec![Ok(1u16), Ok(2), Ok(3), Err(()), Ok(4), Ok(5)]); - let msg_stream = stream.filter_map(|value| async move { - match value { - Ok(number) => Some(StreamMsg::Add(number)), - Err(_) => None, - } - }); - - let ctx = summatory.context(); - spawn_listener(ctx, msg_stream); - - let val = wait_for_async( - || summatory.request(GetValue), - |val| matches!(val, Ok(n) if *n >= 15), - ) - .await - .unwrap(); - assert_eq!(val, 15); - }) + crate::test_wait::bounded( + concat!(module_path!(), "::test_skipping_on_stream_error"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let summatory = Summatory::new(0).start(); + let stream = + tokio_stream::iter(vec![Ok(1u16), Ok(2), Ok(3), Err(()), Ok(4), Ok(5)]); + let msg_stream = stream.filter_map(|value| async move { + match value { + Ok(number) => Some(StreamMsg::Add(number)), + Err(_) => None, + } + }); + + let ctx = summatory.context(); + spawn_listener(ctx, msg_stream); + + let val = wait_for_async( + || summatory.request(GetValue), + |val| matches!(val, Ok(n) if *n >= 15), + ) + .await + .unwrap(); + assert_eq!(val, 15); + }) + }, + ); } diff --git a/concurrency/src/tasks/timer_tests.rs b/concurrency/src/tasks/timer_tests.rs index ff580fe7..894affde 100644 --- a/concurrency/src/tasks/timer_tests.rs +++ b/concurrency/src/tasks/timer_tests.rs @@ -84,30 +84,35 @@ impl Handler for Repeater { #[test] pub fn test_send_interval_and_cancellation() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let repeater = Repeater::new(0).start(); - - // The interval keeps firing. - let count = wait_for_async( - || repeater.request(GetRepCount), - |count| matches!(count, Ok(n) if *n >= 3), - ) - .await - .unwrap(); - assert!(count >= 3, "interval stopped firing at {count}"); - - // StopTimer is handled before the next request. A tick that raced the - // cancellation may still land, but no more than one. - repeater.send(StopTimer).unwrap(); - let stopped_at = repeater.request(GetRepCount).await.unwrap(); - rt::sleep(Duration::from_millis(500)).await; - let after = repeater.request(GetRepCount).await.unwrap(); - assert!( - after <= stopped_at + 1, - "interval kept firing after cancellation: {stopped_at} -> {after}" - ); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::test_send_interval_and_cancellation"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let repeater = Repeater::new(0).start(); + + // The interval keeps firing. + let count = wait_for_async( + || repeater.request(GetRepCount), + |count| matches!(count, Ok(n) if *n >= 3), + ) + .await + .unwrap(); + assert!(count >= 3, "interval stopped firing at {count}"); + + // StopTimer is handled before the next request. A tick that raced the + // cancellation may still land, but no more than one. + repeater.send(StopTimer).unwrap(); + let stopped_at = repeater.request(GetRepCount).await.unwrap(); + rt::sleep(Duration::from_millis(500)).await; + let after = repeater.request(GetRepCount).await.unwrap(); + assert!( + after <= stopped_at + 1, + "interval kept firing after cancellation: {stopped_at} -> {after}" + ); + }); + }, + ); } // --- Delayed (send_after test) --- @@ -178,57 +183,67 @@ impl Handler for Delayed { #[test] pub fn test_send_after_and_cancellation() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let repeater = Delayed::new(0).start(); + crate::test_wait::bounded( + concat!(module_path!(), "::test_send_after_and_cancellation"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let repeater = Delayed::new(0).start(); - let ctx = repeater.context(); - let _ = send_after(Duration::from_millis(100), ctx, Inc); + let ctx = repeater.context(); + let _ = send_after(Duration::from_millis(100), ctx, Inc); - let count = wait_for_async( - || repeater.request(GetDelCount), - |count| matches!(count, Ok(n) if *n >= 1), - ) - .await - .unwrap(); - assert_eq!(1, count); + let count = wait_for_async( + || repeater.request(GetDelCount), + |count| matches!(count, Ok(n) if *n >= 1), + ) + .await + .unwrap(); + assert_eq!(1, count); - let ctx = repeater.context(); - let timer = send_after(Duration::from_millis(100), ctx, Inc); + let ctx = repeater.context(); + let timer = send_after(Duration::from_millis(100), ctx, Inc); - timer.cancellation_token.cancel(); + timer.cancellation_token.cancel(); - rt::sleep(Duration::from_millis(200)).await; + rt::sleep(Duration::from_millis(200)).await; - let count2 = repeater.request(GetDelCount).await.unwrap(); - assert_eq!(1, count2); - }); + let count2 = repeater.request(GetDelCount).await.unwrap(); + assert_eq!(1, count2); + }); + }, + ); } #[test] pub fn test_send_after_gen_server_teardown() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let repeater = Delayed::new(0).start(); - - let ctx = repeater.context(); - let _ = send_after(Duration::from_millis(100), ctx, Inc); - - let count = wait_for_async( - || repeater.request(GetDelCount), - |count| matches!(count, Ok(n) if *n >= 1), - ) - .await - .unwrap(); - assert_eq!(1, count); - - let ctx = repeater.context(); - let _ = send_after(Duration::from_millis(100), ctx, Inc); - - let count2 = repeater.request(StopDelayed).await.unwrap(); - - rt::sleep(Duration::from_millis(200)).await; - - assert_eq!(1, count2); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::test_send_after_gen_server_teardown"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let repeater = Delayed::new(0).start(); + + let ctx = repeater.context(); + let _ = send_after(Duration::from_millis(100), ctx, Inc); + + let count = wait_for_async( + || repeater.request(GetDelCount), + |count| matches!(count, Ok(n) if *n >= 1), + ) + .await + .unwrap(); + assert_eq!(1, count); + + let ctx = repeater.context(); + let _ = send_after(Duration::from_millis(100), ctx, Inc); + + let count2 = repeater.request(StopDelayed).await.unwrap(); + + rt::sleep(Duration::from_millis(200)).await; + + assert_eq!(1, count2); + }); + }, + ); } diff --git a/concurrency/src/test_wait.rs b/concurrency/src/test_wait.rs index 9138ec6b..6c875d60 100644 --- a/concurrency/src/test_wait.rs +++ b/concurrency/src/test_wait.rs @@ -40,3 +40,44 @@ where spawned_rt::tasks::sleep(POLL).await; } } + +/// Bound for a wait a bug can turn into a hang (`tests/support/deadline.rs` +/// uses the same): far above any healthy wait, which are all handshakes. +pub(crate) const HANG: Duration = Duration::from_secs(120); + +/// Run a test's body (its runtime and that runtime's teardown included) on +/// its own thread, named after the test. A body still running after [`HANG`] +/// fails the test by name instead of hanging the job; a panic in the body +/// (a failed assertion, or a `#[should_panic]` test's expected panic) is +/// re-raised unchanged. +pub(crate) fn bounded( + name: &str, + body: impl FnOnce() -> T + Send + 'static, +) -> T { + let (done, finished) = std::sync::mpsc::channel(); + let thread = std::thread::Builder::new() + .name(name.to_string()) + .spawn(move || { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(body)); + let _ = done.send(result); + }) + .expect("spawn the test body"); + match finished.recv_timeout(HANG) { + Ok(Ok(value)) => { + let _ = thread.join(); + value + } + Ok(Err(panic)) => std::panic::resume_unwind(panic), + Err(_) => panic!("timed out after {HANG:?}: {name} did not finish"), + } +} + +/// Async counterpart of [`bounded`] for `#[tokio::test]` bodies: the body's +/// awaits are bounded by [`HANG`] (the runtime's own teardown is not). +#[cfg(feature = "async")] +pub(crate) async fn within(name: &str, body: F) -> F::Output { + match spawned_rt::tasks::timeout(HANG, body).await { + Ok(output) => output, + Err(_) => panic!("timed out after {HANG:?}: {name} did not finish"), + } +} diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 853852f6..d41f9c42 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -1639,275 +1639,343 @@ mod tests { #[test] fn basic_send_and_request() { - let actor = Counter { count: 0 }.start(); - assert_eq!(actor.request(GetCount).unwrap(), 0); - assert_eq!(actor.request(Increment).unwrap(), 1); - // Same mailbox: the request is handled after the send. - actor.send(Increment).unwrap(); - assert_eq!(actor.request(GetCount).unwrap(), 2); - actor.request(StopCounter).unwrap(); + crate::test_wait::bounded( + concat!(module_path!(), "::basic_send_and_request"), + move || { + let actor = Counter { count: 0 }.start(); + assert_eq!(actor.request(GetCount).unwrap(), 0); + assert_eq!(actor.request(Increment).unwrap(), 1); + // Same mailbox: the request is handled after the send. + actor.send(Increment).unwrap(); + assert_eq!(actor.request(GetCount).unwrap(), 2); + actor.request(StopCounter).unwrap(); + }, + ); } #[test] fn join_waits_for_completion() { - struct SlowStop; - struct StopSlow; - impl Message for StopSlow { - type Result = (); - } - impl Actor for SlowStop { - fn stopped(&mut self, _ctx: &Context) { - rt::sleep(Duration::from_millis(300)); - } - } - impl Handler for SlowStop { - fn handle( - &mut self, - _msg: StopSlow, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - ctx.stop(); - reply.reply(()); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::join_waits_for_completion"), + move || { + struct SlowStop; + struct StopSlow; + impl Message for StopSlow { + type Result = (); + } + impl Actor for SlowStop { + fn stopped(&mut self, _ctx: &Context) { + rt::sleep(Duration::from_millis(300)); + } + } + impl Handler for SlowStop { + fn handle( + &mut self, + _msg: StopSlow, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.stop(); + reply.reply(()); + } + } - let actor = SlowStop.start(); - actor.send(StopSlow).unwrap(); - actor.join(); - // If join() returned, stopped() has completed + let actor = SlowStop.start(); + actor.send(StopSlow).unwrap(); + actor.join(); + // If join() returned, stopped() has completed + }, + ); } #[test] fn join_multiple_callers() { - struct SlowStop2; - struct StopSlow2; - impl Message for StopSlow2 { - type Result = (); - } - impl Actor for SlowStop2 { - fn stopped(&mut self, _ctx: &Context) { - rt::sleep(Duration::from_millis(200)); - } - } - impl Handler for SlowStop2 { - fn handle( - &mut self, - _msg: StopSlow2, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - ctx.stop(); - reply.reply(()); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::join_multiple_callers"), + move || { + struct SlowStop2; + struct StopSlow2; + impl Message for StopSlow2 { + type Result = (); + } + impl Actor for SlowStop2 { + fn stopped(&mut self, _ctx: &Context) { + rt::sleep(Duration::from_millis(200)); + } + } + impl Handler for SlowStop2 { + fn handle( + &mut self, + _msg: StopSlow2, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.stop(); + reply.reply(()); + } + } - let actor = SlowStop2.start(); - let a1 = actor.clone(); - let a2 = actor.clone(); - let t1 = thread::spawn(move || { - a1.join(); - 1u32 - }); - let t2 = thread::spawn(move || { - a2.join(); - 2u32 - }); - actor.send(StopSlow2).unwrap(); - assert_eq!(t1.join().unwrap(), 1); - assert_eq!(t2.join().unwrap(), 2); + let actor = SlowStop2.start(); + let a1 = actor.clone(); + let a2 = actor.clone(); + let t1 = thread::spawn(move || { + a1.join(); + 1u32 + }); + let t2 = thread::spawn(move || { + a2.join(); + 2u32 + }); + actor.send(StopSlow2).unwrap(); + assert_eq!(t1.join().unwrap(), 1); + assert_eq!(t2.join().unwrap(), 2); + }, + ); } #[test] fn panic_in_started_stops_actor() { - struct PanicOnStart; - struct PingThread; - impl Message for PingThread { - type Result = (); - } - impl Actor for PanicOnStart { - fn started(&mut self, _ctx: &Context) { - panic!("boom in started"); - } - } - impl Handler for PanicOnStart { - fn handle( - &mut self, - _msg: PingThread, - reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - reply.reply(()); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::panic_in_started_stops_actor"), + move || { + struct PanicOnStart; + struct PingThread; + impl Message for PingThread { + type Result = (); + } + impl Actor for PanicOnStart { + fn started(&mut self, _ctx: &Context) { + panic!("boom in started"); + } + } + impl Handler for PanicOnStart { + fn handle( + &mut self, + _msg: PingThread, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + reply.reply(()); + } + } - let actor = PanicOnStart.start(); - actor.join(); - let result = actor.send(PingThread); - assert!(result.is_err()); + let actor = PanicOnStart.start(); + actor.join(); + let result = actor.send(PingThread); + assert!(result.is_err()); + }, + ); } #[test] fn panic_in_handler_stops_actor() { - struct PanicOnMsg; - struct ExplodeThread; - impl Message for ExplodeThread { - type Result = (); - } - struct CheckThread; - impl Message for CheckThread { - type Result = u32; - } - impl Actor for PanicOnMsg {} - impl Handler for PanicOnMsg { - fn handle( - &mut self, - _msg: ExplodeThread, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom in handler"); + crate::test_wait::bounded( + concat!(module_path!(), "::panic_in_handler_stops_actor"), + move || { + struct PanicOnMsg; + struct ExplodeThread; + impl Message for ExplodeThread { + type Result = (); + } + struct CheckThread; + impl Message for CheckThread { + type Result = u32; + } + impl Actor for PanicOnMsg {} + impl Handler for PanicOnMsg { + fn handle( + &mut self, + _msg: ExplodeThread, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom in handler"); + } + } + } + impl Handler for PanicOnMsg { + fn handle( + &mut self, + _msg: CheckThread, + reply: crate::reply::PendingReply<'_, u32>, + _ctx: &Context, + ) { + reply.reply(42); + } } - } - } - impl Handler for PanicOnMsg { - fn handle( - &mut self, - _msg: CheckThread, - reply: crate::reply::PendingReply<'_, u32>, - _ctx: &Context, - ) { - reply.reply(42); - } - } - let actor = PanicOnMsg.start(); - actor.send(ExplodeThread).unwrap(); - actor.join(); - let result = actor.request(CheckThread); - assert!(result.is_err()); + let actor = PanicOnMsg.start(); + actor.send(ExplodeThread).unwrap(); + actor.join(); + let result = actor.request(CheckThread); + assert!(result.is_err()); + }, + ); } #[test] fn panic_in_stopped_still_completes() { - struct PanicOnStop; - struct StopMeThread; - impl Message for StopMeThread { - type Result = (); - } - impl Actor for PanicOnStop { - fn stopped(&mut self, _ctx: &Context) { - panic!("boom in stopped"); - } - } - impl Handler for PanicOnStop { - fn handle( - &mut self, - _msg: StopMeThread, - reply: crate::reply::PendingReply<'_, ()>, - ctx: &Context, - ) { - ctx.stop(); - reply.reply(()); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::panic_in_stopped_still_completes"), + move || { + struct PanicOnStop; + struct StopMeThread; + impl Message for StopMeThread { + type Result = (); + } + impl Actor for PanicOnStop { + fn stopped(&mut self, _ctx: &Context) { + panic!("boom in stopped"); + } + } + impl Handler for PanicOnStop { + fn handle( + &mut self, + _msg: StopMeThread, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.stop(); + reply.reply(()); + } + } - let actor = PanicOnStop.start(); - actor.send(StopMeThread).unwrap(); - actor.join(); + let actor = PanicOnStop.start(); + actor.send(StopMeThread).unwrap(); + actor.join(); + }, + ); } #[test] fn recipient_type_erasure() { - let actor = Counter { count: 42 }.start(); - let recipient: Recipient = actor.recipient(); - let result = request(&*recipient, GetCount, Duration::from_secs(5)).unwrap(); - assert_eq!(result, 42); + crate::test_wait::bounded( + concat!(module_path!(), "::recipient_type_erasure"), + move || { + let actor = Counter { count: 42 }.start(); + let recipient: Recipient = actor.recipient(); + let result = request(&*recipient, GetCount, Duration::from_secs(5)).unwrap(); + assert_eq!(result, 42); + }, + ); } #[test] fn send_message_on_delivers() { - let actor = Counter { count: 0 }.start(); - let ctx = actor.context(); - send_message_on(ctx, || rt::sleep(Duration::from_millis(10)), Increment); - let count = - crate::test_wait::wait_for(|| actor.request(GetCount), |c| matches!(c, Ok(1))).unwrap(); - assert_eq!(count, 1); + crate::test_wait::bounded( + concat!(module_path!(), "::send_message_on_delivers"), + move || { + let actor = Counter { count: 0 }.start(); + let ctx = actor.context(); + send_message_on(ctx, || rt::sleep(Duration::from_millis(10)), Increment); + let count = + crate::test_wait::wait_for(|| actor.request(GetCount), |c| matches!(c, Ok(1))) + .unwrap(); + assert_eq!(count, 1); + }, + ); } // --- ExitReason tests --- #[test] fn exit_reason_normal_on_clean_stop() { - let actor = Counter { count: 0 }.start(); - actor.request(StopCounter).unwrap(); - let reason = actor.wait_exit(); - assert!(matches!(reason, ExitReason::Normal)); + crate::test_wait::bounded( + concat!(module_path!(), "::exit_reason_normal_on_clean_stop"), + move || { + let actor = Counter { count: 0 }.start(); + actor.request(StopCounter).unwrap(); + let reason = actor.wait_exit(); + assert!(matches!(reason, ExitReason::Normal)); + }, + ); } #[test] fn exit_reason_panic_in_started() { - struct PanicStartThread; - struct PingThread2; - impl Message for PingThread2 { - type Result = (); - } - impl Actor for PanicStartThread { - fn started(&mut self, _ctx: &Context) { - panic!("boom in started"); - } - } - impl Handler for PanicStartThread { - fn handle( - &mut self, - _msg: PingThread2, - reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - reply.reply(()); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::exit_reason_panic_in_started"), + move || { + struct PanicStartThread; + struct PingThread2; + impl Message for PingThread2 { + type Result = (); + } + impl Actor for PanicStartThread { + fn started(&mut self, _ctx: &Context) { + panic!("boom in started"); + } + } + impl Handler for PanicStartThread { + fn handle( + &mut self, + _msg: PingThread2, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + reply.reply(()); + } + } - let actor = PanicStartThread.start(); - let reason = actor.wait_exit(); - assert!(matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in started"))); + let actor = PanicStartThread.start(); + let reason = actor.wait_exit(); + assert!( + matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in started")) + ); + }, + ); } #[test] fn exit_reason_panic_in_handler() { - struct PanicHandlerThread; - struct ExplodeThread2; - impl Message for ExplodeThread2 { - type Result = (); - } - impl Actor for PanicHandlerThread {} - impl Handler for PanicHandlerThread { - fn handle( - &mut self, - _msg: ExplodeThread2, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom in handler"); + crate::test_wait::bounded( + concat!(module_path!(), "::exit_reason_panic_in_handler"), + move || { + struct PanicHandlerThread; + struct ExplodeThread2; + impl Message for ExplodeThread2 { + type Result = (); + } + impl Actor for PanicHandlerThread {} + impl Handler for PanicHandlerThread { + fn handle( + &mut self, + _msg: ExplodeThread2, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom in handler"); + } + } } - } - } - let actor = PanicHandlerThread.start(); - let _ = actor.send(ExplodeThread2); - let reason = actor.wait_exit(); - assert!(matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in handler"))); + let actor = PanicHandlerThread.start(); + let _ = actor.send(ExplodeThread2); + let reason = actor.wait_exit(); + assert!( + matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in handler")) + ); + }, + ); } #[test] fn exit_reason_poll_returns_none_while_running() { - let actor = Counter { count: 0 }.start(); - assert!(actor.exit_reason().is_none()); - actor.request(StopCounter).unwrap(); - actor.join(); - assert!(actor.exit_reason().is_some()); + crate::test_wait::bounded( + concat!( + module_path!(), + "::exit_reason_poll_returns_none_while_running" + ), + move || { + let actor = Counter { count: 0 }.start(); + assert!(actor.exit_reason().is_none()); + actor.request(StopCounter).unwrap(); + actor.join(); + assert!(actor.exit_reason().is_some()); + }, + ); } // --- Monitor tests --- @@ -2005,22 +2073,30 @@ mod tests { #[test] fn monitor_running_actor_delivers_down_on_exit() { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - let watcher = make_watcher(); - - let monitor_ref = watcher.request(StartMonitor(target_handle)).unwrap(); - - target.request(StopCounter).unwrap(); - target.join(); - let downs = crate::test_wait::wait_for( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), - ) - .unwrap(); - assert_eq!(downs.len(), 1); - assert_eq!(downs[0].monitor_ref, monitor_ref); - assert!(matches!(downs[0].reason, ExitReason::Normal)); + crate::test_wait::bounded( + concat!( + module_path!(), + "::monitor_running_actor_delivers_down_on_exit" + ), + move || { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + let watcher = make_watcher(); + + let monitor_ref = watcher.request(StartMonitor(target_handle)).unwrap(); + + target.request(StopCounter).unwrap(); + target.join(); + let downs = crate::test_wait::wait_for( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), + ) + .unwrap(); + assert_eq!(downs.len(), 1); + assert_eq!(downs[0].monitor_ref, monitor_ref); + assert!(matches!(downs[0].reason, ExitReason::Normal)); + }, + ); } /// OTP `demonitor(Ref, [flush])`: a `Down` already in the mailbox @@ -2028,112 +2104,146 @@ mod tests { /// never handled once `demonitor` returned. #[test] fn demonitor_flushes_a_queued_down() { - let target = Counter { count: 0 }.start(); - target.request(StopCounter).unwrap(); - target.join(); - let watcher = make_watcher(); - watcher - .request(MonitorThenDemonitor(target.child_handle())) - .unwrap(); - // FIFO: the queued Down is dispatched before this request. - let downs = watcher.request(GetDowns).unwrap(); - assert!(downs.is_empty(), "a flushed Down was handled: {downs:?}"); + crate::test_wait::bounded( + concat!(module_path!(), "::demonitor_flushes_a_queued_down"), + move || { + let target = Counter { count: 0 }.start(); + target.request(StopCounter).unwrap(); + target.join(); + let watcher = make_watcher(); + watcher + .request(MonitorThenDemonitor(target.child_handle())) + .unwrap(); + // FIFO: the queued Down is dispatched before this request. + let downs = watcher.request(GetDowns).unwrap(); + assert!(downs.is_empty(), "a flushed Down was handled: {downs:?}"); + }, + ); } #[test] fn monitor_already_dead_actor_delivers_down_immediately() { - let target = Counter { count: 0 }.start(); - target.request(StopCounter).unwrap(); - target.join(); - let target_handle = target.child_handle(); - - let watcher = make_watcher(); - let _ = watcher.request(StartMonitor(target_handle)).unwrap(); - let downs = crate::test_wait::wait_for( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), - ) - .unwrap(); - assert_eq!(downs.len(), 1); + crate::test_wait::bounded( + concat!( + module_path!(), + "::monitor_already_dead_actor_delivers_down_immediately" + ), + move || { + let target = Counter { count: 0 }.start(); + target.request(StopCounter).unwrap(); + target.join(); + let target_handle = target.child_handle(); + + let watcher = make_watcher(); + let _ = watcher.request(StartMonitor(target_handle)).unwrap(); + let downs = crate::test_wait::wait_for( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), + ) + .unwrap(); + assert_eq!(downs.len(), 1); + }, + ); } #[test] fn demonitor_before_target_dies_suppresses_down() { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - let watcher = make_watcher(); - - let monitor_ref = watcher.request(StartMonitor(target_handle)).unwrap(); - watcher.request(CallDemonitor(monitor_ref)).unwrap(); - - target.request(StopCounter).unwrap(); - target.join(); - rt::sleep(Duration::from_millis(150)); - - let downs = watcher.request(GetDowns).unwrap(); - assert!(downs.is_empty()); + crate::test_wait::bounded( + concat!( + module_path!(), + "::demonitor_before_target_dies_suppresses_down" + ), + move || { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + let watcher = make_watcher(); + + let monitor_ref = watcher.request(StartMonitor(target_handle)).unwrap(); + watcher.request(CallDemonitor(monitor_ref)).unwrap(); + + target.request(StopCounter).unwrap(); + target.join(); + rt::sleep(Duration::from_millis(150)); + + let downs = watcher.request(GetDowns).unwrap(); + assert!(downs.is_empty()); + }, + ); } #[test] fn multiple_monitors_each_get_own_ref_and_down() { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - let watcher = make_watcher(); - - let r1 = watcher - .request(StartMonitor(target_handle.clone())) - .unwrap(); - let r2 = watcher.request(StartMonitor(target_handle)).unwrap(); - assert_ne!(r1, r2); - - target.request(StopCounter).unwrap(); - target.join(); - let downs = crate::test_wait::wait_for( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| d.len() >= 2), - ) - .unwrap(); - assert_eq!(downs.len(), 2); - let refs: Vec<_> = downs.iter().map(|d| d.monitor_ref).collect(); - assert!(refs.contains(&r1)); - assert!(refs.contains(&r2)); + crate::test_wait::bounded( + concat!( + module_path!(), + "::multiple_monitors_each_get_own_ref_and_down" + ), + move || { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + let watcher = make_watcher(); + + let r1 = watcher + .request(StartMonitor(target_handle.clone())) + .unwrap(); + let r2 = watcher.request(StartMonitor(target_handle)).unwrap(); + assert_ne!(r1, r2); + + target.request(StopCounter).unwrap(); + target.join(); + let downs = crate::test_wait::wait_for( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| d.len() >= 2), + ) + .unwrap(); + assert_eq!(downs.len(), 2); + let refs: Vec<_> = downs.iter().map(|d| d.monitor_ref).collect(); + assert!(refs.contains(&r1)); + assert!(refs.contains(&r2)); + }, + ); } #[test] fn monitor_observes_panic_reason() { - struct PanicMsg; - impl Message for PanicMsg { - type Result = (); - } - struct PanicMe; - impl Actor for PanicMe {} - impl Handler for PanicMe { - fn handle( - &mut self, - _msg: PanicMsg, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("intentional panic"); + crate::test_wait::bounded( + concat!(module_path!(), "::monitor_observes_panic_reason"), + move || { + struct PanicMsg; + impl Message for PanicMsg { + type Result = (); + } + struct PanicMe; + impl Actor for PanicMe {} + impl Handler for PanicMe { + fn handle( + &mut self, + _msg: PanicMsg, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("intentional panic"); + } + } } - } - } - let target = PanicMe.start(); - let target_handle = target.child_handle(); - let watcher = make_watcher(); + let target = PanicMe.start(); + let target_handle = target.child_handle(); + let watcher = make_watcher(); - let _ = watcher.request(StartMonitor(target_handle)).unwrap(); - let _ = target.send(PanicMsg); + let _ = watcher.request(StartMonitor(target_handle)).unwrap(); + let _ = target.send(PanicMsg); - let downs = crate::test_wait::wait_for( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), - ) - .unwrap(); - assert_eq!(downs.len(), 1); - assert!(matches!(downs[0].reason, ExitReason::Panic(_))); + let downs = crate::test_wait::wait_for( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), + ) + .unwrap(); + assert_eq!(downs.len(), 1); + assert!(matches!(downs[0].reason, ExitReason::Panic(_))); + }, + ); } // --- Link tests --- @@ -2194,105 +2304,126 @@ mod tests { #[test] fn link_propagates_panic_to_non_trapping_peer_threads() { - struct Boom; - impl Message for Boom { - type Result = (); - } - impl Handler for TrapActor { - fn handle( - &mut self, - _msg: Boom, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom"); + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_propagates_panic_to_non_trapping_peer_threads" + ), + move || { + struct Boom; + impl Message for Boom { + type Result = (); + } + impl Handler for TrapActor { + fn handle( + &mut self, + _msg: Boom, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom"); + } + } } - } - } - let a = make_trapper(false); - let b = make_trapper(false); - a.request(LinkTo(b.child_handle())).unwrap(); + let a = make_trapper(false); + let b = make_trapper(false); + a.request(LinkTo(b.child_handle())).unwrap(); - let _ = a.send(Boom); - let reason_a = a.wait_exit(); - let reason_b = b.wait_exit(); - assert!(matches!(reason_a, ExitReason::Panic(_))); - assert!(matches!(reason_b, ExitReason::Panic(_))); + let _ = a.send(Boom); + let reason_a = a.wait_exit(); + let reason_b = b.wait_exit(); + assert!(matches!(reason_a, ExitReason::Panic(_))); + assert!(matches!(reason_b, ExitReason::Panic(_))); + }, + ); } #[test] fn link_delivers_exit_to_trapping_peer_threads() { - struct Boom; - impl Message for Boom { - type Result = (); - } - impl Handler for TrapActor { - fn handle( - &mut self, - _msg: Boom, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("boom"); + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_delivers_exit_to_trapping_peer_threads" + ), + move || { + struct Boom; + impl Message for Boom { + type Result = (); + } + impl Handler for TrapActor { + fn handle( + &mut self, + _msg: Boom, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom"); + } + } } - } - } - - let a = make_trapper(false); - let b = make_trapper(true); - b.request(LinkTo(a.child_handle())).unwrap(); - let _ = a.send(Boom); - a.wait_exit(); - let exits = crate::test_wait::wait_for( - || b.request(GetExits), - |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), - ) - .unwrap(); - assert!(b.exit_reason().is_none()); - assert_eq!(exits.len(), 1); - assert_eq!(exits[0].from, a.id()); - assert!(matches!(exits[0].reason, ExitReason::Panic(_))); + let a = make_trapper(false); + let b = make_trapper(true); + b.request(LinkTo(a.child_handle())).unwrap(); - let bh = b.child_handle(); - bh.stop(); - bh.wait_exit_blocking(); + let _ = a.send(Boom); + a.wait_exit(); + let exits = crate::test_wait::wait_for( + || b.request(GetExits), + |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), + ) + .unwrap(); + assert!(b.exit_reason().is_none()); + assert_eq!(exits.len(), 1); + assert_eq!(exits[0].from, a.id()); + assert!(matches!(exits[0].reason, ExitReason::Panic(_))); + + let bh = b.child_handle(); + bh.stop(); + bh.wait_exit_blocking(); + }, + ); } #[test] fn start_linked_threads() { - struct Parent; - struct PanicParent; - impl Message for PanicParent { - type Result = (); - } - impl Actor for Parent {} - impl Handler for Parent { - fn handle( - &mut self, - _msg: PanicParent, - _reply: crate::reply::PendingReply<'_, ()>, - _ctx: &Context, - ) { - { - panic!("parent boom"); + crate::test_wait::bounded( + concat!(module_path!(), "::start_linked_threads"), + move || { + struct Parent; + struct PanicParent; + impl Message for PanicParent { + type Result = (); + } + impl Actor for Parent {} + impl Handler for Parent { + fn handle( + &mut self, + _msg: PanicParent, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("parent boom"); + } + } } - } - } - struct Child; - impl Actor for Child {} + struct Child; + impl Actor for Child {} - let parent = Parent.start(); - let child = Child.start_linked(&parent.context()); + let parent = Parent.start(); + let child = Child.start_linked(&parent.context()); - let _ = parent.send(PanicParent); - parent.wait_exit(); - let reason = child.wait_exit(); - assert!(matches!(reason, ExitReason::Panic(_))); + let _ = parent.send(PanicParent); + parent.wait_exit(); + let reason = child.wait_exit(); + assert!(matches!(reason, ExitReason::Panic(_))); + }, + ); } } @@ -2313,20 +2444,28 @@ mod monitor_cleanup_regressions { } #[test] fn closed_monitor_table_rejects_late_registration() { - let observer = Idle.start(); - let target = Idle.start(); - // Model cleanup during an unexpected unwind before token cancellation. - drop(MonitorCleanup(observer.monitors.clone())); - let child = target.child_handle(); - let waiter = child.lifetime_probe(); - observer.context().monitor(&child); - drop(child); - assert_eq!(waiter.strong_count(), 0); - assert!(observer.monitors.lock().unwrap().is_none()); - observer.child_handle().stop(); - target.child_handle().stop(); - observer.join(); - target.join(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::closed_monitor_table_rejects_late_registration" + ), + move || { + let observer = Idle.start(); + let target = Idle.start(); + // Model cleanup during an unexpected unwind before token cancellation. + drop(MonitorCleanup(observer.monitors.clone())); + let child = target.child_handle(); + let waiter = child.lifetime_probe(); + observer.context().monitor(&child); + drop(child); + assert_eq!(waiter.strong_count(), 0); + assert!(observer.monitors.lock().unwrap().is_none()); + observer.child_handle().stop(); + target.child_handle().stop(); + observer.join(); + target.join(); + }, + ); } fn check_cleanup(stop_owner: bool) { let target = Idle.start(); @@ -2380,9 +2519,14 @@ mod early_stop_regression { } #[test] fn panic_during_early_stop_is_reported() { - let actor = EarlyStop.start(); - assert!( - matches!(actor.wait_exit(), ExitReason::Panic(message) if message.contains("cleanup failed")) + crate::test_wait::bounded( + concat!(module_path!(), "::panic_during_early_stop_is_reported"), + move || { + let actor = EarlyStop.start(); + assert!( + matches!(actor.wait_exit(), ExitReason::Panic(message) if message.contains("cleanup failed")) + ); + }, ); } } @@ -2431,89 +2575,104 @@ mod idle_stop_regression { } #[test] fn stop_while_receiving_does_not_dispatch_a_later_request() { - let (tx, rx) = mpsc::channel(); - let token = CancellationToken::new(); - let (_, completion_rx) = spawned_rt::sync::watch::channel(None); - let monitors = Arc::new(Mutex::new(Some(HashMap::new()))); - let events = Arc::new(Mutex::new(Vec::new())); - let (id, trap, request) = ( - ActorId::next(), - new_trap_exit_flag(), - new_linked_exit_reason(), - ); - let ctx = Context { - id, - sender: tx.clone(), - cancellation_token: token.clone(), - completion: completion_rx, - monitors: monitors.clone(), - trap_exit: trap.clone(), - links: new_link_table(), - linked_reason: request.clone(), - signal: signal_target::<_>(id, None, &tx, trap.clone(), request.clone(), token.clone()), - mailbox: crate::instrumentation::Mailbox::default(), - replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), - jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), - cleaned: Arc::default(), - }; - let count = Arc::new(AtomicUsize::new(0)); - let seen = count.clone(); - let entered = Arc::new(Barrier::new(2)); - let release = Arc::new(Barrier::new(2)); - let (e, r) = (entered.clone(), release.clone()); - let cancel = token.clone(); - let trace = events.clone(); - let actor = std::thread::spawn(move || { - let mut first = true; - let mut counter = Counter(seen, trace); - // The envelope received after the stop comes back undispatched; - // the death point answers it and destroys it before the state. - let discarded = run_actor_receiving( - &mut counter, - ctx, - move || { - // Pause the real receive boundary after the started/cancel - // checks; the standard channel still carries the envelope. - if first { - first = false; - e.wait(); - r.wait(); - } - rx.recv_timeout(Duration::from_secs(5)) - }, - cancel, - ); - drop(discarded); - drop(counter); - }); - entered.wait(); - let (slot, response) = reply_slot(crate::reply::Deadline::Never, None); - tx.send(MailboxItem::Message(Box::new(MessageEnvelope { - msg: Increment(events.clone(), monitors), - slot, - #[cfg(feature = "metrics")] - observation: None, - #[cfg(feature = "lifecycle-trace")] - from: 0, - }))) - .unwrap(); - token.cancel(); - release.wait(); - actor.join().unwrap(); - assert_eq!( - link::lock_request(&request).reason(), - None, - "a plain stop request" - ); - assert_eq!( - count.load(Ordering::SeqCst), - 0, - "thread actor dispatched work after idle stop" - ); - assert!(response.recv().is_err()); - assert_eq!( - *events.lock().unwrap(), - ["stopped", "message-dropped", "actor-dropped"] + crate::test_wait::bounded( + concat!( + module_path!(), + "::stop_while_receiving_does_not_dispatch_a_later_request" + ), + move || { + let (tx, rx) = mpsc::channel(); + let token = CancellationToken::new(); + let (_, completion_rx) = spawned_rt::sync::watch::channel(None); + let monitors = Arc::new(Mutex::new(Some(HashMap::new()))); + let events = Arc::new(Mutex::new(Vec::new())); + let (id, trap, request) = ( + ActorId::next(), + new_trap_exit_flag(), + new_linked_exit_reason(), + ); + let ctx = Context { + id, + sender: tx.clone(), + cancellation_token: token.clone(), + completion: completion_rx, + monitors: monitors.clone(), + trap_exit: trap.clone(), + links: new_link_table(), + linked_reason: request.clone(), + signal: signal_target::<_>( + id, + None, + &tx, + trap.clone(), + request.clone(), + token.clone(), + ), + mailbox: crate::instrumentation::Mailbox::default(), + replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), + jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), + cleaned: Arc::default(), + }; + let count = Arc::new(AtomicUsize::new(0)); + let seen = count.clone(); + let entered = Arc::new(Barrier::new(2)); + let release = Arc::new(Barrier::new(2)); + let (e, r) = (entered.clone(), release.clone()); + let cancel = token.clone(); + let trace = events.clone(); + let actor = std::thread::spawn(move || { + let mut first = true; + let mut counter = Counter(seen, trace); + // The envelope received after the stop comes back undispatched; + // the death point answers it and destroys it before the state. + let discarded = run_actor_receiving( + &mut counter, + ctx, + move || { + // Pause the real receive boundary after the started/cancel + // checks; the standard channel still carries the envelope. + if first { + first = false; + e.wait(); + r.wait(); + } + rx.recv_timeout(Duration::from_secs(5)) + }, + cancel, + ); + drop(discarded); + drop(counter); + }); + entered.wait(); + let (slot, response) = reply_slot(crate::reply::Deadline::Never, None); + tx.send(MailboxItem::Message(Box::new(MessageEnvelope { + msg: Increment(events.clone(), monitors), + slot, + #[cfg(feature = "metrics")] + observation: None, + #[cfg(feature = "lifecycle-trace")] + from: 0, + }))) + .unwrap(); + token.cancel(); + release.wait(); + actor.join().unwrap(); + assert_eq!( + link::lock_request(&request).reason(), + None, + "a plain stop request" + ); + assert_eq!( + count.load(Ordering::SeqCst), + 0, + "thread actor dispatched work after idle stop" + ); + assert!(response.recv().is_err()); + assert_eq!( + *events.lock().unwrap(), + ["stopped", "message-dropped", "actor-dropped"] + ); + }, ); } } @@ -2574,55 +2733,70 @@ mod stop_wakeup_regression { #[test] fn cancellation_wakes_the_real_blocking_receive_boundary() { - let (tx, rx) = mpsc::channel(); - let token = CancellationToken::new(); - register_stop_wakeup(&token, tx.clone()); - let (_, completion) = spawned_rt::sync::watch::channel(None); - let (id, trap, request) = ( - ActorId::next(), - new_trap_exit_flag(), - new_linked_exit_reason(), - ); - let ctx = Context { - id, - sender: tx.clone(), - cancellation_token: token.clone(), - completion, - monitors: Arc::new(Mutex::new(Some(HashMap::new()))), - trap_exit: trap.clone(), - links: new_link_table(), - linked_reason: request.clone(), - signal: signal_target::<_>(id, None, &tx, trap.clone(), request.clone(), token.clone()), - mailbox: crate::instrumentation::Mailbox::default(), - replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), - jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), - cleaned: Arc::default(), - }; - let (entered_tx, entered) = mpsc::channel(); - let (stopped_tx, stopped) = mpsc::channel(); - let cancel = token.clone(); - let actor = std::thread::spawn(move || { - run_actor_receiving( - &mut Idle(stopped_tx), - ctx, - move || { - let _ = entered_tx.send(()); - rx.recv() - .map_err(|_| std::sync::mpsc::RecvTimeoutError::Disconnected) - }, - cancel, - ) - }); - entered.recv_timeout(Duration::from_secs(2)).unwrap(); - token.cancel(); - stopped - .recv_timeout(Duration::from_secs(2)) - .expect("cancel did not wake recv"); - actor.join().unwrap(); - assert_eq!( - link::lock_request(&request).reason(), - None, - "a plain stop request" + crate::test_wait::bounded( + concat!( + module_path!(), + "::cancellation_wakes_the_real_blocking_receive_boundary" + ), + move || { + let (tx, rx) = mpsc::channel(); + let token = CancellationToken::new(); + register_stop_wakeup(&token, tx.clone()); + let (_, completion) = spawned_rt::sync::watch::channel(None); + let (id, trap, request) = ( + ActorId::next(), + new_trap_exit_flag(), + new_linked_exit_reason(), + ); + let ctx = Context { + id, + sender: tx.clone(), + cancellation_token: token.clone(), + completion, + monitors: Arc::new(Mutex::new(Some(HashMap::new()))), + trap_exit: trap.clone(), + links: new_link_table(), + linked_reason: request.clone(), + signal: signal_target::<_>( + id, + None, + &tx, + trap.clone(), + request.clone(), + token.clone(), + ), + mailbox: crate::instrumentation::Mailbox::default(), + replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), + jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), + cleaned: Arc::default(), + }; + let (entered_tx, entered) = mpsc::channel(); + let (stopped_tx, stopped) = mpsc::channel(); + let cancel = token.clone(); + let actor = std::thread::spawn(move || { + run_actor_receiving( + &mut Idle(stopped_tx), + ctx, + move || { + let _ = entered_tx.send(()); + rx.recv() + .map_err(|_| std::sync::mpsc::RecvTimeoutError::Disconnected) + }, + cancel, + ) + }); + entered.recv_timeout(Duration::from_secs(2)).unwrap(); + token.cancel(); + stopped + .recv_timeout(Duration::from_secs(2)) + .expect("cancel did not wake recv"); + actor.join().unwrap(); + assert_eq!( + link::lock_request(&request).reason(), + None, + "a plain stop request" + ); + }, ); } @@ -2647,51 +2821,56 @@ mod stop_wakeup_regression { #[test] fn every_stop_route_wakes_an_idle_thread() { - for route in 0..4 { - let (stopped_tx, stopped) = mpsc::channel(); - let actor = Idle(stopped_tx).start(); - // A reply ensures startup is complete before the cancellation race. - actor.request(Noop).unwrap(); - let failing = if route == 3 { - let peer = Failing.start(); - actor.context().link(&peer.child_handle()); - Some(peer) - } else { - None - }; - match route { - 0 => actor.context().stop(), - 1 => actor.child_handle().stop(), - 2 => actor.cancellation_token.cancel(), - _ => failing.as_ref().unwrap().send(Crash).unwrap(), - } - if route == 3 { - // An untrapped link notice ends the idle actor without - // stopped() (decision 2): the wakeup is observed as its exit. - let reason = actor.child_handle().wait_exit_blocking(); - assert!(matches!(reason, ExitReason::Panic(_)), "got {reason:?}"); - assert!(stopped.try_recv().is_err(), "stopped() ran"); - failing.unwrap().join(); - continue; - } - let result = stopped.recv_timeout(Duration::from_secs(2)); - // A failed test must not hang cleanup on a broken wakeup callback. - if result.is_err() { - actor.child_handle().stop(); - let _ = actor.send(Noop); - } - assert!( - result.is_ok(), - "stop route {route} failed to wake the mailbox" - ); - let reason = actor.wait_exit(); - if route == 3 { - assert!(matches!(reason, ExitReason::Panic(_))); - failing.unwrap().join(); - } else { - assert_eq!(reason, ExitReason::Normal); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::every_stop_route_wakes_an_idle_thread"), + move || { + for route in 0..4 { + let (stopped_tx, stopped) = mpsc::channel(); + let actor = Idle(stopped_tx).start(); + // A reply ensures startup is complete before the cancellation race. + actor.request(Noop).unwrap(); + let failing = if route == 3 { + let peer = Failing.start(); + actor.context().link(&peer.child_handle()); + Some(peer) + } else { + None + }; + match route { + 0 => actor.context().stop(), + 1 => actor.child_handle().stop(), + 2 => actor.cancellation_token.cancel(), + _ => failing.as_ref().unwrap().send(Crash).unwrap(), + } + if route == 3 { + // An untrapped link notice ends the idle actor without + // stopped() (decision 2): the wakeup is observed as its exit. + let reason = actor.child_handle().wait_exit_blocking(); + assert!(matches!(reason, ExitReason::Panic(_)), "got {reason:?}"); + assert!(stopped.try_recv().is_err(), "stopped() ran"); + failing.unwrap().join(); + continue; + } + let result = stopped.recv_timeout(Duration::from_secs(2)); + // A failed test must not hang cleanup on a broken wakeup callback. + if result.is_err() { + actor.child_handle().stop(); + let _ = actor.send(Noop); + } + assert!( + result.is_ok(), + "stop route {route} failed to wake the mailbox" + ); + let reason = actor.wait_exit(); + if route == 3 { + assert!(matches!(reason, ExitReason::Panic(_))); + failing.unwrap().join(); + } else { + assert_eq!(reason, ExitReason::Normal); + } + } + }, + ); } struct Busy { @@ -2720,40 +2899,48 @@ mod stop_wakeup_regression { #[test] fn repeated_stop_during_a_handler_discards_queued_requests() { - let (entered_tx, entered) = mpsc::channel(); - let (release, release_rx) = mpsc::channel(); - let handled = Arc::new(AtomicUsize::new(0)); - let actor = Busy { - entered: entered_tx, - release: release_rx, - handled: handled.clone(), - } - .start(); - let first = actor.send_request(Work, DEFAULT_REQUEST_TIMEOUT).unwrap(); - entered.recv_timeout(Duration::from_secs(2)).unwrap(); - let replies: Vec<_> = (0..256) - .map(|_| actor.send_request(Work, DEFAULT_REQUEST_TIMEOUT).unwrap()) - .collect(); - std::thread::scope(|scope| { - for _ in 0..16 { - let child = actor.child_handle(); - scope.spawn(move || { - for _ in 0..256 { - child.stop(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::repeated_stop_during_a_handler_discards_queued_requests" + ), + move || { + let (entered_tx, entered) = mpsc::channel(); + let (release, release_rx) = mpsc::channel(); + let handled = Arc::new(AtomicUsize::new(0)); + let actor = Busy { + entered: entered_tx, + release: release_rx, + handled: handled.clone(), + } + .start(); + let first = actor.send_request(Work, DEFAULT_REQUEST_TIMEOUT).unwrap(); + entered.recv_timeout(Duration::from_secs(2)).unwrap(); + let replies: Vec<_> = (0..256) + .map(|_| actor.send_request(Work, DEFAULT_REQUEST_TIMEOUT).unwrap()) + .collect(); + std::thread::scope(|scope| { + for _ in 0..16 { + let child = actor.child_handle(); + scope.spawn(move || { + for _ in 0..256 { + child.stop(); + } + }); } }); - } - }); - release.send(()).unwrap(); - first.recv().unwrap(); - for reply in replies { - assert!(matches!( - reply.recv(), - Err(ActorError::ActorStopped(ExitReason::Normal)) - )); - } - assert_eq!(actor.wait_exit(), ExitReason::Normal); - assert_eq!(handled.load(Ordering::SeqCst), 1); + release.send(()).unwrap(); + first.recv().unwrap(); + for reply in replies { + assert!(matches!( + reply.recv(), + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); + } + assert_eq!(actor.wait_exit(), ExitReason::Normal); + assert_eq!(handled.load(Ordering::SeqCst), 1); + }, + ); } } @@ -2835,66 +3022,98 @@ mod api_coverage { #[test] fn context_recipient_and_identity_reach_their_actor() { - let actor = Echo.start(); - let (recipient, id) = actor.request(OwnRecipient).unwrap(); - assert_eq!(id, actor.id()); - assert!(matches!(request(&*recipient, Ask(7), WAIT), Ok(7))); - recipient.send(Ask(1)).unwrap(); - actor.child_handle().stop(); - actor.join(); - // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). - assert!(matches!(recipient.send(Ask(1)), Err(ActorError::NoProc))); + crate::test_wait::bounded( + concat!( + module_path!(), + "::context_recipient_and_identity_reach_their_actor" + ), + move || { + let actor = Echo.start(); + let (recipient, id) = actor.request(OwnRecipient).unwrap(); + assert_eq!(id, actor.id()); + assert!(matches!(request(&*recipient, Ask(7), WAIT), Ok(7))); + recipient.send(Ask(1)).unwrap(); + actor.child_handle().stop(); + actor.join(); + // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). + assert!(matches!(recipient.send(Ask(1)), Err(ActorError::NoProc))); + }, + ); } #[test] fn requests_to_a_stopped_actor_fail_without_waiting() { - let actor = Echo.start(); - let recipient = actor.recipient::(); - actor.child_handle().stop(); - actor.join(); - // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). - assert!(matches!( - actor.request_with_timeout(Ask(1), WAIT), - Err(ActorError::NoProc) - )); - assert!(matches!( - request(&*recipient, Ask(1), WAIT), - Err(ActorError::NoProc) - )); + crate::test_wait::bounded( + concat!( + module_path!(), + "::requests_to_a_stopped_actor_fail_without_waiting" + ), + move || { + let actor = Echo.start(); + let recipient = actor.recipient::(); + actor.child_handle().stop(); + actor.join(); + // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). + assert!(matches!( + actor.request_with_timeout(Ask(1), WAIT), + Err(ActorError::NoProc) + )); + assert!(matches!( + request(&*recipient, Ask(1), WAIT), + Err(ActorError::NoProc) + )); + }, + ); } #[test] fn admitted_requests_fail_when_the_handler_unwinds_before_replying() { - let actor = Echo.start(); - assert!(matches!( - actor.request_with_timeout(Boom, WAIT), - Err(ActorError::ActorStopped(ExitReason::Panic(_))) - )); - let other = Echo.start(); - let recipient = other.recipient::(); - assert!(matches!( - request(&*recipient, Boom, WAIT), - Err(ActorError::ActorStopped(ExitReason::Panic(_))) - )); + crate::test_wait::bounded( + concat!( + module_path!(), + "::admitted_requests_fail_when_the_handler_unwinds_before_replying" + ), + move || { + let actor = Echo.start(); + assert!(matches!( + actor.request_with_timeout(Boom, WAIT), + Err(ActorError::ActorStopped(ExitReason::Panic(_))) + )); + let other = Echo.start(); + let recipient = other.recipient::(); + assert!(matches!( + request(&*recipient, Boom, WAIT), + Err(ActorError::ActorStopped(ExitReason::Panic(_))) + )); + }, + ); } #[test] fn requests_time_out_while_the_actor_is_busy() { - let actor = Echo.start(); - let recipient = actor.recipient::(); - let (release, held) = std_mpsc::channel(); - actor.send(StopAfter(held)).unwrap(); - let short = Duration::from_millis(1); - assert!(matches!( - actor.request_with_timeout(Ask(1), short), - Err(ActorError::RequestTimeout) - )); - assert!(matches!( - request(&*recipient, Ask(1), short), - Err(ActorError::RequestTimeout) - )); - release.send(()).unwrap(); - actor.join(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::requests_time_out_while_the_actor_is_busy" + ), + move || { + let actor = Echo.start(); + let recipient = actor.recipient::(); + let (release, held) = std_mpsc::channel(); + actor.send(StopAfter(held)).unwrap(); + let short = Duration::from_millis(1); + assert!(matches!( + actor.request_with_timeout(Ask(1), short), + Err(ActorError::RequestTimeout) + )); + assert!(matches!( + request(&*recipient, Ask(1), short), + Err(ActorError::RequestTimeout) + )); + release.send(()).unwrap(); + actor.join(); + }, + ); } struct Trapper { @@ -2929,87 +3148,113 @@ mod api_coverage { #[test] fn default_exit_received_keeps_a_trapping_actor_alive() { - let trapper = Trapper { trap: true }.start(); - assert!(matches!(trapper.request(Ask(0)), Ok(0))); - let peer = Echo.start(); - trapper.context().link(&peer.child_handle()); - peer.send(Boom).unwrap(); - assert!(matches!(peer.wait_exit(), ExitReason::Panic(_))); - // The Exit is handled by the default `exit_received`, which ignores it. - assert!(matches!(trapper.request(Ask(3)), Ok(3))); - assert!(trapper.exit_reason().is_none()); - trapper - .child_handle() - .stop_with_reason(crate::ExitReason::Shutdown); - assert_eq!(trapper.wait_exit(), ExitReason::Shutdown); + crate::test_wait::bounded( + concat!( + module_path!(), + "::default_exit_received_keeps_a_trapping_actor_alive" + ), + move || { + let trapper = Trapper { trap: true }.start(); + assert!(matches!(trapper.request(Ask(0)), Ok(0))); + let peer = Echo.start(); + trapper.context().link(&peer.child_handle()); + peer.send(Boom).unwrap(); + assert!(matches!(peer.wait_exit(), ExitReason::Panic(_))); + // The Exit is handled by the default `exit_received`, which ignores it. + assert!(matches!(trapper.request(Ask(3)), Ok(3))); + assert!(trapper.exit_reason().is_none()); + trapper + .child_handle() + .stop_with_reason(crate::ExitReason::Shutdown); + assert_eq!(trapper.wait_exit(), ExitReason::Shutdown); + }, + ); } #[test] fn unlink_prevents_signal_delivery() { - // Not trapping: a linked peer's panic would take this actor down. - let trapper = Trapper { trap: false }.start(); - let peer = Echo.start(); - let peer_handle = peer.child_handle(); - trapper.context().link(&peer_handle); - trapper.request(Unlink(peer_handle.clone())).unwrap(); - // Unlinking yourself or an already-unlinked peer is a no-op. - trapper.request(Unlink(trapper.child_handle())).unwrap(); - trapper.request(Unlink(peer_handle)).unwrap(); - peer.send(Boom).unwrap(); - assert!(matches!(peer.wait_exit(), ExitReason::Panic(_))); - assert!(matches!(trapper.request(Ask(1)), Ok(1))); - assert!(trapper.exit_reason().is_none()); - trapper - .child_handle() - .stop_with_reason(crate::ExitReason::Shutdown); - trapper.join(); + crate::test_wait::bounded( + concat!(module_path!(), "::unlink_prevents_signal_delivery"), + move || { + // Not trapping: a linked peer's panic would take this actor down. + let trapper = Trapper { trap: false }.start(); + let peer = Echo.start(); + let peer_handle = peer.child_handle(); + trapper.context().link(&peer_handle); + trapper.request(Unlink(peer_handle.clone())).unwrap(); + // Unlinking yourself or an already-unlinked peer is a no-op. + trapper.request(Unlink(trapper.child_handle())).unwrap(); + trapper.request(Unlink(peer_handle)).unwrap(); + peer.send(Boom).unwrap(); + assert!(matches!(peer.wait_exit(), ExitReason::Panic(_))); + assert!(matches!(trapper.request(Ask(1)), Ok(1))); + assert!(trapper.exit_reason().is_none()); + trapper + .child_handle() + .stop_with_reason(crate::ExitReason::Shutdown); + trapper.join(); + }, + ); } #[test] fn opposite_links_between_the_same_pair_do_not_deadlock() { - // Each link/unlink locks both link tables. Both directions must take - // the locks in the same global order, or these two loops deadlock. - let a = Trapper { trap: true }.start(); - let b = Trapper { trap: true }.start(); - let (done, finished) = std_mpsc::channel(); - let workers: Vec<_> = [ - (a.context(), b.child_handle()), - (b.context(), a.child_handle()), - ] - .into_iter() - .map(|(own, peer)| { - let done = done.clone(); - std::thread::spawn(move || { - for _ in 0..2_000 { - own.link(&peer); - own.unlink(&peer); + crate::test_wait::bounded( + concat!( + module_path!(), + "::opposite_links_between_the_same_pair_do_not_deadlock" + ), + move || { + // Each link/unlink locks both link tables. Both directions must take + // the locks in the same global order, or these two loops deadlock. + let a = Trapper { trap: true }.start(); + let b = Trapper { trap: true }.start(); + let (done, finished) = std_mpsc::channel(); + let workers: Vec<_> = [ + (a.context(), b.child_handle()), + (b.context(), a.child_handle()), + ] + .into_iter() + .map(|(own, peer)| { + let done = done.clone(); + std::thread::spawn(move || { + for _ in 0..2_000 { + own.link(&peer); + own.unlink(&peer); + } + done.send(()).unwrap(); + }) + }) + .collect(); + for _ in 0..2 { + finished + .recv_timeout(WAIT) + .expect("opposite link directions deadlocked"); } - done.send(()).unwrap(); - }) - }) - .collect(); - for _ in 0..2 { - finished - .recv_timeout(WAIT) - .expect("opposite link directions deadlocked"); - } - for worker in workers { - worker.join().unwrap(); - } - for actor in [a, b] { - actor - .child_handle() - .stop_with_reason(crate::ExitReason::Shutdown); - actor.join(); - } + for worker in workers { + worker.join().unwrap(); + } + for actor in [a, b] { + actor + .child_handle() + .stop_with_reason(crate::ExitReason::Shutdown); + actor.join(); + } + }, + ); } #[test] fn debug_output_names_the_handle() { - let actor = Echo.start(); - assert!(format!("{actor:?}").starts_with("ActorRef")); - assert!(format!("{:?}", actor.context()).starts_with("Context")); - actor.child_handle().stop(); - actor.join(); + crate::test_wait::bounded( + concat!(module_path!(), "::debug_output_names_the_handle"), + move || { + let actor = Echo.start(); + assert!(format!("{actor:?}").starts_with("ActorRef")); + assert!(format!("{:?}", actor.context()).starts_with("Context")); + actor.child_handle().stop(); + actor.join(); + }, + ); } } diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs index cc6da207..7c34d47b 100644 --- a/concurrency/src/threads/jobs.rs +++ b/concurrency/src/threads/jobs.rs @@ -678,39 +678,52 @@ mod tests { /// return only when that thread woke up and exited. #[test] fn a_dropped_unused_permit_returns_its_native_slot_at_once() { - struct One; - impl Actor for One { - const MAX_NATIVE_JOBS: Option = Some(1); - } - let owner = One.start(); - let ctx = owner.context(); - for i in 0..2000 { - let permit = ctx - .reserve_blocking_job(JobOptions::new()) - .unwrap_or_else(|e| panic!("reservation {i} refused: {e}")); - drop(permit); - } - ctx.stop(); - owner.join(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::a_dropped_unused_permit_returns_its_native_slot_at_once" + ), + move || { + struct One; + impl Actor for One { + const MAX_NATIVE_JOBS: Option = Some(1); + } + let owner = One.start(); + let ctx = owner.context(); + for i in 0..2000 { + let permit = ctx + .reserve_blocking_job(JobOptions::new()) + .unwrap_or_else(|e| panic!("reservation {i} refused: {e}")); + drop(permit); + } + ctx.stop(); + owner.join(); + }, + ); } /// Review P1 (Codex 13), threads mode: see the tasks-mode test. #[test] fn a_refused_emit_returns_its_credit_once() { - let owner = Owner.start(); - for credits in [1, u16::MAX] { - let state = JobState::new(owner.id(), credits); - let job = JobCtx { - state: state.clone(), - ctx: owner.context(), - }; - state.acquire_blocking().unwrap(); - state.cancel(); - assert_eq!(job.enqueue(Note), Err(JobGone)); - assert_eq!(state.emit_credits(), credits, "credit returned twice"); - } - owner.context().stop(); - owner.join(); + crate::test_wait::bounded( + concat!(module_path!(), "::a_refused_emit_returns_its_credit_once"), + move || { + let owner = Owner.start(); + for credits in [1, u16::MAX] { + let state = JobState::new(owner.id(), credits); + let job = JobCtx { + state: state.clone(), + ctx: owner.context(), + }; + state.acquire_blocking().unwrap(); + state.cancel(); + assert_eq!(job.enqueue(Note), Err(JobGone)); + assert_eq!(state.emit_credits(), credits, "credit returned twice"); + } + owner.context().stop(); + owner.join(); + }, + ); } /// `cancel_on_caller_gone` applies to reply jobs only: a plain job has no @@ -718,16 +731,24 @@ mod tests { #[cfg(debug_assertions)] #[test] fn cancel_on_caller_gone_on_a_plain_job_is_a_bug() { - let owner = Owner.start(); - let ctx = owner.context(); - let refused = std::panic::catch_unwind(AssertUnwindSafe(|| { - ctx.reserve_blocking_job(JobOptions::new().cancel_on_caller_gone()) - .map(drop) - })); - assert!(refused.is_err(), "the option was accepted on a plain job"); - let plain = ctx.reserve_blocking_job(JobOptions::new()).map(drop); - assert!(plain.is_ok()); - ctx.stop(); - owner.join(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::cancel_on_caller_gone_on_a_plain_job_is_a_bug" + ), + move || { + let owner = Owner.start(); + let ctx = owner.context(); + let refused = std::panic::catch_unwind(AssertUnwindSafe(|| { + ctx.reserve_blocking_job(JobOptions::new().cancel_on_caller_gone()) + .map(drop) + })); + assert!(refused.is_err(), "the option was accepted on a plain job"); + let plain = ctx.reserve_blocking_job(JobOptions::new()).map(drop); + assert!(plain.is_ok()); + ctx.stop(); + owner.join(); + }, + ); } } diff --git a/concurrency/src/threads/stream_tests.rs b/concurrency/src/threads/stream_tests.rs index c6f4ab1c..e1600493 100644 --- a/concurrency/src/threads/stream_tests.rs +++ b/concurrency/src/threads/stream_tests.rs @@ -53,85 +53,100 @@ impl Handler for Collector { #[test] fn listener_consumes_iterator() { - let done = Arc::new(AtomicU64::new(0)); - let actor = Collector { - items: vec![], - done: done.clone(), - } - .start(); - - let ctx = actor.context(); - let items = vec![ - Push { value: 1 }, - Push { value: 2 }, - Push { value: 3 }, - Push { value: 4 }, - Push { value: 5 }, - ]; - let handle = spawn_listener(ctx, items); - handle.join().unwrap(); - // Give the actor time to process all messages - while done.load(Ordering::SeqCst) < 5 { - rt::sleep(Duration::from_millis(10)); - } - - let result = actor.request(GetItems).unwrap(); - assert_eq!(result, vec![1, 2, 3, 4, 5]); + crate::test_wait::bounded( + concat!(module_path!(), "::listener_consumes_iterator"), + move || { + let done = Arc::new(AtomicU64::new(0)); + let actor = Collector { + items: vec![], + done: done.clone(), + } + .start(); + + let ctx = actor.context(); + let items = vec![ + Push { value: 1 }, + Push { value: 2 }, + Push { value: 3 }, + Push { value: 4 }, + Push { value: 5 }, + ]; + let handle = spawn_listener(ctx, items); + handle.join().unwrap(); + // Give the actor time to process all messages + while done.load(Ordering::SeqCst) < 5 { + rt::sleep(Duration::from_millis(10)); + } + + let result = actor.request(GetItems).unwrap(); + assert_eq!(result, vec![1, 2, 3, 4, 5]); + }, + ); } #[test] fn listener_stops_on_cancellation() { - let done = Arc::new(AtomicU64::new(0)); - let actor = Collector { - items: vec![], - done: done.clone(), - } - .start(); - - let ctx = actor.context(); - - // An iterator that blocks between items, giving us time to cancel - let iter = (1..=100).map(move |i| { - if i > 1 { - rt::sleep(Duration::from_millis(50)); - } - Push { value: i } - }); - - let _handle = spawn_listener(ctx, iter); - - // Let at least one item through, then stop the actor - let first = wait_for(|| done.load(Ordering::SeqCst), |n| *n > 0); - assert!(first > 0, "listener never delivered an item"); - actor.context().stop(); - actor.join(); - - let processed = done.load(Ordering::SeqCst); - assert!( - processed < 100, - "Expected partial processing, but all 100 items were processed" - ); - assert!( - processed > 0, - "Expected at least some items to be processed" + crate::test_wait::bounded( + concat!(module_path!(), "::listener_stops_on_cancellation"), + move || { + let done = Arc::new(AtomicU64::new(0)); + let actor = Collector { + items: vec![], + done: done.clone(), + } + .start(); + + let ctx = actor.context(); + + // An iterator that blocks between items, giving us time to cancel + let iter = (1..=100).map(move |i| { + if i > 1 { + rt::sleep(Duration::from_millis(50)); + } + Push { value: i } + }); + + let _handle = spawn_listener(ctx, iter); + + // Let at least one item through, then stop the actor + let first = wait_for(|| done.load(Ordering::SeqCst), |n| *n > 0); + assert!(first > 0, "listener never delivered an item"); + actor.context().stop(); + actor.join(); + + let processed = done.load(Ordering::SeqCst); + assert!( + processed < 100, + "Expected partial processing, but all 100 items were processed" + ); + assert!( + processed > 0, + "Expected at least some items to be processed" + ); + }, ); } #[test] fn stopped_actor_does_not_poll_iterator() { - let polls = Arc::new(AtomicU64::new(0)); - let actor = Collector { - items: vec![], - done: Default::default(), - } - .start(); - actor.child_handle().stop(); - actor.join(); - let count = polls.clone(); - let iter = std::iter::from_fn(move || { - count.fetch_add(1, Ordering::SeqCst); - None:: - }); - spawn_listener(actor.context(), iter).join().unwrap(); - assert_eq!(polls.load(Ordering::SeqCst), 0); + crate::test_wait::bounded( + concat!(module_path!(), "::stopped_actor_does_not_poll_iterator"), + move || { + let polls = Arc::new(AtomicU64::new(0)); + let actor = Collector { + items: vec![], + done: Default::default(), + } + .start(); + actor.child_handle().stop(); + actor.join(); + let count = polls.clone(); + let iter = std::iter::from_fn(move || { + count.fetch_add(1, Ordering::SeqCst); + None:: + }); + spawn_listener(actor.context(), iter).join().unwrap(); + assert_eq!(polls.load(Ordering::SeqCst), 0); + }, + ); } diff --git a/concurrency/src/threads/timer_tests.rs b/concurrency/src/threads/timer_tests.rs index ddca532a..62d4f45e 100644 --- a/concurrency/src/threads/timer_tests.rs +++ b/concurrency/src/threads/timer_tests.rs @@ -84,26 +84,31 @@ impl Handler for Repeater { #[test] pub fn test_send_interval_and_cancellation() { - let repeater = Repeater::new(0).start(); - - // The interval keeps firing. - let count = wait_for( - || repeater.request(GetRepCount), - |count| matches!(count, Ok(n) if *n >= 3), - ) - .unwrap(); - assert!(count >= 3, "interval stopped firing at {count}"); - - // StopTimer is handled before the next request. The timer thread checks - // the token and then sends, so one tick that raced the cancellation may - // still land, but no more than one. - repeater.send(StopTimer).unwrap(); - let stopped_at = repeater.request(GetRepCount).unwrap(); - rt::sleep(Duration::from_millis(500)); - let after = repeater.request(GetRepCount).unwrap(); - assert!( - after <= stopped_at + 1, - "interval kept firing after cancellation: {stopped_at} -> {after}" + crate::test_wait::bounded( + concat!(module_path!(), "::test_send_interval_and_cancellation"), + move || { + let repeater = Repeater::new(0).start(); + + // The interval keeps firing. + let count = wait_for( + || repeater.request(GetRepCount), + |count| matches!(count, Ok(n) if *n >= 3), + ) + .unwrap(); + assert!(count >= 3, "interval stopped firing at {count}"); + + // StopTimer is handled before the next request. The timer thread checks + // the token and then sends, so one tick that raced the cancellation may + // still land, but no more than one. + repeater.send(StopTimer).unwrap(); + let stopped_at = repeater.request(GetRepCount).unwrap(); + rt::sleep(Duration::from_millis(500)); + let after = repeater.request(GetRepCount).unwrap(); + assert!( + after <= stopped_at + 1, + "interval kept firing after cancellation: {stopped_at} -> {after}" + ); + }, ); } @@ -175,51 +180,61 @@ impl Handler for Delayed { #[test] pub fn test_send_after_and_cancellation() { - let actor = Delayed::new(0).start(); + crate::test_wait::bounded( + concat!(module_path!(), "::test_send_after_and_cancellation"), + move || { + let actor = Delayed::new(0).start(); - let ctx = actor.context(); - let _ = send_after(Duration::from_millis(100), ctx, Inc); + let ctx = actor.context(); + let _ = send_after(Duration::from_millis(100), ctx, Inc); - let count = wait_for( - || actor.request(GetDelCount), - |count| matches!(count, Ok(n) if *n >= 1), - ) - .unwrap(); - assert_eq!(1, count); + let count = wait_for( + || actor.request(GetDelCount), + |count| matches!(count, Ok(n) if *n >= 1), + ) + .unwrap(); + assert_eq!(1, count); - let ctx = actor.context(); - let timer = send_after(Duration::from_millis(100), ctx, Inc); + let ctx = actor.context(); + let timer = send_after(Duration::from_millis(100), ctx, Inc); - timer.cancellation_token.cancel(); + timer.cancellation_token.cancel(); - rt::sleep(Duration::from_millis(200)); + rt::sleep(Duration::from_millis(200)); - let count2 = actor.request(GetDelCount).unwrap(); - assert_eq!(1, count2); + let count2 = actor.request(GetDelCount).unwrap(); + assert_eq!(1, count2); + }, + ); } #[test] pub fn test_send_after_actor_shutdown() { - let actor = Delayed::new(0).start(); + crate::test_wait::bounded( + concat!(module_path!(), "::test_send_after_actor_shutdown"), + move || { + let actor = Delayed::new(0).start(); - let ctx = actor.context(); - let _ = send_after(Duration::from_millis(100), ctx, Inc); + let ctx = actor.context(); + let _ = send_after(Duration::from_millis(100), ctx, Inc); - let count = wait_for( - || actor.request(GetDelCount), - |count| matches!(count, Ok(n) if *n >= 1), - ) - .unwrap(); - assert_eq!(1, count); + let count = wait_for( + || actor.request(GetDelCount), + |count| matches!(count, Ok(n) if *n >= 1), + ) + .unwrap(); + assert_eq!(1, count); - let ctx = actor.context(); - let _ = send_after(Duration::from_millis(100), ctx, Inc); + let ctx = actor.context(); + let _ = send_after(Duration::from_millis(100), ctx, Inc); - let count2 = actor.request(StopDelayed).unwrap(); + let count2 = actor.request(StopDelayed).unwrap(); - rt::sleep(Duration::from_millis(200)); + rt::sleep(Duration::from_millis(200)); - assert_eq!(1, count2); + assert_eq!(1, count2); + }, + ); } // --- T4: threads timers use the monotonic clock (`recv_timeout`) --- @@ -274,45 +289,53 @@ impl Handler for Clocked { /// fires early, fires once, and a long delay cancelled halfway never fires. #[test] pub fn threads_send_after_uses_the_monotonic_clock_over_long_delays() { - let handled = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); - let actor = Clocked { - handled: handled.clone(), - stall: None, - } - .start(); - let delay = Duration::from_millis(300); - let long = Duration::from_millis(1_500); - let armed = std::time::Instant::now(); - let _short = send_after(delay, actor.context(), Fired); - let cancelled = send_after(long, actor.context(), Fired); - assert!( - wait_for(|| handled.lock().unwrap().len(), |n| *n >= 1) >= 1, - "timer never fired" - ); - let fired = handled.lock().unwrap()[0]; - assert!( - fired.duration_since(armed) >= delay, - "T4: send_after fired after {:?}, before its {delay:?}", - fired.duration_since(armed) - ); - cancelled.cancellation_token.cancel(); - let _uncancelled = send_after(long, actor.context(), Fired); - let armed_long = std::time::Instant::now(); - assert!( - wait_for(|| handled.lock().unwrap().len(), |n| *n >= 2) >= 2, - "timer never fired" - ); - let fired_long = handled.lock().unwrap()[1]; - assert!( - fired_long.duration_since(armed_long) >= long, - "T4: long send_after fired after {:?}", - fired_long.duration_since(armed_long) + crate::test_wait::bounded( + concat!( + module_path!(), + "::threads_send_after_uses_the_monotonic_clock_over_long_delays" + ), + move || { + let handled = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let actor = Clocked { + handled: handled.clone(), + stall: None, + } + .start(); + let delay = Duration::from_millis(300); + let long = Duration::from_millis(1_500); + let armed = std::time::Instant::now(); + let _short = send_after(delay, actor.context(), Fired); + let cancelled = send_after(long, actor.context(), Fired); + assert!( + wait_for(|| handled.lock().unwrap().len(), |n| *n >= 1) >= 1, + "timer never fired" + ); + let fired = handled.lock().unwrap()[0]; + assert!( + fired.duration_since(armed) >= delay, + "T4: send_after fired after {:?}, before its {delay:?}", + fired.duration_since(armed) + ); + cancelled.cancellation_token.cancel(); + let _uncancelled = send_after(long, actor.context(), Fired); + let armed_long = std::time::Instant::now(); + assert!( + wait_for(|| handled.lock().unwrap().len(), |n| *n >= 2) >= 2, + "timer never fired" + ); + let fired_long = handled.lock().unwrap()[1]; + assert!( + fired_long.duration_since(armed_long) >= long, + "T4: long send_after fired after {:?}", + fired_long.duration_since(armed_long) + ); + // The cancelled 1.5 s timer was due before the uncancelled one fired. + assert!(armed + long <= fired_long); + assert_eq!(handled.lock().unwrap().len(), 2, "T1: fired more than once"); + actor.child_handle().stop(); + actor.join(); + }, ); - // The cancelled 1.5 s timer was due before the uncancelled one fired. - assert!(armed + long <= fired_long); - assert_eq!(handled.lock().unwrap().len(), 2, "T1: fired more than once"); - actor.child_handle().stop(); - actor.join(); } /// T2/T4: threads `send_interval` is fixed delay on the monotonic clock: @@ -320,29 +343,37 @@ pub fn threads_send_after_uses_the_monotonic_clock_over_long_delays() { /// stalled for 3.5 periods, so there is no burst to catch up. #[test] pub fn threads_send_interval_is_fixed_delay_while_the_actor_stalls() { - let period = Duration::from_millis(50); - let sends = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); - let handled = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); - let actor = Clocked { - handled: handled.clone(), - stall: Some(period * 7 / 2), - } - .start(); - let timer = send_interval(period, actor.context(), Stamp(sends.clone())); - assert!( - wait_for(|| handled.lock().unwrap().len(), |n| *n >= 10) >= 10, - "timer never fired" + crate::test_wait::bounded( + concat!( + module_path!(), + "::threads_send_interval_is_fixed_delay_while_the_actor_stalls" + ), + move || { + let period = Duration::from_millis(50); + let sends = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let handled = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let actor = Clocked { + handled: handled.clone(), + stall: Some(period * 7 / 2), + } + .start(); + let timer = send_interval(period, actor.context(), Stamp(sends.clone())); + assert!( + wait_for(|| handled.lock().unwrap().len(), |n| *n >= 10) >= 10, + "timer never fired" + ); + timer.cancellation_token.cancel(); + let sends = sends.lock().unwrap().clone(); + assert!(sends.len() >= 10, "{}", sends.len()); + for pair in sends.windows(2) { + assert!( + pair[1].duration_since(pair[0]) >= period, + "T2/T4: two sends {:?} apart, period {period:?}", + pair[1].duration_since(pair[0]) + ); + } + actor.child_handle().stop(); + actor.join(); + }, ); - timer.cancellation_token.cancel(); - let sends = sends.lock().unwrap().clone(); - assert!(sends.len() >= 10, "{}", sends.len()); - for pair in sends.windows(2) { - assert!( - pair[1].duration_since(pair[0]) >= period, - "T2/T4: two sends {:?} apart, period {period:?}", - pair[1].duration_since(pair[0]) - ); - } - actor.child_handle().stop(); - actor.join(); } diff --git a/docs/TESTING.md b/docs/TESTING.md index af66f477..46b78886 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -1767,8 +1767,13 @@ time on the simulator, and only reached by a bug on Tokio, since every such wait is a handshake, never a fixed sleep. Tests on paused Tokio time (`protocol_timeouts`, `semantics_gaps`) are left alone: paused time auto-advances while work runs outside Tokio, which could fire the bound early. -Unit tests inside `concurrency/src` do not use it yet (see "Planted-bug -sensitivity" for the ones that still hang under a planted bug). +Unit tests inside `concurrency/src` that start, call, join or wait on actors +are bounded the same way (`concurrency/src/test_wait.rs`, 120 s): +synchronous tests run their whole body, Tokio runtime and its teardown +included, through `test_wait::bounded`, on a thread named after the test, and +fail with that name if it does not finish; `#[tokio::test]` bodies go +through `test_wait::within`. A panic in the body (an assertion, or a +`#[should_panic]` test's expected panic) is re-raised unchanged. ### Single-CPU runs From 3c2d68d64e61cd6ee4331e7e8c55b78f7efc7843 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 14:35:26 -0300 Subject: [PATCH 189/208] perf(tasks): one cancellation future per run loop, polled only on an empty mailbox Release gate (burst_async -37% vs main): the tasks run loop built a fresh CancellationToken::cancelled() future for every message and select()ed it against the mailbox receive, registering and dropping a Notify waiter per message. The loop now creates it once and polls it only when the receive is pending. The receive still checks is_cancelled() before taking a message, so a stop is observed exactly where it was (the same lifecycle-trace step), and a waiting loop is still woken by the cancel. Measured on fede (ABBA, cores 12-15, heavy shared load): burst of 512 outstanding raw requests +6.9% (4.53M -> 4.84M/s); sequential requests unchanged. Crate tests, clippy and the sim suite (315) pass. --- concurrency/src/tasks/actor.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 3f7de171..b9e41430 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -1981,6 +1981,9 @@ async fn run_actor_lifecycle( } } + // One cancellation future for the whole loop: polled only when the + // mailbox has nothing, so a busy actor registers no waiter per message. + let mut cancelled = pin!(cancellation_token.cancelled()); while !cancellation_token.is_cancelled() { let msg = { let recv = pin!(future::poll_fn(|cx| { @@ -2013,11 +2016,15 @@ async fn run_actor_lifecycle( } result })); - let cancel = pin!(cancellation_token.cancelled()); - match future::select(recv, cancel).await { - future::Either::Left((msg, _)) => msg, - future::Either::Right(_) => None, - } + let mut recv = recv; + future::poll_fn(|cx| match recv.as_mut().poll(cx) { + std::task::Poll::Pending => match cancelled.as_mut().poll(cx) { + std::task::Poll::Ready(()) => std::task::Poll::Ready(None), + std::task::Poll::Pending => std::task::Poll::Pending, + }, + ready => ready, + }) + .await }; match msg { Some(envelope) => { From 4ebf1f00a086db2c2cfd0cbff583f0faa7d280bf Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 14:35:34 -0300 Subject: [PATCH 190/208] perf(reply): inline the slot word helpers (state, jobs, live, credits, with_state) --- concurrency/src/reply/slot.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 969212e0..4c97c834 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -48,20 +48,25 @@ const JOBS_MASK: u64 = MAX_JOBS << JOBS_SHIFT; /// Deadline value meaning "never" (`Deadline::Never`). pub(crate) const NEVER: u64 = u64::MAX; +#[inline] pub(crate) fn state(word: u64) -> u64 { word & STATE_MASK } +#[inline] pub(crate) fn jobs(word: u64) -> u64 { (word & JOBS_MASK) >> JOBS_SHIFT } +#[inline] fn live(state: u64) -> bool { state == OPEN || state == WRITING } /// Credits a slot holds in its owner's pool. +#[inline] pub(crate) fn credits(word: u64) -> u64 { let token = (word & TOKEN != 0 && live(state(word))) as u64; jobs(word).max(token) } +#[inline] fn with_state(word: u64, to: u64) -> u64 { (word & !STATE_MASK) | to } From b535c158d3e8330e741ac880f446e2ab4fa948f7 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 14:39:52 -0300 Subject: [PATCH 191/208] perf(reply): an in-turn reply publishes with one CAS, the value written under the slot lock PendingReply::reply (a reply from the owner's own turn) went through the general deferred-reply protocol: CAS OPEN -> WRITING, lock, read the deadline, write, CAS WRITING -> REPLIED. In its own turn nothing can shorten the deadline (only a deferred token can) and the owner's death point cannot run, so the only concurrent transitions are the caller's (abandonment, its timer's expiry), each a single CAS out of OPEN. Slot::reply_in_turn writes the value under the slot lock and publishes it with one CAS OPEN -> REPLIED (the same linearization point and trace event, "ok"); a lost CAS hands the value back (CallerGone, Expired). A reply at or past the deadline keeps the WRITING protocol. Deferred tokens (ReplyTo) are unchanged. Loom: reply_in_turn_vs_abandon, reply_in_turn_vs_expiry, polled_receiver_vs_reply_in_turn. Crate tests, sim (315), reply/job/ lifecycle traces (48/48 accepted) pass. --- concurrency/src/reply/mod.rs | 2 +- concurrency/src/reply/slot.rs | 43 +++++++++++++++++++ concurrency/tests/reply_slot_loom.rs | 62 ++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 1 deletion(-) diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 5bd49d24..a7759c48 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -333,7 +333,7 @@ impl<'t, R: Send + 'static> PendingReply<'t, R> { Delivery::CallerGone } Some(slot) => { - let (delivery, unsent) = slot.reply(value, &now_ticks); + let (delivery, unsent) = slot.reply_in_turn(value, &now_ticks); drop(unsent); record(delivery); delivery diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 4c97c834..548025b0 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -602,6 +602,49 @@ impl Slot { self.reply_admitted(value, now, || Some(())) } + /// A reply from the owner's own turn (`PendingReply::reply`): nothing + /// shortens the deadline concurrently (only a deferred token can) and the + /// owner's death point cannot run during its own turn, so the only + /// concurrent transitions are the caller's (abandonment, its timer's + /// expiry), each one CAS out of `OPEN`. With the deadline still ahead, + /// the value is written under the slot lock and published by one CAS + /// `OPEN -> REPLIED` (the same linearization point as `OPEN -> WRITING`: + /// a receiver reads the value under that lock); a lost CAS hands the + /// value back. A reply at or past the deadline takes [`reply`](Self::reply). + pub(crate) fn reply_in_turn(&self, value: R, now: &impl Fn() -> u64) -> (Delivery, Option) { + let deadline = self.deadline(); + if deadline != NEVER && now() >= deadline { + return self.reply(value, now); + } + let mut inner = lock(&self.inner); + inner.value = Some(value); + match self.cas(None, |w| { + if state(w) == OPEN { + Ok(with_state(w, REPLIED)) + } else { + Err(state(w)) + } + }) { + Ok((old, new)) => { + let wake = + Self::collect(&mut inner, credits(old).saturating_sub(credits(new)), true); + drop(inner); + wake.run(&self.parked); + (Delivery::Delivered, None) + } + Err(lost) => { + let value = inner.value.take(); + drop(inner); + let delivery = match lost { + Some(TIMED_OUT) => Delivery::Expired, + Some(STOPPED) => Delivery::OwnerExited, + _ => Delivery::CallerGone, + }; + (delivery, value) + } + } + } + /// [`reply`](Self::reply) admitted by the owner: `admit` returns a /// permit held across the linearizing `OPEN -> WRITING` CAS (and /// released before the value is written), or `None` once the owner is diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index 88d6de03..e3476951 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -180,6 +180,68 @@ fn reply_vs_abandon() { }); } +/// The in-turn reply (one CAS `OPEN -> REPLIED`, value written under the +/// slot lock) versus the caller abandoning: exactly one wins; a lost reply +/// gets its value back. +#[test] +fn reply_in_turn_vs_abandon() { + model(|| { + let slot: Arc> = Arc::new(Slot::new(NEVER)); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply_in_turn(1, &|| 0)) + }; + let abandoned = slot.abandon(); + let (delivery, back) = replier.join().unwrap(); + assert_ne!(abandoned, delivery == Delivery::Delivered); + if abandoned { + assert_eq!(state(slot.word()), slot::ABANDONED); + assert_eq!((delivery, back), (Delivery::CallerGone, Some(1))); + } else { + assert_eq!(state(slot.word()), slot::REPLIED); + assert_eq!(back, None); + } + }); +} + +/// The in-turn reply versus the caller's timer expiring the slot: one wins; +/// an expired slot hands the value back as `Expired`. +#[test] +fn reply_in_turn_vs_expiry() { + model(|| { + let slot: Arc> = Arc::new(Slot::new(10)); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply_in_turn(7, &|| 0)) + }; + let expired = slot.terminate(Terminal::TimedOut); + let (delivery, back) = replier.join().unwrap(); + assert_ne!(expired, delivery == Delivery::Delivered); + if expired { + assert_eq!(state(slot.word()), slot::TIMED_OUT); + assert_eq!((delivery, back), (Delivery::Expired, Some(7))); + } else { + assert_eq!(state(slot.word()), slot::REPLIED); + } + }); +} + +/// A polled receiver sees the in-turn reply's value (written before the +/// publishing CAS, under the lock the receiver takes). +#[test] +fn polled_receiver_vs_reply_in_turn() { + model(|| { + let slot: Arc> = Arc::new(Slot::new(NEVER)); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply_in_turn(4, &|| 0).0) + }; + let got = loom::future::block_on(std::future::poll_fn(|cx| slot.poll(cx.waker()))); + assert_eq!(got, Ok(4)); + assert_eq!(replier.join().unwrap(), Delivery::Delivered); + }); +} + /// Reply versus the owner's registry close at death. A close that finds /// `WRITING` waits for the reply to resolve. #[test] From 836f6a251df68212d1ee44c6576dc8f89561d6ce Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 14:47:47 -0300 Subject: [PATCH 192/208] perf(tasks): the run loop checks cancellation once per message, in the receive tokio-util's CancellationToken::is_cancelled locks the token's node. The loop head re-checked what the receive already checks before taking each message (it returns None once cancelled, which ends the loop), so a stop is observed at the same step with one lock per message instead of two. --- concurrency/src/tasks/actor.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index b9e41430..3446fd98 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -1984,7 +1984,10 @@ async fn run_actor_lifecycle( // One cancellation future for the whole loop: polled only when the // mailbox has nothing, so a busy actor registers no waiter per message. let mut cancelled = pin!(cancellation_token.cancelled()); - while !cancellation_token.is_cancelled() { + // The receive checks the token before taking each message (and returns + // `None` once it is cancelled), so the loop needs no check of its own: + // `is_cancelled` locks the token's node. + loop { let msg = { let recv = pin!(future::poll_fn(|cx| { // Stop and closed-mailbox both take the normal lifecycle exit, From befbd77a29751945a0b032abcd62fa9427eb79a9 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 14:47:47 -0300 Subject: [PATCH 193/208] perf(reply): a turn that dropped no obligation skips the commit Registry::end_turn built and consumed an empty Vec through commit() on every turn. The commit now runs only when the turn recorded drops (the same outcome classification). --- concurrency/src/reply/registry.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index 0248d4fc..27bd107d 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -363,7 +363,10 @@ impl Registry { /// The handler returned (`panicked == false`) or unwound. pub(crate) fn end_turn(&self, panicked: bool) { let (drops, outcome) = self.turn.end_turn(panicked); - self.commit(drops, outcome); + // Most turns drop no obligation: skip the commit entirely. + if !drops.is_empty() { + self.commit(drops, outcome); + } } /// Exit begins: later drops, and the current turn's recorded drops, are From 7f7161528eda2fce545490f1e3474ef5e6a5a2c3 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 14:47:47 -0300 Subject: [PATCH 194/208] bench(baseline): raw requests carry no deadline, as 0.7's request_raw The 0.8 port of the baseline driver gave every raw request a 10 s deadline (send_request(work, Deadline::after(10 s))), while 0.7's request_raw, which the release gate compares against, had none. A raw receiver with a deadline registers it in the owner's deadline heap on its first pending poll (the registry lock, and a deadline-service wake when it is the earliest), so the comparison charged 0.8 for work 0.7 did not do. Deadline::Never is request_raw's documented 0.8 equivalent (MIGRATION); the cost of a deadline is measured separately. --- concurrency/benches/baseline.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/concurrency/benches/baseline.rs b/concurrency/benches/baseline.rs index 22ff4d74..2ace42c1 100644 --- a/concurrency/benches/baseline.rs +++ b/concurrency/benches/baseline.rs @@ -131,16 +131,18 @@ enum Reply { Threads(ReplyReceiver), } impl Target { + /// A raw request with no deadline: the 0.8 form of 0.7's `request_raw` + /// (no timeout), so the comparison with 0.7 measures the same operation. fn request(&self, work: Work) -> Reply { match self { Self::Tasks(actor) => Reply::Tasks( actor - .send_request(work, Deadline::after(Duration::from_secs(10))) + .send_request(work, Deadline::Never) .expect("actor stopped"), ), Self::Threads(actor) => Reply::Threads( actor - .send_request(work, Deadline::after(Duration::from_secs(10))) + .send_request(work, Deadline::Never) .expect("actor stopped"), ), Self::Tokio(tx) => { From c73a0393f3d9bd6b9857e19eef298d0355354079 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 15:35:15 -0300 Subject: [PATCH 195/208] perf(reply): one slot lock for value and waker, lazily created deferral state, no condvar The reply slot keeps its packed atomic word as the single source of truth for every 0.8 guarantee (deferral credits, the death-point reply fence, deadlines, at-most-once, terminal kinds). What changed is what sits beside it: - `Inner` is the value, the caller's waker, a `parked` flag and a boxed `Side` created only by `defer()` (the owner's credit pool and the token's watchers). A plain request no longer carries a `Vec` of watchers and an `Option>`; dropping its slot is two `None` checks and a null check. - Threads-mode `recv()` registers a cached per-thread unparker as the slot's waker and parks (`park_timeout` with a deadline): the `Condvar` is gone. `parked` marks that waker, so the death point's discharge still wakes blocked receivers before any other waker (death-boundary rule). - The credit return happens in the transition's lock section (an atomic, no user code); `Wake` carries only wakers. Measured and rejected: an `AtomicWaker` beside a value mutex (the "lock-free waker" variant). It costs two more read-modify-writes per request than one lock for both: burst_raw 118.3M vs 114.9M before. Callgrind (micro, instructions; seq paths vary with scheduler parking, burst is the stable one): burst_raw 114.9M -> 110.0M (per request 2201 -> 2107; framework code 865 -> 798) seq_raw 108.6M -> 123.6M (scheduler idle/park noise: framework code per request 1083 -> 1034) seq_default 137.1M -> 154.7M (framework code per request 1305 -> 1282) Loom (`reply_slot_loom` 23 tests, `job_core_loom` 8, `registry_turn_loom` 3): all pass. New: parked_receiver_vs_reply_in_turn, parked_receiver_vs_batched_close (close_collect, notify_parked, run_wake), watcher_and_receiver_vs_deferred_reply, polled_receiver_vs_shorten_and_reply. The Loom park model is a mutex+condvar token (Loom's own unpark also wakes a thread blocked in join). Negative check: removing the under-lock recheck in recv_blocking makes Loom report a deadlock (lost wakeup). No linearization point moved; no TLA+ change. --- concurrency/src/reply/mod.rs | 43 +++-- concurrency/src/reply/slot.rs | 241 ++++++++++++++++----------- concurrency/tests/job_core_loom.rs | 40 ++++- concurrency/tests/reply_slot_loom.rs | 125 +++++++++++++- docs/design/deferred-replies.md | 20 ++- 5 files changed, 336 insertions(+), 133 deletions(-) diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index a7759c48..d945876a 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -73,29 +73,40 @@ mod sync { } } - /// A blocking receiver is about to wait on its slot (under the slot's - /// lock): the failpoint site `reply.recv_parked`, a handshake for tests. + /// A blocking receiver registered its unparker and is about to park: + /// the failpoint site `reply.recv_parked`, a handshake for tests. #[inline(always)] pub(crate) fn parked() { fail_point!("reply.recv_parked"); } - /// Wait on `condvar` for at most `ticks` clock ticks (nanoseconds), or - /// until notified when `None`. - pub(crate) fn wait_ticks<'a, T>( - condvar: &Condvar, - guard: MutexGuard<'a, T>, - ticks: Option, - ) -> MutexGuard<'a, T> { - match ticks { - None => condvar.wait(guard).unwrap_or_else(|p| p.into_inner()), - Some(ticks) => { - condvar - .wait_timeout(guard, Duration::from_nanos(ticks)) - .unwrap_or_else(|p| p.into_inner()) - .0 + /// Run `f` with a waker that unparks the current thread (cached per + /// thread: no allocation, and no reference count change when it is + /// already the slot's registered waker). + pub(crate) fn with_thread_waker(f: impl FnOnce(&std::task::Waker) -> T) -> T { + struct Unpark(std::thread::Thread); + impl std::task::Wake for Unpark { + fn wake(self: Arc) { + self.0.unpark(); + } + fn wake_by_ref(self: &Arc) { + self.0.unpark(); } } + thread_local! { + static WAKER: std::task::Waker = + std::task::Waker::from(Arc::new(Unpark(std::thread::current()))); + } + WAKER.with(|waker| f(waker)) + } + + /// Park the current thread for at most `ticks` clock ticks + /// (nanoseconds), or until unparked when `None`. + pub(crate) fn park(ticks: Option) { + match ticks { + None => std::thread::park(), + Some(ticks) => std::thread::park_timeout(Duration::from_nanos(ticks)), + } } use std::time::Duration; } diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 548025b0..7c8d8ecc 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -20,10 +20,17 @@ //! This file is shared with the Loom model (`tests/reply_slot_loom.rs`), which //! supplies the synchronization types through `super::sync`. Time is passed in //! as a `now` closure returning clock ticks, so the model controls the clock. -//! No user code runs under the slot mutex: values and wakers are moved out and +//! No user code runs under a slot lock: values and wakers are moved out and //! dropped or woken after it is released. - -use super::sync::{wait_ticks, yield_now, Arc, AtomicU64, AtomicUsize, Condvar, Mutex, Ordering}; +//! +//! The slot lock guards the value and the caller's waker (a thread blocked +//! in `recv_blocking` registers its unparker there: no condvar). Deferred +//! token state (the credit pool and the token's watchers) is created by +//! `defer()`, so a plain request never allocates it and its drop is cheap. +//! (An `AtomicWaker` beside a value mutex measured more instructions per +//! request than one lock for both: two more read-modify-writes.) + +use super::sync::{yield_now, Arc, AtomicU64, AtomicUsize, Mutex, Ordering}; use std::task::{Poll, Waker}; pub(crate) const OPEN: u64 = 0; @@ -182,19 +189,29 @@ impl CreditPool { } } -struct Inner { - /// Written only by the reply that moved the word to `WRITING`, and - /// published by `WRITING -> REPLIED` under this lock. - value: Option, - /// The caller's receiver. - waker: Option, +/// Deferred-token state, created by `defer()` (which installs the pool +/// before its CAS can take a credit): a plain request never allocates it. +struct Side { + /// The owner's pool, where this slot's credits return. + pool: Arc, /// Token-side waiters: `ReplyTo::closed()` and caller-gone job /// cancellation. Several may wait at once. watchers: Vec, - /// The owner's pool, set by `defer()` before its CAS installs `TOKEN`. - pool: Option>, - /// Threads parked in `recv_blocking`. - parked: usize, +} + +/// What the slot lock guards. Its drop is two `None` checks and a null +/// check for a plain request. +struct Inner { + /// Written only by the reply that won the word, before the CAS that + /// publishes `REPLIED`; taken by the receiver after it reads `REPLIED`. + value: Option, + /// The caller's receiver: an async task's waker, or the unparker of a + /// thread blocked in `recv_blocking` (then `parked` is set). + waker: Option, + /// The registered waker is a parked thread's (no user code): the death + /// point's discharge wakes it before any other waker. + parked: bool, + side: Option>, } pub(crate) struct Slot { @@ -204,41 +221,49 @@ pub(crate) struct Slot { pub(crate) owner: super::sync::SlotOwner, word: AtomicU64, deadline: AtomicU64, + /// The slot lock: the value and waker handoff, and the order of a + /// deadline shortening against a reply's deadline read. inner: Mutex>, - parked: Condvar, } +#[inline] fn lock(mutex: &Mutex) -> super::sync::MutexGuard<'_, T> { mutex.lock().unwrap_or_else(|p| p.into_inner()) } -/// What a winning transition must do after its CAS: return credits, then wake. +/// What a winning transition must do after its CAS (the credit has already +/// returned): wake the receiver and the token's watchers. pub(crate) struct Wake { - pool: Option>, - release: u64, waker: Option, + /// `waker` unparks a thread blocked in `recv_blocking`. + parked: bool, watchers: Vec, - notify: bool, } impl Wake { - /// Wake the slot's parked blocking receivers (no user code). Done first, - /// so a parked `recv()` never waits behind an arbitrary waker. - fn notify_parked(&mut self, slot_parked: &Condvar) { - if std::mem::take(&mut self.notify) { - slot_parked.notify_all(); + const NONE: Self = Self { + waker: None, + parked: false, + watchers: Vec::new(), + }; + /// Wake a parked blocking receiver (no user code). Done first, so a + /// parked `recv()` never waits behind an arbitrary waker. + fn notify_parked(&mut self) { + if std::mem::take(&mut self.parked) { + if let Some(waker) = self.waker.take() { + waker.wake(); + } } } - fn run(mut self, slot_parked: &Condvar) { - if let Some(pool) = self.pool.take() { - pool.release(self.release); - } - self.notify_parked(slot_parked); + #[inline] + fn run(self) { if let Some(waker) = self.waker { waker.wake(); } - for watcher in self.watchers { - watcher.wake(); + if !self.watchers.is_empty() { + for watcher in self.watchers { + watcher.wake(); + } } } } @@ -258,22 +283,23 @@ impl Slot { inner: Mutex::new(Inner { value: None, waker: None, - watchers: Vec::new(), - pool: None, - parked: 0, + parked: false, + side: None, }), - parked: Condvar::new(), } } + #[inline] pub(crate) fn word(&self) -> u64 { self.word.load(Ordering::Acquire) } + #[inline] pub(crate) fn deadline(&self) -> u64 { self.deadline.load(Ordering::Acquire) } + #[inline] pub(crate) fn is_terminal(&self) -> bool { !live(state(self.word())) } @@ -334,41 +360,38 @@ impl Slot { } } - /// After a winning CAS `old -> new`: collect the credit return and, when - /// the state became terminal, the waker. The caller runs it after every - /// lock is released. + /// After a winning CAS `old -> new`: return the credit delta now and, + /// when the state became terminal, collect the wake-up, which the caller + /// runs after every lock is released. fn after(&self, old: u64, new: u64) -> Wake { let release = credits(old).saturating_sub(credits(new)); let terminal = live(state(old)) && !live(state(new)); if release == 0 && !terminal { - return Wake { - pool: None, - release: 0, - waker: None, - watchers: Vec::new(), - notify: false, - }; + return Wake::NONE; } let mut inner = lock(&self.inner); - Self::collect(&mut inner, release, terminal) + Self::finish(&mut inner, release, terminal) } - /// The wake-up work of a transition, taken under the slot lock. - fn collect(inner: &mut Inner, release: u64, terminal: bool) -> Wake { + /// Under the slot lock: return `release` credits (an atomic, no user + /// code) and, for a terminal transition, take the wake-up. + #[inline] + fn finish(inner: &mut Inner, release: u64, terminal: bool) -> Wake { + if release > 0 { + if let Some(side) = &inner.side { + side.pool.release(release); + } + } + if !terminal { + return Wake::NONE; + } Wake { - pool: if release > 0 { - inner.pool.clone() - } else { - None - }, - release, - waker: if terminal { inner.waker.take() } else { None }, - watchers: if terminal { - std::mem::take(&mut inner.watchers) - } else { - Vec::new() + waker: inner.waker.take(), + parked: inner.parked, + watchers: match &mut inner.side { + Some(side) => std::mem::take(&mut side.watchers), + None => Vec::new(), }, - notify: terminal && inner.parked > 0, } } @@ -402,7 +425,7 @@ impl Slot { drop(permit); match transition { Ok((old, new)) => { - self.after(old, new).run(&self.parked); + self.after(old, new).run(); true } Err(_) => false, @@ -415,7 +438,7 @@ impl Slot { pub(crate) fn close(&self) -> bool { match self.close_collect() { Some(wake) => { - wake.run(&self.parked); + wake.run(); true } None => false, @@ -435,12 +458,8 @@ impl Slot { Err(()) } }) { - let mut wake = self.after(old, new); // The credit returns with the transition; only wakes wait. - if let Some(pool) = wake.pool.take() { - pool.release(wake.release); - } - return Some(wake); + return Some(self.after(old, new)); } if state(self.word()) != WRITING { return None; @@ -451,13 +470,13 @@ impl Slot { /// Run the wake-up work [`close_collect`](Self::close_collect) returned. pub(crate) fn run_wake(&self, wake: Wake) { - wake.run(&self.parked); + wake.run(); } /// The parked-receiver part of `wake`, run ahead of every waker of a /// batch (see `Registry::end_discharge`). pub(crate) fn notify_parked(&self, wake: &mut Wake) { - wake.notify_parked(&self.parked); + wake.notify_parked(); } /// `[deadline]`: expire the slot if `now >= deadline`. Returns whether the @@ -486,7 +505,7 @@ impl Slot { /// finds `WRITING` (or later) is a no-op; one that lowers the deadline is /// seen by any reply that has not read it yet. pub(crate) fn shorten(&self, at: u64, now: &impl Fn() -> u64) -> bool { - let (lowered, watchers, waker, parked) = { + let (lowered, watchers, waker) = { let mut inner = lock(&self.inner); let lower = state(self.word()) == OPEN && at < self.deadline(); if lower { @@ -494,28 +513,24 @@ impl Slot { super::sync::shortened(&self.owner, self as *const Self as usize); // Moved out, not cloned: no executor code under the lock. A // woken receiver re-registers when it polls again. - ( - true, - std::mem::take(&mut inner.watchers), - inner.waker.take(), - inner.parked > 0, - ) + let watchers = match &mut inner.side { + Some(side) => std::mem::take(&mut side.watchers), + None => Vec::new(), + }; + (true, watchers, inner.waker.take()) } else { - (false, Vec::new(), None, false) + (false, Vec::new(), None) } }; // Both sides re-arm their timers for the earlier deadline: the - // token's waiters, the caller's receiver and parked `recv_blocking` - // threads (which recompute their wait). - if parked { - self.parked.notify_all(); + // token's waiters and the caller's receiver (a parked `recv_blocking` + // thread recomputes its wait). + if let Some(waker) = waker { + waker.wake(); } for watcher in watchers { watcher.wake(); } - if let Some(waker) = waker { - waker.wake(); - } self.check_deadline(now); lowered } @@ -531,7 +546,14 @@ impl Slot { if self.check_deadline(now) { return Err(Refused::Closed); } - lock(&self.inner).pool = Some(pool.clone()); + // The side state (the pool credits return to) exists before the CAS + // can take a credit. + lock(&self.inner).side.get_or_insert_with(|| { + Box::new(Side { + pool: pool.clone(), + watchers: Vec::new(), + }) + }); match self.cas(Some(pool), |w| { if state(w) == OPEN && w & TOKEN == 0 { Ok(w | TOKEN | REG) @@ -554,7 +576,10 @@ impl Slot { if self.check_deadline(now) { return Err(Refused::Closed); } - let pool = lock(&self.inner).pool.clone(); + let pool = lock(&self.inner) + .side + .as_ref() + .map(|side| side.pool.clone()); match self.cas(pool.as_deref(), |w| { if state(w) != OPEN || w & TOKEN == 0 { Err(Refused::Closed) @@ -577,7 +602,7 @@ impl Slot { debug_assert!(jobs(w) >= 1, "job_end without a running job"); Ok(w - JOB_ONE) }) { - Ok((old, new)) => self.after(old, new).run(&self.parked), + Ok((old, new)) => self.after(old, new).run(), Err(_) => unreachable!("job_end never aborts or reserves"), } } @@ -627,9 +652,9 @@ impl Slot { }) { Ok((old, new)) => { let wake = - Self::collect(&mut inner, credits(old).saturating_sub(credits(new)), true); + Self::finish(&mut inner, credits(old).saturating_sub(credits(new)), true); drop(inner); - wake.run(&self.parked); + wake.run(); (Delivery::Delivered, None) } Err(lost) => { @@ -687,14 +712,15 @@ impl Slot { (Delivery::Delivered, None, REPLIED) }; let (old, new) = self.resolve_writing(to); - let wake = Self::collect(&mut inner, credits(old).saturating_sub(credits(new)), true); + let wake = Self::finish(&mut inner, credits(old).saturating_sub(credits(new)), true); drop(inner); - wake.run(&self.parked); + wake.run(); (delivery, unsent) } /// Receiver side: take the outcome of a terminal word. `Replied` with no /// value means it was already taken. + #[inline] fn take(&self, state: u64) -> Option> { let terminal = Terminal::of(state)?; Some(match terminal { @@ -720,6 +746,8 @@ impl Slot { return Poll::Ready(result); } let mut inner = lock(&self.inner); + // Checked under the lock a terminal transition takes the waker + // under: a waker stored here is taken by it. if !live(state(self.word())) { continue; } @@ -741,7 +769,7 @@ impl Slot { } /// Token side: ready once the slot is terminal (or its deadline passed, - /// which expires it). Registers `waker` as the watcher otherwise. + /// which expires it). Registers `waker` as a watcher otherwise. #[cfg_attr(not(feature = "async"), allow(dead_code))] pub(crate) fn poll_closed(&self, waker: &Waker, now: &impl Fn() -> u64) -> Poll<()> { let mut fresh: Option = None; @@ -753,7 +781,12 @@ impl Slot { if !live(state(self.word())) { continue; } - if inner.watchers.iter().any(|w| w.will_wake(waker)) { + let Some(side) = &mut inner.side else { + // Only a deferred token (or its linked job) watches its slot. + debug_assert!(false, "poll_closed on a slot that was never deferred"); + return Poll::Ready(()); + }; + if side.watchers.iter().any(|w| w.will_wake(waker)) { drop(inner); drop(fresh); return Poll::Pending; @@ -763,7 +796,7 @@ impl Slot { fresh = Some(waker.clone()); continue; }; - inner.watchers.push(new); + side.watchers.push(new); return Poll::Pending; } } @@ -789,19 +822,27 @@ impl Slot { if self.check_deadline(now) { continue; } - let mut inner = lock(&self.inner); - if !live(state(self.word())) { - continue; - } let deadline = self.deadline(); let remaining = (deadline != NEVER).then(|| deadline.saturating_sub(now())); if remaining == Some(0) { continue; } - inner.parked += 1; + { + let mut inner = lock(&self.inner); + if !live(state(self.word())) { + continue; + } + // The thread's own unparker (no user code to clone or drop). + super::sync::with_thread_waker(|waker| { + if !inner.waker.as_ref().is_some_and(|w| w.will_wake(waker)) { + inner.waker = Some(waker.clone()); + } + }); + inner.parked = true; + } super::sync::parked(); - let mut inner = wait_ticks(&self.parked, inner, remaining); - inner.parked -= 1; + // A stale unpark or a spurious return only loops. + super::sync::park(remaining); } } diff --git a/concurrency/tests/job_core_loom.rs b/concurrency/tests/job_core_loom.rs index 41f69bd0..08c737a6 100644 --- a/concurrency/tests/job_core_loom.rs +++ b/concurrency/tests/job_core_loom.rs @@ -25,14 +25,40 @@ mod sync { pub(crate) fn parked() {} pub(crate) use super::model_arc::{JobArc, JobWeak}; - /// Loom has no timed wait; the model never parks on a slot. - pub(crate) fn wait_ticks<'a, T>( - condvar: &Condvar, - guard: MutexGuard<'a, T>, - ticks: Option, - ) -> MutexGuard<'a, T> { + /// The thread's park token: `std::thread::park`'s semantics (a wake + /// sets the token; a park consumes it), modelled with a Loom mutex and + /// condvar because Loom's own `unpark` also wakes a thread blocked in a + /// `join`. + #[derive(Default)] + pub(crate) struct Token { + set: loom::sync::Mutex, + cv: loom::sync::Condvar, + } + impl std::task::Wake for Token { + fn wake(self: std::sync::Arc) { + *self.set.lock().unwrap() = true; + self.cv.notify_all(); + } + } + loom::thread_local! { + static TOKEN: std::sync::Arc = std::sync::Arc::new(Token::default()); + } + + /// A waker that unparks the current Loom thread. + pub(crate) fn with_thread_waker(f: impl FnOnce(&std::task::Waker) -> T) -> T { + let waker = TOKEN.with(|token| std::task::Waker::from(token.clone())); + f(&waker) + } + + /// Loom has no timed park; tests park only on slots without a deadline. + pub(crate) fn park(ticks: Option) { assert!(ticks.is_none(), "timed park is not modelled"); - condvar.wait(guard).unwrap_or_else(|p| p.into_inner()) + let token = TOKEN.with(|token| token.clone()); + let mut set = token.set.lock().unwrap(); + while !*set { + set = token.cv.wait(set).unwrap(); + } + *set = false; } } diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index e3476951..9a427574 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -9,7 +9,7 @@ mod sync { pub(crate) use loom::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; - pub(crate) use loom::sync::{Arc, Condvar, Mutex, MutexGuard}; + pub(crate) use loom::sync::{Arc, Mutex, MutexGuard}; pub(crate) use loom::thread::yield_now; pub(crate) type SlotOwner = Option<()>; pub(crate) fn section(f: impl FnOnce() -> T) -> T { @@ -19,14 +19,40 @@ mod sync { pub(crate) fn shortened(_: &SlotOwner, _: usize) {} pub(crate) fn parked() {} - /// Loom has no timed wait; tests park only on slots without a deadline. - pub(crate) fn wait_ticks<'a, T>( - condvar: &Condvar, - guard: MutexGuard<'a, T>, - ticks: Option, - ) -> MutexGuard<'a, T> { + /// The thread's park token: `std::thread::park`'s semantics (a wake + /// sets the token; a park consumes it), modelled with a Loom mutex and + /// condvar because Loom's own `unpark` also wakes a thread blocked in a + /// `join`. + #[derive(Default)] + pub(crate) struct Token { + set: loom::sync::Mutex, + cv: loom::sync::Condvar, + } + impl std::task::Wake for Token { + fn wake(self: std::sync::Arc) { + *self.set.lock().unwrap() = true; + self.cv.notify_all(); + } + } + loom::thread_local! { + static TOKEN: std::sync::Arc = std::sync::Arc::new(Token::default()); + } + + /// A waker that unparks the current Loom thread. + pub(crate) fn with_thread_waker(f: impl FnOnce(&std::task::Waker) -> T) -> T { + let waker = TOKEN.with(|token| std::task::Waker::from(token.clone())); + f(&waker) + } + + /// Loom has no timed park; tests park only on slots without a deadline. + pub(crate) fn park(ticks: Option) { assert!(ticks.is_none(), "timed park is not modelled"); - condvar.wait(guard).unwrap_or_else(|p| p.into_inner()) + let token = TOKEN.with(|token| token.clone()); + let mut set = token.set.lock().unwrap(); + while !*set { + set = token.cv.wait(set).unwrap(); + } + *set = false; } } @@ -536,3 +562,86 @@ fn defer_at_the_bound_vs_abandon() { assert_eq!(pool.used(), 0); }); } + +/// A parked threads-mode receiver is woken by the in-turn reply (one CAS +/// publishing the value; the receiver's unparker sits in the waker cell). +#[test] +fn parked_receiver_vs_reply_in_turn() { + model(|| { + let slot: Arc> = Arc::new(Slot::new(NEVER)); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply_in_turn(5, &|| 0).0) + }; + assert_eq!(slot.recv_blocking(&|| 0), Ok(5)); + assert_eq!(replier.join().unwrap(), Delivery::Delivered); + }); +} + +/// The death point's batched close: the close collects the wake-up, the +/// parked receivers are notified first (`notify_parked`), the rest runs +/// after. A parked receiver is never lost between the two steps. +#[test] +fn parked_receiver_vs_batched_close() { + model(|| { + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, NEVER); + let closer = { + let slot = slot.clone(); + loom::thread::spawn(move || { + let mut wake = slot.close_collect().expect("the close wins an open slot"); + slot.notify_parked(&mut wake); + slot.run_wake(wake); + }) + }; + assert_eq!(slot.recv_blocking(&|| 0), Err(Terminal::Stopped)); + closer.join().unwrap(); + assert_eq!(pool.used(), 0, "the credit returned with the close"); + assert_balanced(&pool, &[&slot]); + }); +} + +/// A token watcher (`ReplyTo::closed`, caller-gone job cancellation) is +/// woken by a deferred reply from another thread (no lost wakeup), and the +/// caller's receiver gets the value. +#[test] +fn watcher_and_receiver_vs_deferred_reply() { + model(|| { + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, NEVER); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply(8, &|| 0).0) + }; + loom::future::block_on(std::future::poll_fn(|cx| { + slot.poll_closed(cx.waker(), &|| 0) + })); + assert_eq!(replier.join().unwrap(), Delivery::Delivered); + let got = loom::future::block_on(std::future::poll_fn(|cx| slot.poll(cx.waker()))); + assert_eq!(got, Ok(8)); + assert_balanced(&pool, &[&slot]); + }); +} + +/// A polled receiver versus a shortening (which wakes it to re-arm) and +/// the expiry that follows: the receiver ends `TimedOut` or with the +/// reply, never blocked on a taken waker. +#[test] +fn polled_receiver_vs_shorten_and_reply() { + model(|| { + let clock = Clock::at(5); + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, 100); + let owner = { + let (slot, clock) = (slot.clone(), clock.clone()); + loom::thread::spawn(move || { + slot.shorten(5, &clock.now()); + slot.reply(2, &clock.now()).0 + }) + }; + let got = loom::future::block_on(std::future::poll_fn(|cx| slot.poll(cx.waker()))); + assert_eq!(got, Err(Terminal::TimedOut)); + assert_eq!(owner.join().unwrap(), Delivery::Expired); + assert_balanced(&pool, &[&slot]); + }); +} diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index dbf74f2e..d7835af1 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -88,10 +88,26 @@ word: AtomicU64 = state:4 | TOKEN:1 | REG:1 | JOBS:16 | (spare) JOBS: running linked jobs (pipe_reply); MAX_OUTSTANDING is limited to 65535 at configuration, so the field cannot overflow deadline: AtomicU64 rt clock ticks; u64::MAX = never; only decreases -value: written only by the thread that moved the word to WRITING -waker: AtomicWaker (tasks) / Mutex+Condvar (threads) +inner: one mutex (the slot lock) over + value: written only by the reply that won the word, before the + CAS that publishes REPLIED + waker: the caller's task waker, or the unparker of a thread + blocked in recv() (then `parked` is set, so the death + point's discharge wakes it before any other waker) + side: boxed, created by defer(): the owner's credit pool and + the token's watchers (ReplyTo::closed, caller-gone job + cancellation); a plain request never allocates it + the slot lock also orders a shortening against a reply's + deadline read ``` +A terminal transition takes the waker under the slot lock after its CAS; +a receiver checks the word under that lock before it registers, so either +the transition finds its waker or the check sees the terminal state (no +lost wakeup; `tests/reply_slot_loom.rs`). A blocking receiver parks its +thread (no condvar). An `AtomicWaker` beside a value mutex was measured and +costs more per request than one lock for both. + **Credits held by a slot:** ```text From 9c1f09d7de7d9622bf91229bd6c65ac8b566bc15 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 15:42:53 -0300 Subject: [PATCH 196/208] perf(tasks): the envelope lends its request slot to the turn (no per-message reference count) The tasks run loop cloned the caller's slot (`envelope.caller()`, an `Arc`) for `DeathFirst` on every message and dropped it after the turn: two atomic read-modify-writes per request on the slot's count, which the caller's thread also touches. Now `Envelope::handle` takes `&mut self` and returns the turn's future with the caller it answers (`Dispatch<'a>`): the envelope keeps the request's slot for the whole turn, the handler's `PendingReply` borrows it (`PendingReply::lent`, a `Cow`), and `DeathFirst` borrows it as a `Caller`. A reference is taken only when it must outlive the turn: `defer()` (`ReplyTo` owns its slot), an obligation dropped unanswered (recorded on the turn), and the rare runtime drop of a pending turn (`Caller::retain`, then answered at the death point as before). Payloads move out through `Option`s at the one dispatch. Threads mode never cloned and is unchanged. `DeathFirst` now returns the turn's caught panic instead of resuming it, and the handler's future is built under a plain `catch_unwind` (`dispatch`): the turn no longer runs inside a second `AssertUnwindSafe(async {..}).catch_unwind()` state machine. `Slot::poll` clones the waker before taking the lock on the first registration: one lock section instead of two. Callgrind (micro, burst_raw 51200 + warm-up): 110.0M -> 110.4M (framework code per request 798 -> 801: the saved work is two contended RMWs, which instruction counts do not show; the turn's cost moved from the removed async wrapper into the run loop). seq_raw 123.6M -> 117.9M, seq_default 154.7M -> 151.0M (scheduler-noise dominated). Loom: reply_slot_loom 24 (new lent_reply_vs_receiver_last_reference: the in-turn reply through the borrow versus the caller taking or abandoning, either side dropping the last reference, the value dropped exactly once), job_core_loom 8, registry_turn_loom 3: all pass. No linearization point moved; death_boundary and failpoints suites pass (the runtime-drop path answers the lent caller first, as before). --- concurrency/src/reply/mod.rs | 48 +++-- concurrency/src/reply/slot.rs | 10 +- concurrency/src/tasks/actor.rs | 254 +++++++++++++++------------ concurrency/src/tasks/jobs.rs | 56 +++--- concurrency/tests/reply_slot_loom.rs | 47 +++++ 5 files changed, 265 insertions(+), 150 deletions(-) diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index d945876a..3a0f8448 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -21,8 +21,10 @@ pub use slot::Delivery; use crate::error::ActorError; use crate::ActorId; use registry::Registry; -use slot::{Slot, Terminal, NEVER}; +pub(crate) use slot::Slot; +use slot::{Terminal, NEVER}; use spawned_rt::time::Instant; +use std::borrow::Cow; use std::marker::PhantomData; use std::sync::Arc; use std::time::Duration; @@ -242,13 +244,19 @@ impl SlotHandle { pub(crate) fn take(&mut self) -> Option>> { self.0.take() } - /// The request's slot, type-erased (the turn's caller, for the death - /// point to answer if the turn is destroyed unfinished). - #[cfg_attr(not(feature = "async"), allow(dead_code))] - pub(crate) fn caller(&self) -> Option> { - self.0 - .as_ref() - .map(|slot| slot.clone() as Arc) +} + +/// A turn's caller, lent by its envelope for the turn: retained (one +/// reference) only if the runtime destroys the turn unfinished, for the +/// death point to answer. +#[cfg_attr(not(feature = "async"), allow(dead_code))] +pub(crate) trait Caller: Sync { + fn retain(&self) -> Arc; +} + +impl Caller for Arc> { + fn retain(&self) -> Arc { + self.clone() } } @@ -295,7 +303,8 @@ pub enum DeferError { /// `PendingReply`: `reply` drops the value and `defer` returns /// [`DeferError::NoCaller`]. pub struct PendingReply<'t, R: Send + 'static> { - slot: Option>>, + /// Lent by the dispatching envelope for the turn (tasks mode), or owned. + slot: Option>>>, registry: Option<&'t Arc>, _turn: PhantomData<&'t mut ()>, } @@ -310,12 +319,24 @@ impl std::fmt::Debug for PendingReply<'_, R> { impl<'t, R: Send + 'static> PendingReply<'t, R> { pub(crate) fn new(slot: Option>>, registry: &'t Arc) -> Self { + Self::with_slot(slot.map(Cow::Owned), registry) + } + + /// The obligation of a request whose slot the dispatching envelope + /// keeps for the turn: no reference is taken unless it is deferred or + /// dropped unanswered. + #[cfg_attr(not(feature = "async"), allow(dead_code))] + pub(crate) fn lent(slot: Option<&'t Arc>>, registry: &'t Arc) -> Self { + Self::with_slot(slot.map(Cow::Borrowed), registry) + } + + fn with_slot(slot: Option>>>, registry: &'t Arc) -> Self { #[cfg(feature = "lifecycle-trace")] if let Some(slot) = &slot { crate::lifecycle_trace::reply( registry.owner().raw(), "Dispatch", - Arc::as_ptr(slot) as u64, + Arc::as_ptr(&**slot) as u64, "", ); } @@ -364,7 +385,7 @@ impl<'t, R: Send + 'static> PendingReply<'t, R> { /// caller is answered `Overloaded` and this returns /// [`DeferError::Overloaded`]. pub fn defer(mut self) -> Result, DeferError> { - let slot = self.slot.take().ok_or(DeferError::NoCaller)?; + let slot = self.slot.take().ok_or(DeferError::NoCaller)?.into_owned(); let registry = self.registry.take().ok_or(DeferError::NoCaller)?; match slot.defer(registry.pool(), &now_ticks) { Ok(()) => { @@ -410,8 +431,11 @@ impl<'t, R: Send + 'static> PendingReply<'t, R> { impl Drop for PendingReply<'_, R> { fn drop(&mut self) { if let Some(slot) = self.slot.take() { + if slot.is_terminal() { + return; + } match self.registry { - Some(registry) => registry.dropped(slot), + Some(registry) => registry.dropped(slot.into_owned()), None => { slot.terminate(Terminal::Dropped); } diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 7c8d8ecc..f3298d65 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -738,11 +738,17 @@ impl Slot { /// reads no clock. #[cfg_attr(not(feature = "async"), allow(dead_code))] pub(crate) fn poll(&self, waker: &Waker) -> Poll> { + if let Some(result) = self.take(state(self.word())) { + return Poll::Ready(result); + } // `Waker::clone` and dropping a waker run executor code, so neither - // happens under the slot lock. - let mut fresh: Option = None; + // happens under the slot lock. The first registration clones before + // locking (one lock section instead of two); a later poll with the + // same waker finds it registered and drops the copy. + let mut fresh: Option = Some(waker.clone()); loop { if let Some(result) = self.take(state(self.word())) { + drop(fresh); return Poll::Ready(result); } let mut inner = lock(&self.inner); diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 3446fd98..6c09761e 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -9,7 +9,7 @@ use crate::link::{ use crate::mailbox::tasks as mpsc; use crate::message::Message; use crate::monitor::{Down, MonitorCleanup, MonitorRef, MonitorTable}; -use crate::reply::{registry::Registry, reply_slot, SlotHandle}; +use crate::reply::{registry::Registry, reply_slot, Caller, Slot, SlotHandle}; use crate::reply::{Deadline, PendingReply, ReplyReceiver, DEFAULT_MAX_OUTSTANDING}; use core::pin::pin; use futures::future::{self, FutureExt as _}; @@ -164,17 +164,39 @@ trait Envelope: Send { /// mailbox is destroyed. No user code runs. fn discharge(&mut self) {} - /// The request's caller, if any (answered at the death point if the - /// runtime destroys the turn before it finishes). - fn caller(&self) -> Option> { - None - } + /// Dispatch once: the turn's future, and the request's caller if any. + /// The envelope keeps the request's slot for the whole turn and lends + /// it to the handler's `PendingReply` (no reference count change per + /// message); the run loop answers the lent caller at the death point if + /// the runtime destroys the turn before it finishes. + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a>; +} + +/// A dispatched turn: its future and the caller it answers, lent by the +/// envelope for the turn. +type Dispatch<'a> = ( + Pin + Send + 'a>>, + Option<&'a dyn Caller>, +); + +/// Dispatch `envelope`, catching a panic while the handler builds its +/// future (a manual `Handler` may panic before it returns one): a handler +/// panic like any other. +fn dispatch<'a, A: Actor>( + envelope: &'a mut (dyn Envelope + Send), + actor: &'a mut A, + ctx: &'a Context, +) -> Result, Box> { + let mut parts = Some((envelope, actor)); + std::panic::catch_unwind(AssertUnwindSafe(move || { + let (envelope, actor) = parts.take().expect("dispatched once"); + envelope.handle(actor, ctx) + })) +} - fn handle<'a>( - self: Box, - actor: &'a mut A, - ctx: &'a Context, - ) -> Pin + Send + 'a>>; +/// Take an envelope's payload at its one dispatch. +fn dispatched(payload: &mut Option) -> T { + payload.take().expect("an envelope is dispatched once") } /// Answer every request queued in a dead owner's mailbox, destroying @@ -211,7 +233,9 @@ fn erase(items: Vec) -> Vec> { struct MessageEnvelope { // Before the message: a plain drop also answers the request first. slot: SlotHandle, - msg: M, + /// The slot lent to the dispatched turn (taken from `slot`). + lent: Option>>, + msg: Option, #[cfg(feature = "metrics")] observation: Option, /// The recorded actor that sent it (0: outside code or a request); its @@ -251,17 +275,18 @@ where fn discharge(&mut self) { drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); } - fn caller(&self) -> Option> { - self.slot.caller() - } - fn handle<'a>( - self: Box, - actor: &'a mut A, - ctx: &'a Context, - ) -> Pin + Send + 'a>> { - let mut this = *self; - let reply = PendingReply::new(this.slot.take(), &ctx.shared.replies); - Box::pin(actor.handle(this.msg, reply, ctx)) + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let Self { + slot, lent, msg, .. + } = self; + let msg = dispatched(msg); + *lent = slot.take(); + let lent: &'a Option>> = lent; + let reply = PendingReply::lent(lent.as_ref(), &ctx.shared.replies); + ( + Box::pin(actor.handle(msg, reply, ctx)), + lent.as_ref().map(|slot| slot as &dyn Caller), + ) } } @@ -284,19 +309,16 @@ impl> Envelope for DownEnvelope { observation.reject(); } } - fn handle<'a>( - self: Box, - actor: &'a mut A, - ctx: &'a Context, - ) -> Pin + Send + 'a>> { - let down = self.down; - Box::pin(async move { + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let down = dispatched(&mut self.down); + let turn = Box::pin(async move { fail_point!("monitor.down_before_dispatch", async); if crate::monitor::fence::take_queued(&ctx.shared.monitors, &down.monitor_ref) { let reply = PendingReply::new(None, &ctx.shared.replies); actor.handle(down, reply, ctx).await; } - }) + }); + (turn, None) } } @@ -317,8 +339,9 @@ where { let from = crate::lifecycle_trace::life_sender::(replies.owner().raw(), slot.is_detached()); let envelope: Box + Send> = Box::new(MessageEnvelope { - msg, + msg: Some(msg), slot, + lent: None, #[cfg(feature = "metrics")] observation: Some(_mailbox.enqueued()), #[cfg(feature = "lifecycle-trace")] @@ -334,7 +357,7 @@ where /// `demonitor(Ref, [flush])`). A `Down` user code sends is an ordinary /// message. struct DownEnvelope { - down: Down, + down: Option, #[cfg(feature = "metrics")] observation: Option, } @@ -346,7 +369,7 @@ fn enqueue_down>( down: Down, ) -> Result<(), ActorError> { let envelope: Box + Send> = Box::new(DownEnvelope { - down, + down: Some(down), #[cfg(feature = "metrics")] observation: Some(_mailbox.enqueued()), }); @@ -392,7 +415,8 @@ fn admit( type InTurn = Box) -> R + Send>; struct InTurnEnvelope { slot: SlotHandle, - f: InTurn, + lent: Option>>, + f: Option>, } impl Envelope for InTurnEnvelope { @@ -403,41 +427,38 @@ impl Envelope for InTurnEnvelope { fn discharge(&mut self) { drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); } - fn caller(&self) -> Option> { - self.slot.caller() - } - fn handle<'a>( - self: Box, - actor: &'a mut A, - ctx: &'a Context, - ) -> Pin + Send + 'a>> { - let InTurnEnvelope { f, mut slot } = *self; + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let Self { slot, lent, f } = self; + let f = dispatched(f); + *lent = slot.take(); + let lent: &'a Option>> = lent; // Inside the future, so a panicking closure is a handler panic. - Box::pin(async move { - let reply = PendingReply::new(slot.take(), &ctx.shared.replies); + let turn = Box::pin(async move { + let reply = PendingReply::lent(lent.as_ref(), &ctx.shared.replies); let value = f(actor, ctx); let _ = reply.reply(value); - }) + }); + (turn, lent.as_ref().map(|slot| slot as &dyn Caller)) } } // Private lifecycle signal, excluded from application message/handler metrics. // Nested supervisor adapters have no public mailbox traffic to bury this signal. -struct FailureEnvelope(String); +struct FailureEnvelope(Option); impl Envelope for FailureEnvelope { #[cfg(feature = "diagnostics")] fn message_type(&self) -> &'static str { "Exit" } - fn handle<'a>( - self: Box, - _: &'a mut A, - _: &'a Context, - ) -> Pin + Send + 'a>> { - Box::pin(async move { - panic!("nested supervisor failed: {}", self.0); - }) + fn handle<'a>(&'a mut self, _: &'a mut A, _: &'a Context) -> Dispatch<'a> { + let failure = dispatched(&mut self.0); + ( + Box::pin(async move { + panic!("nested supervisor failed: {failure}"); + }), + None, + ) } } @@ -446,7 +467,7 @@ impl Envelope for FailureEnvelope { /// `Handler` on the actor — every actor can receive `Exit` via the /// trait's default-no-op `exit_received` callback. struct ExitEnvelope { - exit: Exit, + exit: Option, } impl Envelope for ExitEnvelope { @@ -460,21 +481,21 @@ impl Envelope for ExitEnvelope { "Exit" } - fn handle<'a>( - self: Box, - actor: &'a mut A, - ctx: &'a Context, - ) -> Pin + Send + 'a>> { - Box::pin(async move { - actor.exit_received(self.exit, ctx).await; - }) + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let exit = dispatched(&mut self.exit); + ( + Box::pin(async move { + actor.exit_received(exit, ctx).await; + }), + None, + ) } } /// A parent exit (E3): when dispatched, exit begins with the parent's reason /// and `stopped()` runs. `exit_received` never sees it. struct ParentExitEnvelope { - exit: Exit, + exit: Option, } impl Envelope for ParentExitEnvelope { @@ -488,12 +509,9 @@ impl Envelope for ParentExitEnvelope { "ParentExit" } - fn handle<'a>( - self: Box, - _actor: &'a mut A, - ctx: &'a Context, - ) -> Pin + Send + 'a>> { - Box::pin(async move { ctx.parent_exit(self.exit.reason) }) + fn handle<'a>(&'a mut self, _actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let exit = dispatched(&mut self.exit); + (Box::pin(async move { ctx.parent_exit(exit.reason) }), None) } } @@ -501,7 +519,7 @@ impl Envelope for ParentExitEnvelope { /// (phase 0). If the actor trapped exits when `started()` returned it is an /// `Exit` (or parent exit); otherwise it was resolved then and is inert. struct HeldEnvelope { - exit: Exit, + exit: Option, parent: bool, } @@ -516,21 +534,20 @@ impl Envelope for HeldEnvelope { "Exit" } - fn handle<'a>( - self: Box, - actor: &'a mut A, - ctx: &'a Context, - ) -> Pin + Send + 'a>> { - Box::pin(async move { + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let exit = dispatched(&mut self.exit); + let parent = self.parent; + let turn = Box::pin(async move { if !link::lock_request(&ctx.shared.linked_reason).held_trapped() { return; } - if self.parent { - ctx.parent_exit(self.exit.reason); + if parent { + ctx.parent_exit(exit.reason); } else { - actor.exit_received(self.exit, ctx).await; + actor.exit_received(exit, ctx).await; } - }) + }); + (turn, None) } } @@ -547,15 +564,19 @@ fn signal_target( let parent_tx = sender.clone(); let held_tx = sender.clone(); let send_exit: SendExitFn = Arc::new(move |exit: Exit| { - let envelope: Box + Send> = Box::new(ExitEnvelope { exit }); + let envelope: Box + Send> = Box::new(ExitEnvelope { exit: Some(exit) }); exit_tx.send(envelope).map_err(|_| ActorError::NoProc) }); let send_parent_exit: SendExitFn = Arc::new(move |exit: Exit| { - let envelope: Box + Send> = Box::new(ParentExitEnvelope { exit }); + let envelope: Box + Send> = + Box::new(ParentExitEnvelope { exit: Some(exit) }); parent_tx.send(envelope).map_err(|_| ActorError::NoProc) }); let send_held: SendHeldFn = Arc::new(move |exit: Exit, parent: bool| { - let envelope: Box + Send> = Box::new(HeldEnvelope { exit, parent }); + let envelope: Box + Send> = Box::new(HeldEnvelope { + exit: Some(exit), + parent, + }); held_tx.send(envelope).map_err(|_| ActorError::NoProc) }); Arc::new(SignalTarget { @@ -1239,7 +1260,10 @@ impl ActorRef { .await } pub(crate) fn signal_failure(&self, error: String) { - let _ = self.shared.sender.send(Box::new(FailureEnvelope(error))); + let _ = self + .shared + .sender + .send(Box::new(FailureEnvelope(Some(error)))); } /// Send a fire-and-forget message to the actor. pub fn send(&self, msg: M) -> Result<(), ActorError> @@ -1302,8 +1326,9 @@ impl ActorRef { { let (slot, rx) = reply_slot(Deadline::Never, Some(&self.shared.replies)); let envelope: Box + Send> = Box::new(InTurnEnvelope { - f: Box::new(f), + f: Some(Box::new(f)), slot, + lent: None, }); // Past the death point the actor is gone (E4), as for `enqueue`. if self.shared.replies.refuses_admission() { @@ -1780,14 +1805,14 @@ struct DeathFirst<'a, A: Actor, C: Fn()> { inner: Option + Send + 'a>>>, rx: &'a mut mpsc::Receiver + Send>>, cleanup: &'a mut crate::actor_cleanup::ActorCleanup, - caller: Option>, + caller: Option<&'a dyn Caller>, } impl<'a, A: Actor, C: Fn()> DeathFirst<'a, A, C> { fn new( inner: Pin + Send + 'a>>, rx: &'a mut mpsc::Receiver + Send>>, cleanup: &'a mut crate::actor_cleanup::ActorCleanup, - caller: Option>, + caller: Option<&'a dyn Caller>, ) -> Self { Self { inner: Some(inner), @@ -1797,25 +1822,31 @@ impl<'a, A: Actor, C: Fn()> DeathFirst<'a, A, C> { } } } +/// A caught panic payload. +type Panic = Box; + impl Future for DeathFirst<'_, A, C> { - type Output = (); - fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll<()> { + /// The future's panic, caught: the turn's (or hook's) panic, not a + /// runtime drop. The future is gone before it is returned. + type Output = Result<(), Panic>; + fn poll( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll { let this = self.get_mut(); let inner = this.inner.as_mut().expect("polled after completion"); match std::panic::catch_unwind(AssertUnwindSafe(|| inner.as_mut().poll(cx))) { Ok(std::task::Poll::Pending) => std::task::Poll::Pending, Ok(std::task::Poll::Ready(())) => { this.inner = None; - std::task::Poll::Ready(()) + std::task::Poll::Ready(Ok(())) } - // A panic is the turn's (or hook's) panic, not a runtime drop: - // the future is gone before the panic continues. Err(panic) => { let inner = this.inner.take(); if let Err(secondary) = std::panic::catch_unwind(AssertUnwindSafe(|| drop(inner))) { crate::mailbox::dispose_panic(secondary); } - std::panic::resume_unwind(panic) + std::task::Poll::Ready(Err(panic)) } } } @@ -1828,7 +1859,7 @@ impl Drop for DeathFirst<'_, A, C> { // Destroyed while pending: the runtime dropped the actor. self.cleanup.die_dropped(); if let Some(caller) = self.caller.take() { - self.cleanup.answer_turn(caller); + self.cleanup.answer_turn(caller.retain()); } let queued = answer_mailbox(self.rx); self.cleanup.discharged(); @@ -1949,6 +1980,7 @@ async fn run_actor_lifecycle( None, ) .await + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)) }) .catch_unwind() .await; @@ -2051,17 +2083,16 @@ async fn run_actor_lifecycle( // The handler's future is built inside the guarded future: a // manual `Handler` may panic before it returns one, and that // is a handler panic like any other (threads mode catches the - // same call). - let caller = envelope.caller(); - let result = AssertUnwindSafe(async { - let turn = crate::lifecycle_trace::in_actor( - ctx.shared.id.raw(), - envelope.handle(actor, &ctx), - ); - DeathFirst::new(turn, &mut *rx, &mut *cleanup, caller).await - }) - .catch_unwind() - .await; + // same call). The envelope outlives the turn and lends it the + // request's slot. + let mut envelope = envelope; + let result = match dispatch(&mut *envelope, &mut *actor, &ctx) { + Ok((turn, caller)) => { + let turn = crate::lifecycle_trace::in_actor(ctx.shared.id.raw(), turn); + DeathFirst::new(turn, &mut *rx, &mut *cleanup, caller).await + } + Err(panic) => Err(panic), + }; ctx.shared.replies.end_turn(result.is_err()); #[cfg(feature = "diagnostics")] drop(_turn); @@ -2143,6 +2174,7 @@ async fn run_actor_lifecycle( None, ) .await + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)) }) .catch_unwind() .await; @@ -4394,8 +4426,9 @@ mod turn_regressions { let count = Arc::new(AtomicUsize::new(0)); for _ in 0..2 { tx.send(Box::new(MessageEnvelope { - msg: Inc, + msg: Some(Inc), slot: SlotHandle::detached(), + lent: None, #[cfg(feature = "metrics")] observation: None, #[cfg(feature = "lifecycle-trace")] @@ -4479,8 +4512,9 @@ mod idle_stop_regression { assert!(futures::poll!(actor.as_mut()).is_pending()); let (slot, response) = reply_slot(crate::reply::Deadline::Never, None); tx.send(Box::new(MessageEnvelope { - msg: Increment, + msg: Some(Increment), slot, + lent: None, #[cfg(feature = "metrics")] observation: None, #[cfg(feature = "lifecycle-trace")] diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index 6f6706a4..3ef94277 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -4,7 +4,7 @@ //! A child module of `tasks::actor`, so it reaches the private envelope and //! shared-state types. -use super::{Actor, Context, Envelope, Handler}; +use super::{dispatched, Actor, Context, Dispatch, Envelope, Handler}; use crate::message::Message; use crate::reply::deadline::Expirable; use crate::reply::jobs::{ @@ -16,7 +16,7 @@ use crate::reply::{now_ticks, PendingReply, ReplyTo}; use futures::future::{self, FutureExt as _}; use std::future::Future; use std::panic::AssertUnwindSafe; -use std::pin::{pin, Pin}; +use std::pin::pin; use std::sync::{Arc, Mutex, Weak}; // --------------------------------------------------------------------------- @@ -35,8 +35,8 @@ impl Drop for EmitCredit { /// A message emitted by a job; fenced at dispatch when the job is cancelled /// or belongs to another instance. struct EmitEnvelope { - msg: M, - credit: EmitCredit, + msg: Option, + credit: Option, } impl Envelope for EmitEnvelope @@ -48,20 +48,20 @@ where fn message_type(&self) -> &'static str { std::any::type_name::() } - fn handle<'a>( - self: Box, - actor: &'a mut A, - ctx: &'a Context, - ) -> Pin + Send + 'a>> { - let EmitEnvelope { msg, credit } = *self; + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let msg = dispatched(&mut self.msg); + let credit = dispatched(&mut self.credit); let fenced = credit.0.fenced(ctx.id(), "emit"); // The credit returns at dispatch (or here, when fenced). drop(credit); if fenced { ctx.shared.sender.failures().discard(msg); - return Box::pin(async {}); + return (Box::pin(async {}), None); } - Box::pin(actor.handle(msg, PendingReply::detached(), ctx)) + ( + Box::pin(actor.handle(msg, PendingReply::detached(), ctx)), + None, + ) } } @@ -71,8 +71,8 @@ type Done = Box) -> M + Send>; /// produced message is dispatched in the same turn. struct TerminalEnvelope { job: Arc, - done: Done, - outcome: JobOutcome, + /// `done` and the outcome, taken at dispatch. + payload: Option<(Done, JobOutcome)>, } impl Envelope for TerminalEnvelope @@ -85,12 +85,10 @@ where fn message_type(&self) -> &'static str { std::any::type_name::() } - fn handle<'a>( - self: Box, - actor: &'a mut A, - ctx: &'a Context, - ) -> Pin + Send + 'a>> { - let TerminalEnvelope { job, done, outcome } = *self; + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let Self { job, payload } = self; + let job: &'a Arc = job; + let (done, outcome) = dispatched(payload); let kind = if matches!(outcome, Err(JobError::DeadlineExceeded)) { "deadline" } else { @@ -99,7 +97,7 @@ where // Everything runs inside the returned future, so a panicking `done` // is a handler panic caught by the run loop (`stopped()`, turn end // and job cancellation still run). - Box::pin(async move { + let turn = Box::pin(async move { if job.fenced(ctx.id(), kind) { // Fenced: `done` and its captures (a linked token) are // discarded in the turn, through the mailbox drop path. @@ -108,7 +106,8 @@ where } let msg = done(outcome); actor.handle(msg, PendingReply::detached(), ctx).await - }) + }); + (turn, None) } } @@ -165,8 +164,7 @@ where drop(hold); let envelope: Box + Send> = Box::new(TerminalEnvelope { job: self.state.clone(), - done, - outcome, + payload: Some((done, outcome)), }); self.ctx.shared.sender.send(envelope).err().map(|e| e.0) } @@ -381,7 +379,10 @@ impl JobCtx { let credit = EmitCredit(self.state.clone()); let ctx = self.ctx.clone(); let send: SendScheduled = Box::new(move || { - let envelope: Box + Send> = Box::new(EmitEnvelope { msg, credit }); + let envelope: Box + Send> = Box::new(EmitEnvelope { + msg: Some(msg), + credit: Some(credit), + }); ctx.shared .sender .send(envelope) @@ -414,7 +415,10 @@ impl JobCtx { let credit = EmitCredit(self.state.clone()); self.state .enqueue_emit(|| { - let envelope: Box + Send> = Box::new(EmitEnvelope { msg, credit }); + let envelope: Box + Send> = Box::new(EmitEnvelope { + msg: Some(msg), + credit: Some(credit), + }); self.ctx.shared.sender.send(envelope) }) .map_err(|(gone, unsent)| { diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index 9a427574..48f8e561 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -645,3 +645,50 @@ fn polled_receiver_vs_shorten_and_reply() { assert_balanced(&pool, &[&slot]); }); } + +/// The dispatching envelope lends its slot to the turn (no reference taken +/// per message): the in-turn reply goes through the borrow while the +/// caller takes the value or abandons, and either side may drop the last +/// reference. The value is dropped exactly once, by whoever holds it last, +/// and never while the other side still reads it. +#[test] +fn lent_reply_vs_receiver_last_reference() { + use loom::sync::atomic::AtomicUsize; + struct Counted(Arc); + impl Drop for Counted { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + model(|| { + let drops = Arc::new(AtomicUsize::new(0)); + let lent: Arc> = Arc::new(Slot::new(NEVER)); + let receiver = { + let slot = lent.clone(); + loom::thread::spawn(move || match slot.try_take(&|| 0) { + Some(Ok(value)) => { + drop(value); + true + } + Some(Err(_)) => false, + None => { + slot.abandon(); + false + } + }) + }; + let borrowed: &Arc> = &lent; + let (delivery, back) = borrowed.reply_in_turn(Counted(drops.clone()), &|| 0); + drop(back); + drop(lent); + let took = receiver.join().unwrap(); + assert_eq!( + drops.load(Ordering::SeqCst), + 1, + "the value is dropped exactly once" + ); + if took { + assert_eq!(delivery, Delivery::Delivered); + } + }); +} From 9a12d149dabdd4026d76b3bafe6c5db53f40e014 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 15:46:47 -0300 Subject: [PATCH 197/208] perf(tasks): a turn that finishes in its first poll needs no DeathFirst guard `DeathFirst` exists for one case: the runtime dropping the actor while a turn is pending, when the turn's caller, the deferred tokens and the queued requests must be answered before the turn's future is destroyed. A turn that completes in its first poll is never pending at an await point, so the run loop now polls it once itself (`poll_caught`, the same `catch_unwind`) with no await point before it returns, and builds `DeathFirst` only for a turn that stays pending (`DeathFirst::pending`, which returns `Pending` without polling the future a second time in the same task poll: its waker is registered). A panic in the first poll drops the future as before (`drop_panicked`, a destructor panic disposed of) and is the handler's panic. The turn bookkeeping (`begin_turn`, `end_turn`, `dropped`) is inlined. Callgrind (micro, burst_raw 51200 + warm-up): 110.4M -> 106.2M (per request 2115 -> 2034; framework code 801 -> 717). seq_raw 117.9M -> 112.2M, seq_default 151.0M -> 148.2M. No concurrency protocol changed: Loom models unchanged and passing (reply_slot_loom 24, job_core_loom 8, registry_turn_loom 3); death_boundary and failpoints suites (runtime drop of a pending turn, panics while building or polling a turn) pass. --- concurrency/src/reply/registry.rs | 3 ++ concurrency/src/reply/turn.rs | 3 ++ concurrency/src/tasks/actor.rs | 75 ++++++++++++++++++++++++++++--- 3 files changed, 74 insertions(+), 7 deletions(-) diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index 27bd107d..5677f8db 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -266,6 +266,7 @@ impl Registry { } /// A reply obligation (`PendingReply` or `ReplyTo`) was dropped unanswered. + #[inline] pub(crate) fn dropped(&self, slot: Arc>) { if slot.is_terminal() { return; @@ -356,11 +357,13 @@ impl Registry { } /// The run loop is about to dispatch a handler. + #[inline] pub(crate) fn begin_turn(&self) { self.turn.begin_turn(); } /// The handler returned (`panicked == false`) or unwound. + #[inline] pub(crate) fn end_turn(&self, panicked: bool) { let (drops, outcome) = self.turn.end_turn(panicked); // Most turns drop no obligation: skip the commit entirely. diff --git a/concurrency/src/reply/turn.rs b/concurrency/src/reply/turn.rs index f5256f7b..0c3068d3 100644 --- a/concurrency/src/reply/turn.rs +++ b/concurrency/src/reply/turn.rs @@ -54,6 +54,7 @@ fn take(drops: &Mutex>) -> Vec { } impl TurnPhase { + #[inline] pub(crate) fn load(&self) -> u8 { self.phase.0.load(Ordering::Acquire) } @@ -94,12 +95,14 @@ impl TurnPhase { /// The run loop is about to dispatch a handler. Only the run loop moves /// the phase out of `RUNNING`, so a plain store suffices. + #[inline] pub(crate) fn begin_turn(&self) { self.phase.0.store(IN_TURN, Ordering::Release); } /// The handler returned (`panicked == false`) or unwound: the recorded /// drops and their outcome. + #[inline] pub(crate) fn end_turn(&self, panicked: bool) -> (Vec, Outcome) { let outcome = if panicked { Outcome::Stopped diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 6c09761e..03f87d2e 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -1806,6 +1806,9 @@ struct DeathFirst<'a, A: Actor, C: Fn()> { rx: &'a mut mpsc::Receiver + Send>>, cleanup: &'a mut crate::actor_cleanup::ActorCleanup, caller: Option<&'a dyn Caller>, + /// The run loop already polled `inner` once in this task poll (it was + /// pending): the next poll returns `Pending` without polling it again. + polled: bool, } impl<'a, A: Actor, C: Fn()> DeathFirst<'a, A, C> { fn new( @@ -1819,8 +1822,40 @@ impl<'a, A: Actor, C: Fn()> DeathFirst<'a, A, C> { rx, cleanup, caller, + polled: false, } } + + /// A turn the run loop polled once and found pending. + fn pending( + inner: Pin + Send + 'a>>, + rx: &'a mut mpsc::Receiver + Send>>, + cleanup: &'a mut crate::actor_cleanup::ActorCleanup, + caller: Option<&'a dyn Caller>, + ) -> Self { + let mut turn = Self::new(inner, rx, cleanup, caller); + turn.polled = true; + turn + } +} + +/// A boxed turn of the lifecycle. +type TurnFuture<'a> = Pin + Send + 'a>>; + +/// Poll `turn` once, catching its panic. +#[inline] +fn poll_caught( + turn: &mut TurnFuture<'_>, + cx: &mut std::task::Context<'_>, +) -> Result, Panic> { + std::panic::catch_unwind(AssertUnwindSafe(|| turn.as_mut().poll(cx))) +} + +/// Drop a turn that panicked; a panic of its destructor is disposed of. +fn drop_panicked(turn: Option>) { + if let Err(secondary) = std::panic::catch_unwind(AssertUnwindSafe(|| drop(turn))) { + crate::mailbox::dispose_panic(secondary); + } } /// A caught panic payload. type Panic = Box; @@ -1834,18 +1869,19 @@ impl Future for DeathFirst<'_, A, C> { cx: &mut std::task::Context<'_>, ) -> std::task::Poll { let this = self.get_mut(); + if std::mem::take(&mut this.polled) { + // Polled (and its waker registered) in this same task poll. + return std::task::Poll::Pending; + } let inner = this.inner.as_mut().expect("polled after completion"); - match std::panic::catch_unwind(AssertUnwindSafe(|| inner.as_mut().poll(cx))) { + match poll_caught(inner, cx) { Ok(std::task::Poll::Pending) => std::task::Poll::Pending, Ok(std::task::Poll::Ready(())) => { this.inner = None; std::task::Poll::Ready(Ok(())) } Err(panic) => { - let inner = this.inner.take(); - if let Err(secondary) = std::panic::catch_unwind(AssertUnwindSafe(|| drop(inner))) { - crate::mailbox::dispose_panic(secondary); - } + drop_panicked(this.inner.take()); std::task::Poll::Ready(Err(panic)) } } @@ -2088,8 +2124,33 @@ async fn run_actor_lifecycle( let mut envelope = envelope; let result = match dispatch(&mut *envelope, &mut *actor, &ctx) { Ok((turn, caller)) => { - let turn = crate::lifecycle_trace::in_actor(ctx.shared.id.raw(), turn); - DeathFirst::new(turn, &mut *rx, &mut *cleanup, caller).await + let mut turn = + Some(crate::lifecycle_trace::in_actor(ctx.shared.id.raw(), turn)); + // Most turns finish in their first poll, which runs + // here, with no await point before it returns: only + // a turn that stays pending is guarded by + // `DeathFirst` against the runtime dropping it. + let first = future::poll_fn(|cx| { + std::task::Poll::Ready(poll_caught( + turn.as_mut().expect("polled once"), + cx, + )) + }) + .await; + match first { + Ok(std::task::Poll::Ready(())) => { + drop(turn); + Ok(()) + } + Ok(std::task::Poll::Pending) => { + let turn = turn.take().expect("pending turn"); + DeathFirst::pending(turn, &mut *rx, &mut *cleanup, caller).await + } + Err(panic) => { + drop_panicked(turn.take()); + Err(panic) + } + } } Err(panic) => Err(panic), }; From 9799c4c882cd8b5a9b3c6fcd6c7d19c5d9f64be9 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 16:02:12 -0300 Subject: [PATCH 198/208] perf(request): the first timer of a request sleeps its timeout, without another clock read `request`, `request_with_timeout` and the protocol `request` helper send with `Deadline::after(timeout)` (one clock read, the deadline fixed at send) and then arm the runtime timer on the first pending poll. Arming computed `deadline - now_ticks()`: a second clock read per request, besides Tokio's own. Now `ReplyReceiver::into_timed_future_after(timeout)` records the timeout (u32 milliseconds, rounded up: the timer wheel's resolution; it fits in the future's existing padding, so `future_layout` budgets hold) and the send-time deadline in `armed`, and the first arm sleeps the timeout. Semantics: the request was sent at most the send-to-poll gap ago, so the timer fires at or after the deadline, never before; expiry still needs `now >= deadline` (`DeferredReplies.tla` `Expire(r)`), and the reply side still decides delivered-vs-expired by its own read of the deadline. A deadline shortened between send and the first poll (`armed != deadline`) re-arms from the clock as before. No linearization point moved; no model change. Callgrind seq_default (micro, 20000): framework instructions per request unchanged within scheduler noise (the saved work is one vDSO clock read, which callgrind counts inside the scheduler's park timing). Validated together with the next commit: fmt, clippy (3 configurations), workspace tests, failpoints + death_boundary, sim 315/315, Loom 24+8+3. --- concurrency/src/reply/mod.rs | 28 +++++++++++++++++++++++++++- concurrency/src/tasks/actor.rs | 6 +++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 3a0f8448..8d6f9871 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -651,12 +651,31 @@ impl ReplyReceiver { let slot = self .slot .take() + first_ms: 0, .expect("ReplyReceiver used after completion"); slot.recv_blocking(&now_ticks) .map_err(|terminal| terminal_error(terminal, &slot)) } /// Non-blocking check: `None` while the request is still open. + /// [`into_timed_future`](Self::into_timed_future) for a request just + /// sent with `Deadline::after(timeout)`: its first timer sleeps + /// `timeout` (in whole milliseconds, rounded up: the runtime timer's + /// resolution) instead of reading the clock again. The request was sent + /// at most the send-to-poll gap ago, so the timer fires at or after the + /// deadline, never before. A deadline shortened meanwhile re-arms from + /// the clock. + #[cfg(feature = "async")] + pub(crate) fn into_timed_future_after(self, timeout: Duration) -> ReplyFuture { + let deadline = self.slot().deadline(); + let first_ms = u32::try_from(timeout.as_nanos().div_ceil(1_000_000)).unwrap_or(0); + ReplyFuture { + armed: deadline, + first_ms, + ..self.into_timed_future() + } + } + pub fn try_recv(&mut self) -> Option> { let slot = self.slot(); let result = slot @@ -698,6 +717,7 @@ pin_project_lite::pin_project! { } } + first_ms: 0, #[cfg(feature = "async")] impl std::fmt::Debug for ReplyFuture { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -712,6 +732,9 @@ impl std::future::Future for ReplyFuture { fn poll( self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>, + // A `request`'s timeout in milliseconds (0: none), with `armed` its + // send-time deadline: the first timer sleeps it (no clock read). + first_ms: u32, ) -> std::task::Poll { use std::task::Poll; let mut this = self.project(); @@ -739,7 +762,10 @@ impl std::future::Future for ReplyFuture { } } if *this.armed != deadline || this.sleep.is_none() { - let wait = Duration::from_nanos(deadline.saturating_sub(now_ticks())); + let wait = match std::mem::take(this.first_ms) { + ms if ms != 0 && *this.armed == deadline => Duration::from_millis(ms.into()), + _ => Duration::from_nanos(deadline.saturating_sub(now_ticks())), + }; this.sleep.set(Some(spawned_rt::tasks::sleep(wait))); *this.armed = deadline; } diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 03f87d2e..fb582f5b 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -795,7 +795,7 @@ impl Context { return result; } }; - let result = rx.into_timed_future().await; + let result = rx.into_timed_future_after(duration).await; observation.finish(&result); result } @@ -1209,7 +1209,7 @@ pub async fn request( return result; } }; - let result = rx.into_timed_future().await; + let result = rx.into_timed_future_after(timeout_duration).await; observation.finish(&result); result } @@ -1363,7 +1363,7 @@ impl ActorRef { return result; } }; - let result = rx.into_timed_future().await; + let result = rx.into_timed_future_after(duration).await; observation.finish(&result); result } From 9f65a06b1bd5f0073c2bdc0c32c4835053895774 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 16:02:12 -0300 Subject: [PATCH 199/208] perf(reply): a request without a deadline holds no reference to its owner's registry Every reply slot cloned the owner's `Arc` at send and dropped it when the slot was freed: two atomic read-modify-writes per request on the registry's reference count, a cache line shared by every sender and by whichever thread frees the slot (in a burst, the producer and the consumer). Removing them measured burst_raw -22.6% -> -10.0% against main (interleaved, 6 reps, cores 12-15, load ~36). The owner was used for three things; each now has another source: - the caller's final reason on `STOPPED`: the registry's close stores its sealed final reason in the slot (`set_stopped_reason`, in the lazily created side state) before the `STOPPED` CAS, and the caller reads it from the slot after seeing `STOPPED` (the same order the registry gave: the reason is sealed before any slot closes); - an undispatched envelope at the owner's death point: mailbox disposal now passes the registry (`Envelope::discharge(replies)`, `SlotHandle::dispose`), so a queued request is still held and answered with the final reason, in both modes and for a never-polled actor; - an awaited raw receiver's deadline (the owner's heap): a request with a deadline keeps its owner, as before. Lifecycle-trace builds keep the owner in every slot (trace events name it), so recorded traces are unchanged. A `SlotHandle` dropped undisposed (only a refused admission, whose caller gets `NoProc` from the send itself and drops the receiver) is closed `STOPPED` at once, as an ownerless slot always was. Hot slot functions (`reply_in_turn`, `poll`, `after`) are marked `#[inline]`: the added code had pushed them out of line. Callgrind (micro): burst_raw 106.2M -> 105.7M (per request 2034 -> 2026); seq_raw 112.2M -> 118.2M and seq_default 148.2M -> 147.7M (scheduler noise). The removed RMWs are contention, not instructions. fmt, clippy (all features, no default features, failpoints), workspace tests, failpoints + death_boundary, sim 315/315, Loom reply_slot 24 / job_core 8 / registry_turn 3: all pass. No linearization point moved. --- concurrency/src/actor_cleanup.rs | 6 ++++ concurrency/src/reply/mod.rs | 33 +++++++++++++++--- concurrency/src/reply/registry.rs | 15 ++++++-- concurrency/src/reply/slot.rs | 52 +++++++++++++++++++++------- concurrency/src/tasks/actor.rs | 18 ++++++---- concurrency/src/threads/actor.rs | 16 +++++---- concurrency/tests/job_core_loom.rs | 1 + concurrency/tests/reply_slot_loom.rs | 1 + docs/design/deferred-replies.md | 14 ++++++-- 9 files changed, 121 insertions(+), 35 deletions(-) diff --git a/concurrency/src/actor_cleanup.rs b/concurrency/src/actor_cleanup.rs index e0f06f5a..5e7a1507 100644 --- a/concurrency/src/actor_cleanup.rs +++ b/concurrency/src/actor_cleanup.rs @@ -183,6 +183,12 @@ impl ActorCleanup { reason } + /// The owner's registry (none for an actor without replies). + #[cfg_attr(not(feature = "async"), allow(dead_code))] + pub(crate) fn replies(&self) -> Option<&crate::reply::registry::Registry> { + self.replies.as_deref() + } + /// Answer a destroyed turn's own caller at the death point (its slot is /// held by the turn's `PendingReply`, not listed in the registry). #[cfg_attr(not(feature = "async"), allow(dead_code))] diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 8d6f9871..401b546b 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -36,6 +36,8 @@ mod sync { pub(crate) use std::sync::{Arc, Arc as JobArc, Condvar, Mutex, MutexGuard, Weak as JobWeak}; pub(crate) use std::thread::yield_now; pub(crate) type SlotOwner = Option>; + /// The owner's final reason, as the slot records it at its close. + pub(crate) type Reason = crate::error::ExitReason; /// Run a slot transition as one step of the lifecycle-trace /// linearization (the identity without the feature). @@ -197,9 +199,12 @@ fn terminal_error(terminal: Terminal, slot: &Slot) -> ActorError { Terminal::Overloaded => ActorError::Overloaded, Terminal::TimedOut => ActorError::RequestTimeout, Terminal::Stopped | Terminal::Abandoned | Terminal::Replied => ActorError::ActorStopped( - slot.owner - .as_ref() - .and_then(|owner| owner.final_reason().cloned()) + slot.stopped_reason() + .or_else(|| { + slot.owner + .as_ref() + .and_then(|owner| owner.final_reason().cloned()) + }) .unwrap_or(crate::error::ExitReason::NoProc), ), } @@ -208,12 +213,20 @@ fn terminal_error(terminal: Terminal, slot: &Slot) -> ActorError { /// A new reply slot for a request with `deadline`, and the caller's receiver. /// /// `owner` is the receiving instance's registry: an awaited raw receiver -/// puts its deadline in that owner's heap instead of arming its own timer. +/// puts its deadline in that owner's heap instead of arming its own timer, +/// so a request with a deadline keeps a reference to it. A request without +/// one takes none (no shared reference count per request): the owner's +/// close records the final reason in the slot, and its envelope is +/// disposed of through the owner's mailbox ([`SlotHandle::dispose`]). pub(crate) fn reply_slot( deadline: Deadline, owner: Option<&Arc>, ) -> (SlotHandle, ReplyReceiver) { - let slot = Arc::new(Slot::with_owner(deadline.ticks(), owner.cloned())); + let keep = deadline != Deadline::Never || cfg!(feature = "lifecycle-trace"); + let slot = Arc::new(Slot::with_owner( + deadline.ticks(), + owner.filter(|_| keep).cloned(), + )); #[cfg(feature = "lifecycle-trace")] if let Some(owner) = owner { crate::lifecycle_trace::reply(owner.owner().raw(), "Send", Arc::as_ptr(&slot) as u64, ""); @@ -244,6 +257,13 @@ impl SlotHandle { pub(crate) fn take(&mut self) -> Option>> { self.0.take() } + /// Undispatched at its owner's death point (mailbox disposal): answered + /// with the final reason, through the owner's registry. + pub(crate) fn dispose(&mut self, owner: Option<&Registry>) { + if let (Some(owner), Some(slot)) = (owner, self.0.take()) { + owner.dispose(slot); + } + } } /// A turn's caller, lent by its envelope for the turn: retained (one @@ -260,6 +280,9 @@ impl Caller for Arc> { } } +/// Dropped undisposed (a refused admission, whose caller is answered +/// `NoProc` by the send itself): a slot that knows its owner is disposed of +/// there; one that does not (no deadline) is closed `STOPPED` now. impl Drop for SlotHandle { fn drop(&mut self) { if let Some(slot) = self.0.take() { diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index 5677f8db..2caaccd3 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -17,8 +17,9 @@ //! earlier; nobody holds a token that could still reply. //! //! The final reason is stored once, at the death point ([`Registry::seal`]), -//! before any slot of this owner becomes `STOPPED`; a caller that reads a -//! `STOPPED` slot reads it from here ([`Registry::final_reason`]). +//! before any slot of this owner becomes `STOPPED`; the close copies it into +//! each slot it closes before the transition, and a caller that reads a +//! `STOPPED` slot reads it from there (a slot does not keep its owner alive). //! //! No rule uses `thread::panicking()`. The phase is one atomic so a turn costs //! two uncontended atomic stores; the turn list is locked only when a drop @@ -39,6 +40,8 @@ use super::turn::{Outcome, TurnPhase, DEAD, EXITING}; pub(crate) trait Closeable: Send + Sync { /// Close without waking; the returned work runs with `run_wake`. fn close_collect(&self) -> Option; + /// Record the owner's final reason before closing. + fn set_stopped_reason(&self, reason: &ExitReason); fn run_wake(&self, wake: Wake); fn notify_parked(&self, wake: &mut Wake); fn terminate(&self, to: Terminal) -> bool; @@ -65,6 +68,9 @@ impl Closeable for Slot { fn close_collect(&self) -> Option { Slot::close_collect(self) } + fn set_stopped_reason(&self, reason: &ExitReason) { + Slot::set_stopped_reason(self, reason.clone()) + } fn run_wake(&self, wake: Wake) { Slot::run_wake(self, wake) } @@ -445,6 +451,11 @@ impl Registry { /// already terminal: this may be the last strong reference to an unread /// reply, whose destructor must not run before every caller is answered. fn close_slot(&self, slot: Arc) -> bool { + // The final reason is sealed before any close: the caller reads it + // from the slot once it sees `STOPPED`. + if let Some(reason) = self.final_reason() { + slot.set_stopped_reason(reason); + } let wake = slot.close_collect(); let closed = wake.is_some(); let mut batch = self.discharge.lock().unwrap_or_else(|p| p.into_inner()); diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index f3298d65..9ded8850 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -189,14 +189,19 @@ impl CreditPool { } } -/// Deferred-token state, created by `defer()` (which installs the pool -/// before its CAS can take a credit): a plain request never allocates it. +/// Rarely needed state, created on demand: by `defer()` (which installs the +/// pool before its CAS can take a credit) and by the owner's close at its +/// death point (the final reason). A plain request never allocates it. +#[derive(Default)] struct Side { /// The owner's pool, where this slot's credits return. - pool: Arc, + pool: Option>, /// Token-side waiters: `ReplyTo::closed()` and caller-gone job /// cancellation. Several may wait at once. watchers: Vec, + /// The owner's final exit reason, stored by its close before the slot + /// becomes `STOPPED`, so the caller reads it from the slot. + reason: Option, } /// What the slot lock guards. Its drop is two `None` checks and a null @@ -363,6 +368,7 @@ impl Slot { /// After a winning CAS `old -> new`: return the credit delta now and, /// when the state became terminal, collect the wake-up, which the caller /// runs after every lock is released. + #[inline] fn after(&self, old: u64, new: u64) -> Wake { let release = credits(old).saturating_sub(credits(new)); let terminal = live(state(old)) && !live(state(new)); @@ -378,8 +384,8 @@ impl Slot { #[inline] fn finish(inner: &mut Inner, release: u64, terminal: bool) -> Wake { if release > 0 { - if let Some(side) = &inner.side { - side.pool.release(release); + if let Some(pool) = inner.side.as_ref().and_then(|side| side.pool.as_ref()) { + pool.release(release); } } if !terminal { @@ -468,6 +474,27 @@ impl Slot { } } + /// The owner's close records its final reason before the `STOPPED` + /// transition can publish it (a caller reads it after seeing `STOPPED`). + pub(crate) fn set_stopped_reason(&self, reason: super::sync::Reason) { + if self.is_terminal() { + return; + } + lock(&self.inner) + .side + .get_or_insert_with(Default::default) + .reason = Some(reason); + } + + /// The final reason the owner's close recorded, if it closed this slot. + #[allow(dead_code)] // the Loom model has no reasons to read + pub(crate) fn stopped_reason(&self) -> Option { + lock(&self.inner) + .side + .as_ref() + .and_then(|side| side.reason.clone()) + } + /// Run the wake-up work [`close_collect`](Self::close_collect) returned. pub(crate) fn run_wake(&self, wake: Wake) { wake.run(); @@ -548,12 +575,11 @@ impl Slot { } // The side state (the pool credits return to) exists before the CAS // can take a credit. - lock(&self.inner).side.get_or_insert_with(|| { - Box::new(Side { - pool: pool.clone(), - watchers: Vec::new(), - }) - }); + lock(&self.inner) + .side + .get_or_insert_with(Default::default) + .pool + .get_or_insert_with(|| pool.clone()); match self.cas(Some(pool), |w| { if state(w) == OPEN && w & TOKEN == 0 { Ok(w | TOKEN | REG) @@ -579,7 +605,7 @@ impl Slot { let pool = lock(&self.inner) .side .as_ref() - .map(|side| side.pool.clone()); + .and_then(|side| side.pool.clone()); match self.cas(pool.as_deref(), |w| { if state(w) != OPEN || w & TOKEN == 0 { Err(Refused::Closed) @@ -636,6 +662,7 @@ impl Slot { /// `OPEN -> REPLIED` (the same linearization point as `OPEN -> WRITING`: /// a receiver reads the value under that lock); a lost CAS hands the /// value back. A reply at or past the deadline takes [`reply`](Self::reply). + #[inline] pub(crate) fn reply_in_turn(&self, value: R, now: &impl Fn() -> u64) -> (Delivery, Option) { let deadline = self.deadline(); if deadline != NEVER && now() >= deadline { @@ -737,6 +764,7 @@ impl Slot { /// at its deadline (`ReplyReceiver` sleeps until it), so the poll itself /// reads no clock. #[cfg_attr(not(feature = "async"), allow(dead_code))] + #[inline] pub(crate) fn poll(&self, waker: &Waker) -> Poll> { if let Some(result) = self.take(state(self.word())) { return Poll::Ready(result); diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index fb582f5b..793345af 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -162,7 +162,7 @@ trait Envelope: Send { /// The owner reached its death point with this envelope undispatched: /// answer its request now (the final reason), before any payload of the /// mailbox is destroyed. No user code runs. - fn discharge(&mut self) {} + fn discharge(&mut self, _replies: Option<&Registry>) {} /// Dispatch once: the turn's future, and the request's caller if any. /// The envelope keeps the request's slot for the whole turn and lends @@ -205,10 +205,11 @@ fn dispatched(payload: &mut Option) -> T { /// after. fn answer_mailbox( rx: &mut mpsc::Receiver + Send>>, + replies: Option<&Registry>, ) -> Vec + Send>> { let mut queued = rx.drain(); for envelope in &mut queued { - envelope.discharge(); + envelope.discharge(replies); } queued } @@ -221,7 +222,8 @@ type StartupPayload = ( CancellationToken, ); fn discharge_startup(payload: &mut StartupPayload) -> Vec> { - erase(answer_mailbox(&mut payload.2)) + let replies = payload.1.shared.replies.clone(); + erase(answer_mailbox(&mut payload.2, Some(&replies))) } fn erase(items: Vec) -> Vec> { items @@ -272,7 +274,8 @@ where observation.reject(); } } - fn discharge(&mut self) { + fn discharge(&mut self, replies: Option<&Registry>) { + self.slot.dispose(replies); drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); } fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { @@ -424,7 +427,8 @@ impl Envelope for InTurnEnvelope { fn message_type(&self) -> &'static str { "InTurn" } - fn discharge(&mut self) { + fn discharge(&mut self, replies: Option<&Registry>) { + self.slot.dispose(replies); drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); } fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { @@ -1897,7 +1901,7 @@ impl Drop for DeathFirst<'_, A, C> { if let Some(caller) = self.caller.take() { self.cleanup.answer_turn(caller.retain()); } - let queued = answer_mailbox(self.rx); + let queued = answer_mailbox(self.rx, self.cleanup.replies()); self.cleanup.discharged(); self.rx.discard_all(queued); if let Err(panic) = std::panic::catch_unwind(AssertUnwindSafe(|| drop(inner))) { @@ -1931,7 +1935,7 @@ impl ExitOnDrop<'_, A, C> { } else { self.cleanup.die(); } - let queued = answer_mailbox(&mut self.rx); + let queued = answer_mailbox(&mut self.rx, self.cleanup.replies()); // Every caller is answered: wake them, then destroy (queued // payloads, then state). A destructor panic is a cleanup failure, // never the reason. diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index d41f9c42..7e80c984 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -101,7 +101,7 @@ trait Envelope: Send { /// The owner reached its death point with this envelope undispatched: /// answer its request now (the final reason), before any payload of the /// mailbox is destroyed. No user code runs. - fn discharge(&mut self) {} + fn discharge(&mut self, _replies: Option<&Registry>) {} fn handle(self: Box, actor: &mut A, ctx: &Context); } @@ -113,12 +113,13 @@ trait Envelope: Send { fn answer_mailbox( rx: &mut mpsc::Receiver>, extra: Option + Send>>, + replies: Option<&Registry>, ) -> Vec> { let mut queued = rx.drain(); queued.extend(extra.map(MailboxItem::Message)); for item in &mut queued { if let MailboxItem::Message(envelope) = item { - envelope.discharge(); + envelope.discharge(replies); } } queued @@ -132,7 +133,8 @@ type StartupPayload = ( CancellationToken, ); fn discharge_startup(payload: &mut StartupPayload) -> Vec> { - erase(answer_mailbox(&mut payload.2, None)) + let replies = payload.1.replies.clone(); + erase(answer_mailbox(&mut payload.2, None, Some(&replies))) } fn erase(items: Vec) -> Vec> { items @@ -177,7 +179,8 @@ where observation.reject(); } } - fn discharge(&mut self) { + fn discharge(&mut self, replies: Option<&Registry>) { + self.slot.dispose(replies); drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); } fn handle(self: Box, actor: &mut A, ctx: &Context) { @@ -309,7 +312,8 @@ struct InTurnEnvelope { } impl Envelope for InTurnEnvelope { - fn discharge(&mut self) { + fn discharge(&mut self, replies: Option<&Registry>) { + self.slot.dispose(replies); drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); } fn handle(self: Box, actor: &mut A, ctx: &Context) { @@ -1321,7 +1325,7 @@ impl ActorRef { // requests (and one received after the stop) are answered, all // before any destructor. cleanup.die(); - let queued = answer_mailbox(&mut rx, discarded); + let queued = answer_mailbox(&mut rx, discarded, Some(&replies)); // Every caller is answered: wake them, then destroy (queued // payloads, then state). cleanup.discharged(); diff --git a/concurrency/tests/job_core_loom.rs b/concurrency/tests/job_core_loom.rs index 08c737a6..7400759f 100644 --- a/concurrency/tests/job_core_loom.rs +++ b/concurrency/tests/job_core_loom.rs @@ -17,6 +17,7 @@ mod sync { pub(crate) use loom::sync::{Arc, Condvar, Mutex, MutexGuard}; pub(crate) use loom::thread::yield_now; pub(crate) type SlotOwner = Option<()>; + pub(crate) type Reason = (); pub(crate) fn section(f: impl FnOnce() -> T) -> T { f() } diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index 48f8e561..2c3067e1 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -12,6 +12,7 @@ mod sync { pub(crate) use loom::sync::{Arc, Mutex, MutexGuard}; pub(crate) use loom::thread::yield_now; pub(crate) type SlotOwner = Option<()>; + pub(crate) type Reason = (); pub(crate) fn section(f: impl FnOnce() -> T) -> T { f() } diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index d7835af1..a3267022 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -94,9 +94,17 @@ inner: one mutex (the slot lock) over waker: the caller's task waker, or the unparker of a thread blocked in recv() (then `parked` is set, so the death point's discharge wakes it before any other waker) - side: boxed, created by defer(): the owner's credit pool and - the token's watchers (ReplyTo::closed, caller-gone job - cancellation); a plain request never allocates it + side: boxed, created on demand: by defer() (the owner's credit + pool and the token's watchers: ReplyTo::closed, + caller-gone job cancellation) and by the owner's close + (its final reason, stored before the STOPPED CAS so the + caller reads it from the slot); a plain request never + allocates it +owner: the owner's registry, kept only by a request with a deadline (an + awaited raw receiver puts it in the owner's heap) and in + lifecycle-trace builds; a plain request takes no shared reference + count. Its envelope, if never dispatched, is answered through the + owner's mailbox disposal at the death point (`discharge(replies)`) the slot lock also orders a shortening against a reply's deadline read ``` From 50d88dc87e6281a4aabc8dab4d85a9b2d67b218a Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 16:04:50 -0300 Subject: [PATCH 200/208] fix(request): place the first-timer fields where 9799c4c8 meant them 9799c4c8 was staged hunk by hunk with zero-context patches, and git placed four of its insertions at the wrong lines of `reply/mod.rs` (inside `ReplyReceiver::recv`, `try_recv`'s doc comment and `ReplyFuture::poll`'s signature): that commit and 9f65a06b do not compile. This moves them to the intended places (`into_timed_future`, `IntoFuture`, the `ReplyFuture` struct and `into_timed_future_after` after `into_timed_future`). The resulting tree is the one validated for 9799c4c8 and 9f65a06b: fmt, clippy (three configurations), workspace tests, failpoints + death_boundary, sim 315/315, Loom 24 + 8 + 3. --- concurrency/src/reply/mod.rs | 46 ++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 401b546b..f899d781 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -651,12 +651,31 @@ impl ReplyReceiver { ReplyFuture { rx: self, armed: NEVER, + first_ms: 0, sleep: None, timer: true, tracked: false, } } + /// [`into_timed_future`](Self::into_timed_future) for a request just + /// sent with `Deadline::after(timeout)`: its first timer sleeps + /// `timeout` (in whole milliseconds, rounded up: the runtime timer's + /// resolution) instead of reading the clock again. The request was sent + /// at most the send-to-poll gap ago, so the timer fires at or after the + /// deadline, never before. A deadline shortened meanwhile re-arms from + /// the clock. + #[cfg(feature = "async")] + pub(crate) fn into_timed_future_after(self, timeout: Duration) -> ReplyFuture { + let deadline = self.slot().deadline(); + let first_ms = u32::try_from(timeout.as_nanos().div_ceil(1_000_000)).unwrap_or(0); + ReplyFuture { + armed: deadline, + first_ms, + ..self.into_timed_future() + } + } + fn slot(&self) -> &Arc> { self.slot .as_ref() @@ -674,31 +693,12 @@ impl ReplyReceiver { let slot = self .slot .take() - first_ms: 0, .expect("ReplyReceiver used after completion"); slot.recv_blocking(&now_ticks) .map_err(|terminal| terminal_error(terminal, &slot)) } /// Non-blocking check: `None` while the request is still open. - /// [`into_timed_future`](Self::into_timed_future) for a request just - /// sent with `Deadline::after(timeout)`: its first timer sleeps - /// `timeout` (in whole milliseconds, rounded up: the runtime timer's - /// resolution) instead of reading the clock again. The request was sent - /// at most the send-to-poll gap ago, so the timer fires at or after the - /// deadline, never before. A deadline shortened meanwhile re-arms from - /// the clock. - #[cfg(feature = "async")] - pub(crate) fn into_timed_future_after(self, timeout: Duration) -> ReplyFuture { - let deadline = self.slot().deadline(); - let first_ms = u32::try_from(timeout.as_nanos().div_ceil(1_000_000)).unwrap_or(0); - ReplyFuture { - armed: deadline, - first_ms, - ..self.into_timed_future() - } - } - pub fn try_recv(&mut self) -> Option> { let slot = self.slot(); let result = slot @@ -717,6 +717,7 @@ impl std::future::IntoFuture for ReplyReceiver { ReplyFuture { rx: self, armed: NEVER, + first_ms: 0, sleep: None, timer: false, tracked: false, @@ -731,6 +732,9 @@ pin_project_lite::pin_project! { pub struct ReplyFuture { rx: ReplyReceiver, armed: u64, + // A `request`'s timeout in milliseconds (0: none), with `armed` its + // send-time deadline: the first timer sleeps it (no clock read). + first_ms: u32, // Arm a runtime timer (`request`/`Response`), rather than putting the // deadline in the owner's heap (raw receivers). timer: bool, @@ -740,7 +744,6 @@ pin_project_lite::pin_project! { } } - first_ms: 0, #[cfg(feature = "async")] impl std::fmt::Debug for ReplyFuture { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -755,9 +758,6 @@ impl std::future::Future for ReplyFuture { fn poll( self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>, - // A `request`'s timeout in milliseconds (0: none), with `armed` its - // send-time deadline: the first timer sleeps it (no clock read). - first_ms: u32, ) -> std::task::Poll { use std::task::Poll; let mut this = self.project(); From 9473fdc4e4c669e0560db37d999e370278f762a8 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 16:07:50 -0300 Subject: [PATCH 201/208] perf(reply): a caller that owns an answered slot alone takes the value without the lock Once the owner's turn has replied and released its (lent) reference, the caller's receiver is usually the slot's only owner. `ReplyFuture::poll` now checks for `REPLIED` and a strong count of one, takes the slot with `Arc::try_unwrap` and moves the value out of it (`Slot::into_reply`, the mutex consumed by value): one CAS instead of the lock, the unlock and the final reference-count decrement. When the slot is still shared (the owner not yet past its turn, a deferred token, a racing weak upgrade), it falls back to the locked take as before. Ordering: the caller read `REPLIED` with acquire after the publishing CAS, which follows the value's write; `try_unwrap` makes the caller the slot's only owner, so nothing else can read or write it. Safe code only. Measured (micro burst_raw, 16 interleaved runs each, cores 12-15, load ~45): 5.108M -> 5.407M requests/s (+5.9%). Callgrind burst_raw 105.7M -> 105.3M (per request 2026 -> 2018). Loom: new `unshared_take_vs_lent_reply_and_release` (the caller's lock-free take racing the owner's in-turn reply through its lent reference and its release: the replied value, exactly once, by either path); reply_slot_loom 25, job_core_loom 8, registry_turn_loom 3 pass. fmt, clippy (three configurations), workspace tests, failpoints + death_boundary, sim 315/315 pass. No linearization point moved. CHANGELOG: one Unreleased bullet for this round's request-path work. --- CHANGELOG.md | 10 ++++++++ concurrency/src/reply/mod.rs | 17 ++++++++++++++ concurrency/src/reply/slot.rs | 12 ++++++++++ concurrency/tests/reply_slot_loom.rs | 35 ++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65b51eb0..f12f8cc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## Unreleased +- Performance (0.8 request path, no API change): the reply slot keeps one + lock for value and waker with lazily created deferral state (no condvar: + a blocking `recv()` parks its thread); the tasks envelope lends its + request slot to the turn instead of cloning it; a turn that finishes in + its first poll skips the runtime-drop guard; `request` arms its first + timer from the timeout (no second clock read); a request without a + deadline holds no reference to the owner's registry (the owner's close + records the final reason in the slot); and a caller that owns an answered + slot alone takes the value without the lock. Guarantees, Loom models and + the TLA+ models are unchanged (no linearization point moved). - `InsertError::kind()` (a `#[non_exhaustive]` `InsertErrorKind`) and `InsertError::into_parts()`: a refused insert's reason and token come together, so a caller answers the token for any reason, including one diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index f899d781..114a2a76 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -762,6 +762,23 @@ impl std::future::Future for ReplyFuture { use std::task::Poll; let mut this = self.project(); loop { + let slot = this.rx.slot(); + // Answered and no longer shared (the owner's turn already + // released its reference, the usual case once the caller gets + // to it): the value moves out without the slot lock. + if slot::state(slot.word()) == slot::REPLIED && Arc::strong_count(slot) == 1 { + let shared = this.rx.slot.take().expect("checked above"); + match Arc::try_unwrap(shared) { + Ok(owned) => { + return Poll::Ready( + owned + .into_reply() + .ok_or(ActorError::ActorStopped(crate::error::ExitReason::NoProc)), + ); + } + Err(shared) => this.rx.slot = Some(shared), + } + } let slot = this.rx.slot(); if let Poll::Ready(result) = slot.poll(cx.waker()) { let result = result.map_err(|terminal| terminal_error(terminal, slot)); diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index 9ded8850..e0789810 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -880,6 +880,18 @@ impl Slot { } } + /// The reply of a `REPLIED` slot the caller now owns alone: moved out + /// without taking the lock (the publishing CAS, read by the caller, + /// ordered the value's write before it, and owning the slot excludes + /// every other access). `None` if it was already taken. + #[allow(dead_code)] // the Loom model takes through the lock + pub(crate) fn into_reply(self) -> Option { + self.inner + .into_inner() + .unwrap_or_else(|p| p.into_inner()) + .value + } + /// The caller dropped its receiver: `OPEN -> ABANDONED`. pub(crate) fn abandon(&self) -> bool { self.terminate(Terminal::Abandoned) diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index 2c3067e1..8e7a12cf 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -693,3 +693,38 @@ fn lent_reply_vs_receiver_last_reference() { } }); } + +/// The caller takes a reply without the slot lock once it owns the slot +/// alone (`Arc::try_unwrap`, then `into_reply`), while the owner's turn +/// replies through its lent reference and drops it: the value is the +/// replied one, taken exactly once, by the lock-free path or the locked one. +#[test] +fn unshared_take_vs_lent_reply_and_release() { + model(|| { + let lent: Arc> = Arc::new(Slot::new(NEVER)); + let mut caller = Some(lent.clone()); + let owner = loom::thread::spawn(move || { + let (delivery, back) = lent.reply_in_turn(6, &|| 0); + drop(lent); + (delivery, back) + }); + let got = loop { + let shared = caller.take().expect("the caller holds its slot"); + if state(shared.word()) == slot::REPLIED && Arc::strong_count(&shared) == 1 { + match Arc::try_unwrap(shared) { + Ok(owned) => break owned.into_reply(), + Err(shared) => caller = Some(shared), + } + } else { + caller = Some(shared); + } + let shared = caller.as_ref().expect("put back"); + if let Some(result) = shared.try_take(&|| 0) { + break result.ok(); + } + loom::thread::yield_now(); + }; + assert_eq!(got, Some(6)); + assert_eq!(owner.join().unwrap(), (Delivery::Delivered, None)); + }); +} From f66429c0dafd5af4cc5a419167de23c0517b158b Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 16:39:57 -0300 Subject: [PATCH 202/208] perf(reply): the slot holds no owner; raw deadlines go to the owner's heap at send The gate's burst_async stayed ~20% behind main although its instructions per request were already lower (4384 vs 4464, callgrind): the cost was the allocator. With 0.7's 48-byte `Ack` reply, the `Arc>` was 136 bytes, a 144-byte chunk, just above glibc's fast-bin limit (128); main's oneshot is 112. In a burst, frees overflow the per-thread cache and every slot went through the unsorted bin and consolidation, and the slower allocator turned into parking (5-10x the voluntary context switches). Raising `glibc.malloc.mxfast` to 160 confirmed it (cand 1.42M -> 2.01M/s, main unchanged). The slot is now 16 bytes smaller (120 -> 104 for `Ack`): - `parked` became the word bit `PARKED` (set under the slot lock before a blocking receiver stores its unparker; the terminal transition reads it from the word it installed, so a racing CAS retried with it); - the slot keeps no owner outside lifecycle-trace builds (`SlotOwner = ()`). The final reason already came from the slot; an undispatched envelope is disposed of through the mailbox; and the one remaining use, a raw receiver awaited with a deadline, is served at send: the public `send_request` with a deadline puts it in the owner's heap then and sets the word bit `TRACKED`, so the awaited receiver waits for the heap's expiry without an owner. `request`, `request_with_timeout` and the `request` helper send through `send_request_with(.., raw = false)` / `Receiver::send_request_timed` and arm their own timer as before. Behaviour (documented in semantics.md R3, deferred-replies.md, CHANGELOG): a raw request's deadline is registered at send instead of at its receiver's first pending poll, so a receiver never awaited expires on time too. Expiry still needs `now >= deadline`; no linearization point moved. `shutdown_publication` gates the producer's first large allocation after admission; its loop now sends without a deadline (a deadline's heap entry would allocate first) and still exercises the mailbox block allocation. Measured (gate harness bins, interleaved, cores 12-15, load ~40, 5 runs): burst_async main 2490k, previous head 2290k, this 4386k req/s; rpc_async main 235k, previous 254k, this 248k. Callgrind micro burst_raw 105.3M -> 104.1M (per request 2018 -> 1995), seq_default 148.1M -> 130.7M. fmt, clippy (all features, no default features, failpoints), workspace tests, failpoints + death_boundary, sim 315/315, Loom reply_slot 25 / job_core 8 / registry_turn 3, lifecycle traces 93/93 accepted with 90 negative-control counterexamples, model-sync events/models/paths: pass. --- CHANGELOG.md | 11 +++- concurrency/src/reply/mod.rs | 74 +++++++++++------------ concurrency/src/reply/slot.rs | 52 ++++++++++++---- concurrency/src/tasks/actor.rs | 72 +++++++++++++++++++--- concurrency/src/threads/actor.rs | 72 +++++++++++++++++++--- concurrency/tests/shutdown_publication.rs | 7 ++- docs/design/deferred-replies.md | 14 +++-- docs/design/semantics.md | 10 +-- 8 files changed, 234 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f12f8cc1..622d3b11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,16 @@ its first poll skips the runtime-drop guard; `request` arms its first timer from the timeout (no second clock read); a request without a deadline holds no reference to the owner's registry (the owner's close - records the final reason in the slot); and a caller that owns an answered - slot alone takes the value without the lock. Guarantees, Loom models and + records the final reason in the slot); a caller that owns an answered + slot alone takes the value without the lock; and the slot no longer + holds its owner at all (two flags became word bits), which keeps it in + glibc's fast bins for typical reply sizes. Guarantees, Loom models and the TLA+ models are unchanged (no linearization point moved). +- **Behaviour (0.8)**: `send_request(msg, deadline)` puts a deadline in the + owner's deadline heap when the request is sent (it was when its receiver + was first awaited and pending): a raw receiver that is never awaited now + expires on time too. `request`/`request_with_timeout`/`Response` still + arm their own runtime timer. - `InsertError::kind()` (a `#[non_exhaustive]` `InsertErrorKind`) and `InsertError::into_parts()`: a refused insert's reason and token come together, so a caller answers the token for any reason, including one diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 114a2a76..235baa4d 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -35,7 +35,12 @@ mod sync { pub(crate) use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, AtomicUsize, Ordering}; pub(crate) use std::sync::{Arc, Arc as JobArc, Condvar, Mutex, MutexGuard, Weak as JobWeak}; pub(crate) use std::thread::yield_now; + /// Lifecycle-trace builds keep the owner in each slot (its events name + /// it); other builds keep nothing, so a slot stays small. + #[cfg(feature = "lifecycle-trace")] pub(crate) type SlotOwner = Option>; + #[cfg(not(feature = "lifecycle-trace"))] + pub(crate) type SlotOwner = (); /// The owner's final reason, as the slot records it at its close. pub(crate) type Reason = crate::error::ExitReason; @@ -200,11 +205,6 @@ fn terminal_error(terminal: Terminal, slot: &Slot) -> ActorError { Terminal::TimedOut => ActorError::RequestTimeout, Terminal::Stopped | Terminal::Abandoned | Terminal::Replied => ActorError::ActorStopped( slot.stopped_reason() - .or_else(|| { - slot.owner - .as_ref() - .and_then(|owner| owner.final_reason().cloned()) - }) .unwrap_or(crate::error::ExitReason::NoProc), ), } @@ -212,21 +212,28 @@ fn terminal_error(terminal: Terminal, slot: &Slot) -> ActorError { /// A new reply slot for a request with `deadline`, and the caller's receiver. /// -/// `owner` is the receiving instance's registry: an awaited raw receiver -/// puts its deadline in that owner's heap instead of arming its own timer, -/// so a request with a deadline keeps a reference to it. A request without -/// one takes none (no shared reference count per request): the owner's -/// close records the final reason in the slot, and its envelope is -/// disposed of through the owner's mailbox ([`SlotHandle::dispose`]). +/// `owner` is the receiving instance's registry. The slot takes no +/// reference to it (no shared reference count per request): the owner's +/// close records the final reason in the slot, and an undispatched +/// envelope is disposed of through the owner's mailbox +/// ([`SlotHandle::dispose`]). A `raw` request with a deadline (the public +/// `send_request`) puts its deadline in the owner's heap now, so its +/// receiver, awaited directly, needs no runtime timer and no owner; the +/// `request` paths arm their own timer and pass `raw = false`. pub(crate) fn reply_slot( deadline: Deadline, owner: Option<&Arc>, + raw: bool, ) -> (SlotHandle, ReplyReceiver) { - let keep = deadline != Deadline::Never || cfg!(feature = "lifecycle-trace"); - let slot = Arc::new(Slot::with_owner( - deadline.ticks(), - owner.filter(|_| keep).cloned(), - )); + #[cfg(feature = "lifecycle-trace")] + let slot_owner = owner.cloned(); + #[cfg(not(feature = "lifecycle-trace"))] + let slot_owner = (); + let slot = Arc::new(Slot::with_owner(deadline.ticks(), slot_owner)); + if let (true, Deadline::At(_), Some(owner)) = (raw, deadline, owner) { + slot.mark_tracked(); + owner.track_deadline(&slot); + } #[cfg(feature = "lifecycle-trace")] if let Some(owner) = owner { crate::lifecycle_trace::reply(owner.owner().raw(), "Send", Arc::as_ptr(&slot) as u64, ""); @@ -281,17 +288,17 @@ impl Caller for Arc> { } /// Dropped undisposed (a refused admission, whose caller is answered -/// `NoProc` by the send itself): a slot that knows its owner is disposed of -/// there; one that does not (no deadline) is closed `STOPPED` now. +/// `NoProc` by the send itself): closed `STOPPED` now (trace builds, whose +/// slots know their owner, dispose of it there). impl Drop for SlotHandle { fn drop(&mut self) { if let Some(slot) = self.0.take() { - match slot.owner.clone() { - Some(owner) => owner.dispose(slot), - None => { - slot.close(); - } + #[cfg(feature = "lifecycle-trace")] + if let Some(owner) = slot.owner.clone() { + owner.dispose(slot); + return; } + slot.close(); } } } @@ -654,7 +661,6 @@ impl ReplyReceiver { first_ms: 0, sleep: None, timer: true, - tracked: false, } } @@ -720,7 +726,6 @@ impl std::future::IntoFuture for ReplyReceiver { first_ms: 0, sleep: None, timer: false, - tracked: false, } } } @@ -738,7 +743,6 @@ pin_project_lite::pin_project! { // Arm a runtime timer (`request`/`Response`), rather than putting the // deadline in the owner's heap (raw receivers). timer: bool, - tracked: bool, #[pin] sleep: Option, } @@ -789,17 +793,11 @@ impl std::future::Future for ReplyFuture { if deadline == NEVER { return Poll::Pending; } - // A raw receiver registers its deadline once in the owner's - // deadline heap; the service expires the slot (waking this - // future) without a per-request runtime timer. - if !*this.timer { - if let Some(owner) = &slot.owner { - if !*this.tracked { - owner.track_deadline(slot); - *this.tracked = true; - } - return Poll::Pending; - } + // A raw request put its deadline in the owner's heap at send; + // the service expires the slot (waking this future) without a + // per-request runtime timer. + if !*this.timer && slot.word() & slot::TRACKED != 0 { + return Poll::Pending; } if *this.armed != deadline || this.sleep.is_none() { let wait = match std::mem::take(this.first_ms) { @@ -835,7 +833,7 @@ impl std::future::Future for ReplyFuture { pub(crate) fn reply_pair( deadline: Deadline, ) -> (PendingReply<'static, R>, ReplyReceiver) { - let (mut handle, receiver) = reply_slot(deadline, None); + let (mut handle, receiver) = reply_slot(deadline, None, false); let reply = PendingReply::new(handle.take(), Registry::detached()); (reply, receiver) } diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index e0789810..a0182ad2 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -2,7 +2,7 @@ //! (docs/design/deferred-replies.md §1, "Slot synchronization protocol"). //! //! ```text -//! word: state:4 | TOKEN:1 | REG:1 | JOBS:16 +//! word: state:4 | TOKEN:1 | REG:1 | JOBS:16 | PARKED:1 //! credits(word) = max(JOBS, TOKEN and state is live ? 1 : 0) //! ``` //! @@ -51,6 +51,17 @@ const JOB_ONE: u64 = 1 << JOBS_SHIFT; /// this value at configuration so the field cannot overflow. pub(crate) const MAX_JOBS: u64 = 0xFFFF; const JOBS_MASK: u64 = MAX_JOBS << JOBS_SHIFT; +/// The registered waker unparks a thread blocked in `recv_blocking` (no +/// user code): the death point's discharge wakes it before any other +/// waker. Set under the slot lock, before the waker is stored; no +/// transition reads or clears it otherwise (`with_state`, `JOBS` arithmetic +/// and `credits` leave it alone). A word bit, not a field: it keeps the +/// slot within the allocator's small size classes. +pub(crate) const PARKED: u64 = 1 << (JOBS_SHIFT + 16); +/// The deadline was put in the owner's heap at send (a raw request with a +/// deadline): an awaited receiver waits for the heap's expiry instead of +/// arming its own timer. Set before the slot is shared; never changes. +pub(crate) const TRACKED: u64 = PARKED << 1; /// Deadline value meaning "never" (`Deadline::Never`). pub(crate) const NEVER: u64 = u64::MAX; @@ -211,11 +222,8 @@ struct Inner { /// publishes `REPLIED`; taken by the receiver after it reads `REPLIED`. value: Option, /// The caller's receiver: an async task's waker, or the unparker of a - /// thread blocked in `recv_blocking` (then `parked` is set). + /// thread blocked in `recv_blocking` (then the word has `PARKED`). waker: Option, - /// The registered waker is a parked thread's (no user code): the death - /// point's discharge wakes it before any other waker. - parked: bool, side: Option>, } @@ -276,6 +284,7 @@ impl Wake { impl Slot { /// A fresh `OPEN` slot whose deadline is `deadline` ticks (`NEVER` for none). #[allow(dead_code)] // the Loom model and unit tests build ownerless slots + #[allow(clippy::unit_arg)] // the owner is `()` outside trace builds pub(crate) fn new(deadline: u64) -> Self { Self::with_owner(deadline, Default::default()) } @@ -288,7 +297,6 @@ impl Slot { inner: Mutex::new(Inner { value: None, waker: None, - parked: false, side: None, }), } @@ -376,13 +384,13 @@ impl Slot { return Wake::NONE; } let mut inner = lock(&self.inner); - Self::finish(&mut inner, release, terminal) + Self::finish(&mut inner, release, terminal, new) } /// Under the slot lock: return `release` credits (an atomic, no user /// code) and, for a terminal transition, take the wake-up. #[inline] - fn finish(inner: &mut Inner, release: u64, terminal: bool) -> Wake { + fn finish(inner: &mut Inner, release: u64, terminal: bool, new: u64) -> Wake { if release > 0 { if let Some(pool) = inner.side.as_ref().and_then(|side| side.pool.as_ref()) { pool.release(release); @@ -393,7 +401,9 @@ impl Slot { } Wake { waker: inner.waker.take(), - parked: inner.parked, + // `PARKED` is set under this lock before a thread stores its + // waker, so a CAS that raced it failed and retried with it. + parked: new & PARKED != 0, watchers: match &mut inner.side { Some(side) => std::mem::take(&mut side.watchers), None => Vec::new(), @@ -486,6 +496,12 @@ impl Slot { .reason = Some(reason); } + /// Mark the deadline as tracked by the owner's heap (before sharing). + #[allow(dead_code)] // the Loom model has no heap + pub(crate) fn mark_tracked(&self) { + self.word.fetch_or(TRACKED, Ordering::Release); + } + /// The final reason the owner's close recorded, if it closed this slot. #[allow(dead_code)] // the Loom model has no reasons to read pub(crate) fn stopped_reason(&self) -> Option { @@ -678,8 +694,12 @@ impl Slot { } }) { Ok((old, new)) => { - let wake = - Self::finish(&mut inner, credits(old).saturating_sub(credits(new)), true); + let wake = Self::finish( + &mut inner, + credits(old).saturating_sub(credits(new)), + true, + new, + ); drop(inner); wake.run(); (Delivery::Delivered, None) @@ -739,7 +759,12 @@ impl Slot { (Delivery::Delivered, None, REPLIED) }; let (old, new) = self.resolve_writing(to); - let wake = Self::finish(&mut inner, credits(old).saturating_sub(credits(new)), true); + let wake = Self::finish( + &mut inner, + credits(old).saturating_sub(credits(new)), + true, + new, + ); drop(inner); wake.run(); (delivery, unsent) @@ -866,13 +891,14 @@ impl Slot { if !live(state(self.word())) { continue; } + // Before the waker is stored (see `PARKED`). + self.word.fetch_or(PARKED, Ordering::AcqRel); // The thread's own unparker (no user code to clone or drop). super::sync::with_thread_waker(|waker| { if !inner.waker.as_ref().is_some_and(|w| w.will_wake(waker)) { inner.waker = Some(waker.clone()); } }); - inner.parked = true; } super::sync::parked(); // A stale unpark or a spurious return only loops. diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 793345af..a67b12cb 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -756,7 +756,23 @@ impl Context { M: Message, { let deadline = deadline.into(); - let (slot, rx) = reply_slot(deadline, Some(&self.shared.replies)); + self.send_request_with(msg, deadline, true) + } + + /// `send_request`; `raw` puts a deadline in the owner's heap now (the + /// receiver may be awaited directly). The `request` paths arm their own + /// timer and pass `false`. + pub(crate) fn send_request_with( + &self, + msg: M, + deadline: Deadline, + raw: bool, + ) -> Result, ActorError> + where + A: Handler, + M: Message, + { + let (slot, rx) = reply_slot(deadline, Some(&self.shared.replies), raw); enqueue( &self.shared.sender, &self.shared.mailbox, @@ -791,7 +807,7 @@ impl Context { // Bind the receiver before awaiting it: awaiting inside a `match` on // the `Result` keeps the scrutinee's storage alive beside the awaited // receiver and doubles this future's size. - let rx = match self.send_request(msg, Deadline::after(duration)) { + let rx = match self.send_request_with(msg, Deadline::after(duration), false) { Ok(rx) => rx, Err(error) => { let result = Err(error); @@ -1168,6 +1184,14 @@ where ) -> Result, ActorError> { Context::send_request(self, msg, deadline) } + + fn send_request_timed( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + Context::send_request_with(self, msg, deadline, false) + } } // --------------------------------------------------------------------------- @@ -1189,6 +1213,16 @@ pub trait Receiver: Send + Sync + crate::__private::Sealed { msg: M, deadline: Deadline, ) -> Result, ActorError>; + /// `send_request` for the `request` helper, which arms its own timer: + /// the deadline does not go to the owner's heap. + #[doc(hidden)] + fn send_request_timed( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + self.send_request(msg, deadline) + } } /// Type-erased reference for sending a single message type. @@ -1205,7 +1239,7 @@ pub async fn request( let observation = crate::instrumentation::RequestObservation::new(); // The deadline is computed before enqueueing (send time, as OTP's call). // Bind the receiver before awaiting it (see `ActorRef::request_with_timeout`). - let rx = match recipient.send_request(msg, Deadline::after(timeout_duration)) { + let rx = match recipient.send_request_timed(msg, Deadline::after(timeout_duration)) { Ok(rx) => rx, Err(error) => { let result = Err(error); @@ -1299,7 +1333,23 @@ impl ActorRef { M: Message, { let deadline = deadline.into(); - let (slot, rx) = reply_slot(deadline, Some(&self.shared.replies)); + self.send_request_with(msg, deadline, true) + } + + /// `send_request`; `raw` puts a deadline in the owner's heap now (the + /// receiver may be awaited directly). The `request` paths arm their own + /// timer and pass `false`. + pub(crate) fn send_request_with( + &self, + msg: M, + deadline: Deadline, + raw: bool, + ) -> Result, ActorError> + where + A: Handler, + M: Message, + { + let (slot, rx) = reply_slot(deadline, Some(&self.shared.replies), raw); enqueue( &self.shared.sender, &self.shared.mailbox, @@ -1328,7 +1378,7 @@ impl ActorRef { R: Send + 'static, F: FnOnce(&mut A, &Context) -> R + Send + 'static, { - let (slot, rx) = reply_slot(Deadline::Never, Some(&self.shared.replies)); + let (slot, rx) = reply_slot(Deadline::Never, Some(&self.shared.replies), false); let envelope: Box + Send> = Box::new(InTurnEnvelope { f: Some(Box::new(f)), slot, @@ -1359,7 +1409,7 @@ impl ActorRef { // Bind the receiver before awaiting it: awaiting inside a `match` on // the `Result` keeps the scrutinee's storage alive beside the awaited // receiver and doubles this future's size. - let rx = match self.send_request(msg, Deadline::after(duration)) { + let rx = match self.send_request_with(msg, Deadline::after(duration), false) { Ok(rx) => rx, Err(error) => { let result = Err(error); @@ -1479,6 +1529,14 @@ where ) -> Result, ActorError> { ActorRef::send_request(self, msg, deadline) } + + fn send_request_timed( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + ActorRef::send_request_with(self, msg, deadline, false) + } } // --------------------------------------------------------------------------- @@ -4575,7 +4633,7 @@ mod idle_stop_regression { // Start and park the actual actor loop, without allowing another // executor task to run it between admission and stop below. assert!(futures::poll!(actor.as_mut()).is_pending()); - let (slot, response) = reply_slot(crate::reply::Deadline::Never, None); + let (slot, response) = reply_slot(crate::reply::Deadline::Never, None, false); tx.send(Box::new(MessageEnvelope { msg: Some(Increment), slot, diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 7e80c984..18edd00f 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -608,7 +608,23 @@ impl Context { M: Message, { let deadline = deadline.into(); - let (slot, rx) = reply_slot(deadline, Some(&self.replies)); + self.send_request_with(msg, deadline, true) + } + + /// `send_request`; `raw` puts a deadline in the owner's heap now (the + /// receiver may be awaited directly). The `request` paths arm their own + /// timer and pass `false`. + pub(crate) fn send_request_with( + &self, + msg: M, + deadline: Deadline, + raw: bool, + ) -> Result, ActorError> + where + A: Handler, + M: Message, + { + let (slot, rx) = reply_slot(deadline, Some(&self.replies), raw); enqueue(&self.sender, &self.mailbox, &self.replies, msg, slot)?; Ok(rx) } @@ -633,7 +649,7 @@ impl Context { M: Message, { let observation = crate::instrumentation::RequestObservation::new(); - let result = match self.send_request(msg, Deadline::after(duration)) { + let result = match self.send_request_with(msg, Deadline::after(duration), false) { Err(error) => Err(error), Ok(rx) => rx.recv(), }; @@ -940,6 +956,14 @@ where ) -> Result, ActorError> { Context::send_request(self, msg, deadline) } + + fn send_request_timed( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + Context::send_request_with(self, msg, deadline, false) + } } // --------------------------------------------------------------------------- @@ -961,6 +985,16 @@ pub trait Receiver: Send + Sync + crate::__private::Sealed { msg: M, deadline: Deadline, ) -> Result, ActorError>; + /// `send_request` for the `request` helper, which arms its own timer: + /// the deadline does not go to the owner's heap. + #[doc(hidden)] + fn send_request_timed( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + self.send_request(msg, deadline) + } } /// Type-erased reference for sending a single message type. @@ -973,7 +1007,7 @@ pub fn request( timeout: Duration, ) -> Result { let observation = crate::instrumentation::RequestObservation::new(); - let result = match recipient.send_request(msg, Deadline::after(timeout)) { + let result = match recipient.send_request_timed(msg, Deadline::after(timeout)) { Err(error) => Err(error), Ok(rx) => rx.recv(), }; @@ -1070,7 +1104,23 @@ impl ActorRef { M: Message, { let deadline = deadline.into(); - let (slot, rx) = reply_slot(deadline, Some(&self.replies)); + self.send_request_with(msg, deadline, true) + } + + /// `send_request`; `raw` puts a deadline in the owner's heap now (the + /// receiver may be awaited directly). The `request` paths arm their own + /// timer and pass `false`. + pub(crate) fn send_request_with( + &self, + msg: M, + deadline: Deadline, + raw: bool, + ) -> Result, ActorError> + where + A: Handler, + M: Message, + { + let (slot, rx) = reply_slot(deadline, Some(&self.replies), raw); enqueue(&self.sender, &self.mailbox, &self.replies, msg, slot)?; Ok(rx) } @@ -1092,7 +1142,7 @@ impl ActorRef { R: Send + 'static, F: FnOnce(&mut A, &Context) -> R + Send + 'static, { - let (slot, rx) = reply_slot(Deadline::Never, Some(&self.replies)); + let (slot, rx) = reply_slot(Deadline::Never, Some(&self.replies), false); let envelope: Box + Send> = Box::new(InTurnEnvelope { f: Box::new(f), slot, @@ -1118,7 +1168,7 @@ impl ActorRef { M: Message, { let observation = crate::instrumentation::RequestObservation::new(); - let result = match self.send_request(msg, Deadline::after(duration)) { + let result = match self.send_request_with(msg, Deadline::after(duration), false) { Err(error) => Err(error), Ok(rx) => rx.recv(), }; @@ -1211,6 +1261,14 @@ where ) -> Result, ActorError> { ActorRef::send_request(self, msg, deadline) } + + fn send_request_timed( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + ActorRef::send_request_with(self, msg, deadline, false) + } } // --------------------------------------------------------------------------- @@ -2648,7 +2706,7 @@ mod idle_stop_regression { drop(counter); }); entered.wait(); - let (slot, response) = reply_slot(crate::reply::Deadline::Never, None); + let (slot, response) = reply_slot(crate::reply::Deadline::Never, None, false); tx.send(MailboxItem::Message(Box::new(MessageEnvelope { msg: Increment(events.clone(), monitors), slot, diff --git a/concurrency/tests/shutdown_publication.rs b/concurrency/tests/shutdown_publication.rs index 1aaf4c40..80550c99 100644 --- a/concurrency/tests/shutdown_publication.rs +++ b/concurrency/tests/shutdown_publication.rs @@ -123,8 +123,11 @@ fn admitted_request_racing_shutdown_drops_its_reply_before_join_completes() { let producer = std::thread::spawn(move || { PRODUCER.with(|p| p.set(true)); for _ in 0..1024 { - let reply = sender - .send_request(Command::Work, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT); + // No deadline: a raw request with one puts it in the owner's + // deadline heap at send, whose growth would trip the gate + // before the mailbox send this test holds. + let reply = + sender.send_request(Command::Work, spawned_concurrency::Deadline::Never); if PAUSED.load(Ordering::SeqCst) { return reply; } diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index a3267022..a574c97d 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -100,11 +100,15 @@ inner: one mutex (the slot lock) over (its final reason, stored before the STOPPED CAS so the caller reads it from the slot); a plain request never allocates it -owner: the owner's registry, kept only by a request with a deadline (an - awaited raw receiver puts it in the owner's heap) and in - lifecycle-trace builds; a plain request takes no shared reference - count. Its envelope, if never dispatched, is answered through the - owner's mailbox disposal at the death point (`discharge(replies)`) +owner: none outside lifecycle-trace builds (whose events name it): a + request takes no shared reference count on its owner. A raw + request with a deadline (`send_request`) puts the deadline in the + owner's heap at send and sets TRACKED in the word (an awaited + receiver then needs no timer and no owner); an undispatched + envelope is answered through the owner's mailbox disposal at the + death point (`discharge(replies)`). PARKED (bit 22) and TRACKED + (bit 23) are word bits no transition reads or clears; as bits, + they keep the slot inside the allocator's small size classes the slot lock also orders a shortening against a reply's deadline read ``` diff --git a/docs/design/semantics.md b/docs/design/semantics.md index 4db3d3b8..4567a594 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -146,10 +146,12 @@ enqueue. threads: the call blocks. Expired requests are still dispatched (OTP); a late reply is dropped (`Delivery::Expired`). Which clock expires a waiting caller is part of the contract: `request`, `request_with_timeout` and `Response::from(rx)` arm the runtime's timer, so -they follow paused Tokio time; a `ReplyReceiver` awaited directly -(`send_request(..)?.await`) puts its deadline in the owner's deadline heap -instead, serviced on the real clock (the simulated one under `spawned_sim`), -not paused Tokio time. The reason is cost: a raw receiver is the building +they follow paused Tokio time; a request sent with `send_request` and a +deadline puts that deadline in the owner's deadline heap when it is sent, +and its `ReplyReceiver` awaited directly (`send_request(..)?.await`) waits +for that heap's expiry, serviced on the real clock (the simulated one under +`spawned_sim`), not paused Tokio time. (A receiver never awaited expires +then too.) The reason is cost: a raw receiver is the building block for fan-out and needs no runtime timer per request. Tests that pause Tokio time wait through `request` or `Response::from`. Either way a shortened deadline (`ReplyTo::shorten_deadline`) wakes the waiting caller. From 30178be1ed32307923dbca0435b1d4cdc2270967 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 14:22:33 -0300 Subject: [PATCH 203/208] test: Sync bounds and 0.8 fixtures for the D1-D3 port onto the 0.8 candidate --- concurrency/tests/deferred_sim.rs | 2 +- concurrency/tests/exit_signals_sim.rs | 2 +- concurrency/tests/sim.rs | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/concurrency/tests/deferred_sim.rs b/concurrency/tests/deferred_sim.rs index a6967974..c3b19d47 100644 --- a/concurrency/tests/deferred_sim.rs +++ b/concurrency/tests/deferred_sim.rs @@ -303,7 +303,7 @@ const STRATEGIES: [(&str, usize); 2] = [("pct:d=3", 300), ("dfs:bound=3,depth=30 fn check(name: &str, body: F) where - F: Fn(u64) -> Fut, + F: Fn(u64) -> Fut + std::marker::Sync, Fut: Future, { for (strategy, budget) in STRATEGIES { diff --git a/concurrency/tests/exit_signals_sim.rs b/concurrency/tests/exit_signals_sim.rs index 9e2564af..d821626b 100644 --- a/concurrency/tests/exit_signals_sim.rs +++ b/concurrency/tests/exit_signals_sim.rs @@ -539,7 +539,7 @@ const DFS_SEEDS: u64 = 12; fn check(name: &str, body: F) where - F: Fn(u64) -> Fut, + F: Fn(u64) -> Fut + Sync, Fut: Future, { OUTCOMES.with(|o| o.borrow_mut().clear()); diff --git a/concurrency/tests/sim.rs b/concurrency/tests/sim.rs index 73686011..be60fab4 100644 --- a/concurrency/tests/sim.rs +++ b/concurrency/tests/sim.rs @@ -370,7 +370,8 @@ fn blocking_backend_actor_fails_the_run() { let log = Log::default(); let node = Node { id: 1, - crash_after: None, + fail_start_after: None, + trap: false, log, } .start_with_backend(spawned_concurrency::tasks::Backend::Blocking); From 291f4ae3b54d605046ddbee8a7b57cd290f5d0bf Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 14:25:32 -0300 Subject: [PATCH 204/208] feat(sim): native jobs (pipe_blocking) run inline as one simulator step on the 0.8 port; outcomes keyed by test in exit_signals_sim --- concurrency/src/reply/job_core.rs | 12 ++++++++++++ concurrency/src/tasks/jobs.rs | 21 ++++++++++++++------- concurrency/tests/exit_signals_sim.rs | 26 ++++++++++++++++++-------- 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/concurrency/src/reply/job_core.rs b/concurrency/src/reply/job_core.rs index d4d3ebfd..02bfc837 100644 --- a/concurrency/src/reply/job_core.rs +++ b/concurrency/src/reply/job_core.rs @@ -229,6 +229,18 @@ impl JobCore { inner.emit_credits -= 1; return Ok(()); } + // A blocking job runs inline on the simulation thread (D2): the + // credit it waits for comes back only when the owner dispatches + // an emit, which cannot happen while this poll blocks. + #[cfg(all(spawned_sim, feature = "async"))] + if spawned_rt::tasks::Handle::try_current().is_ok() { + drop(inner); + panic!( + "spawned_sim: a blocking job ran out of emit credits inline on the \ + simulation thread (the owner cannot dispatch its emits until the job's \ + step ends); raise JobOptions emit credits or emit less per job" + ); + } inner = self.emitters.wait(inner).unwrap_or_else(|p| p.into_inner()); } } diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs index 3ef94277..16f0b2da 100644 --- a/concurrency/src/tasks/jobs.rs +++ b/concurrency/src/tasks/jobs.rs @@ -255,10 +255,15 @@ enum RunnerPool { /// current one. #[cfg(not(spawned_sim))] Tokio(spawned_rt::tasks::Handle), - /// Outside a Tokio runtime, and under the simulator: an owned thread, - /// spawned at reservation and parked until the job starts (dropping the - /// sender unused lets it exit). + /// Outside a Tokio runtime: an owned thread, spawned at reservation and + /// parked until the job starts (dropping the sender unused lets it exit). + #[cfg(not(spawned_sim))] Thread(std::sync::mpsc::Sender>), + /// Under the simulator: `spawned_rt::tasks::spawn_blocking`, which runs + /// the body inline as one step of a simulated task (determinism D2), so + /// its clock reads, emits and result are part of the seeded schedule. + #[cfg(spawned_sim)] + Sim, } impl NativeRunner { @@ -275,9 +280,12 @@ impl NativeRunner { RunnerPool::Tokio(handle) => drop(handle.spawn_blocking(task)), // The thread is parked on `recv` until this send (or the // sender's drop); it cannot have exited. + #[cfg(not(spawned_sim))] RunnerPool::Thread(start) => { let _ = start.send(Box::new(task)); } + #[cfg(spawned_sim)] + RunnerPool::Sim => drop(spawned_rt::tasks::spawn_blocking(task)), } } } @@ -596,12 +604,11 @@ impl Context { None => spawned_rt::tasks::Handle::try_current().ok(), }; #[cfg(spawned_sim)] - let handle: Option<()> = None; + let pool = RunnerPool::Sim; + #[cfg(not(spawned_sim))] let pool = match handle { - #[cfg(not(spawned_sim))] Some(handle) => RunnerPool::Tokio(handle), - #[allow(unreachable_patterns)] - _ => { + None => { let (start, started) = std::sync::mpsc::channel::>(); std::thread::Builder::new() .name("spawned-job".into()) diff --git a/concurrency/tests/exit_signals_sim.rs b/concurrency/tests/exit_signals_sim.rs index d821626b..25b376cc 100644 --- a/concurrency/tests/exit_signals_sim.rs +++ b/concurrency/tests/exit_signals_sim.rs @@ -24,7 +24,6 @@ use spawned_rt::tasks::{ sleep, spawn, }; use std::{ - cell::RefCell, collections::BTreeMap, future::Future, sync::{Arc, Mutex}, @@ -46,14 +45,25 @@ fn position(log: &[String], entry: &str) -> Option { } // Outcome counts per body, printed by the driver: every branch the -// assertions distinguish must be reached. Per test thread (the simulator -// runs a body on the thread that explores it). -thread_local! { - static OUTCOMES: RefCell> = const { RefCell::new(BTreeMap::new()) }; +// assertions distinguish must be reached. Keyed by test: the simulator runs +// each body on a fresh thread named like the test thread that explores it. +static OUTCOMES: std::sync::Mutex>> = + std::sync::Mutex::new(BTreeMap::new()); + +fn test_name() -> String { + std::thread::current().name().unwrap_or("").to_owned() +} + +fn outcomes() -> std::sync::MutexGuard<'static, BTreeMap>> { + OUTCOMES.lock().unwrap_or_else(|p| p.into_inner()) } fn outcome(name: impl Into) { - OUTCOMES.with(|o| *o.borrow_mut().entry(name.into()).or_default() += 1); + *outcomes() + .entry(test_name()) + .or_default() + .entry(name.into()) + .or_default() += 1; } fn ms(n: u64) -> Duration { @@ -542,7 +552,7 @@ where F: Fn(u64) -> Fut + Sync, Fut: Future, { - OUTCOMES.with(|o| o.borrow_mut().clear()); + outcomes().remove(&test_name()); let report = explore(&Strategy::parse(PCT.0).unwrap(), 0, PCT.1, &body); assert!( report.found.is_none(), @@ -570,7 +580,7 @@ where "{name} {}: {runs} runs over {DFS_SEEDS} seeds, {exhausted} searches exhausted", DFS.0 ); - let outcomes = OUTCOMES.with(|o| o.borrow().clone()); + let outcomes = outcomes().remove(&test_name()).unwrap_or_default(); assert!(!outcomes.is_empty(), "{name}: no outcome recorded"); println!("{name} outcomes: {outcomes:?}"); } From e3878af54da1558bdc4b96efdaacee3f0a6ba7de Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Thu, 12 Nov 2026 19:49:36 -0300 Subject: [PATCH 205/208] feat(reply)!: the caller's expiry is the only timeout authority, as OTP's gen_server:call A reply no longer reads the clock. It writes its value under the slot lock and publishes it with one CAS `OPEN -> REPLIED` (a deferred reply holds the owner's reply permit across it, the death-point fence). The deadline is decided only by an expiry `OPEN -> TIMED_OUT`: the caller's runtime timer, the owner's deadline heap, or a `[deadline]` check (`is_closed`, `defer`, attach, `shorten_deadline`). A reply that loses that race finds the slot `TIMED_OUT`, gets its value back as `Delivery::Expired`, and the caller sees `RequestTimeout`; a reply that lands after the deadline instant but before the expiry runs is `Delivered` and the caller gets the value, as an OTP reply already in the caller's mailbox is received. The `WRITING` slot state (the reply's deadline read between its CAS and its publication) is gone, and with it the registry close's wait for it. Checked for dependents of the reply-side check: `shorten_deadline` keeps its semantics (lower under the slot lock while open, then the `[deadline]` check, which races the reply on the word; exactly one wins); job deadlines (`JobOptions::deadline`, `pipe_reply`) are a separate mechanism and are unchanged; the death boundary (fence, registry close, batched discharge) no longer waits for `WRITING` and is otherwise unchanged (death_boundary, failpoints pass). Model (verification/tla/DeferredReplies.tla): `TryReply` discharges `ok` whenever the slot is open (no `now < dl`); `Expire` alone discharges `timeout`; `NoSuccessAfterDeadline` is `okAt <= dl` (a reply can land at the deadline instant before the expiry, never after it: the clock does not pass an open slot's deadline); `CallerConsistent` uses `okAt <= sent`; the `lateOk` ghost drops `now < dl`. Negative control `LateSuccess` now models the new ordering's failure: a reply that overrides an expired slot (counterexample to NoSuccessAfterDeadline). Trace mapping (TraceReplies.tla): a reply's discharge is `OPEN -> REPLIED`, so a "timeout" is always `Expire`; trace_check.py needed no change. Docs: semantics.md R3, deferred-replies.md (state machine, rules 1-2, deadline sources, verification), GUIDE (Delivery), MIGRATION, CHANGELOG, `Delivery::Expired` rustdoc. Results: - TLC (check-tla --deep --only MCDeferredReplies*, <= 4 workers): MCDeferredReplies 27,961,270 states generated, 5,503,955 distinct, no error; MCDeferredRepliesDeep 76,892,564 generated, 15,141,666 distinct, no error. - Traces: 93/93 accepted; negative controls: 130 counterexamples (MCDeferredReplies/LateSuccess violates NoSuccessAfterDeadline), all trace mutations rejected. - Loom: reply_slot 25 (reply_vs_expiry and reply_vs_shortening restated: exactly one of reply and expiry wins the word; a won reply is delivered even at the deadline, a lost one returns Expired and the caller reads TimedOut), job_core 8, link_fence 10, registry_turn 3. - fmt, clippy (3 configurations), workspace tests, failpoints + death_boundary, sim 320/320, model-sync events/models/paths + unit tests. - Callgrind (micro; before = f66429c0): burst_raw 104.1M -> 103.4M, seq_raw 112.6M -> 105.8M, seq_default 130.7M -> 137.0M (seq paths are scheduler-noise dominated; the reply path no longer reads the clock). --- CHANGELOG.md | 9 ++ concurrency/src/reply/mod.rs | 6 +- concurrency/src/reply/registry.rs | 4 +- concurrency/src/reply/slot.rs | 177 ++++++++++----------------- concurrency/tests/job_core_loom.rs | 1 - concurrency/tests/reply_slot_loom.rs | 75 +++++------- docs/GUIDE.md | 6 +- docs/MIGRATION.md | 9 ++ docs/design/deferred-replies.md | 47 +++---- docs/design/semantics.md | 12 +- verification/tla/DeferredReplies.tla | 33 +++-- verification/tla/TraceReplies.tla | 4 +- 12 files changed, 182 insertions(+), 201 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45344814..ad672eb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +- **Behaviour (0.8)**: request timeouts are the caller's, as OTP's + `gen_server:call`: a reply reads no clock, and only the caller's expiry + (its timer, the owner's deadline heap, a deadline check) times a request + out. A reply that loses that race is dropped (`Delivery::Expired`, the + caller sees `RequestTimeout`); one that lands after the deadline but + before the expiry runs is delivered. The `WRITING` slot state is gone and + no reply path reads the clock. `DeferredReplies.tla` models it (a reply + delivers while the slot is open; `LateSuccess` now overrides an expired + slot). - Performance (0.8 request path, no API change): the reply slot keeps one lock for value and waker with lazily created deferral state (no condvar: a blocking `recv()` parks its thread); the tasks envelope lends its diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 235baa4d..cb0216d9 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -34,7 +34,6 @@ use std::time::Duration; mod sync { pub(crate) use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, AtomicUsize, Ordering}; pub(crate) use std::sync::{Arc, Arc as JobArc, Condvar, Mutex, MutexGuard, Weak as JobWeak}; - pub(crate) use std::thread::yield_now; /// Lifecycle-trace builds keep the owner in each slot (its events name /// it); other builds keep nothing, so a slot stays small. #[cfg(feature = "lifecycle-trace")] @@ -395,7 +394,7 @@ impl<'t, R: Send + 'static> PendingReply<'t, R> { Delivery::CallerGone } Some(slot) => { - let (delivery, unsent) = slot.reply_in_turn(value, &now_ticks); + let (delivery, unsent) = slot.reply_in_turn(value); drop(unsent); record(delivery); delivery @@ -510,8 +509,7 @@ impl ReplyTo { .expect("ReplyTo holds its slot until consumed"); // Admitted by the owner's death-point fence: once `NoProc` is // observable no reply succeeds (the owner's close answers the slot). - let (delivery, unsent) = - slot.reply_admitted(value, &now_ticks, || self.registry.reply_permit()); + let (delivery, unsent) = slot.reply_admitted(value, || self.registry.reply_permit()); drop(unsent); record(delivery); if delivery == Delivery::Delivered { diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index 2caaccd3..c5c36b50 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -322,7 +322,7 @@ impl Registry { } } - /// Admit a deferred reply's linearization (its `OPEN -> WRITING` CAS), + /// Admit a deferred reply's linearization (its `OPEN -> REPLIED` CAS), /// or `None` once the owner reached its death point: the slot then stays /// listed and the registry's close answers it with the final reason. pub(crate) fn reply_permit(&self) -> Option> { @@ -596,7 +596,7 @@ mod tests { let registry = Registry::new(ActorId::next(), 8); let open = deferred(®istry, NEVER); let answered = deferred(®istry, NEVER); - answered.reply(1, &|| 0); + answered.reply(1); assert_eq!(registry.close(), 1); assert_eq!( Terminal::of(crate::reply::slot::state(open.word())), diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index a0182ad2..a3684155 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -12,10 +12,11 @@ //! reserved before the CAS and rolled back if the CAS fails; a negative delta //! is returned after it, before the receiver is woken. //! -//! `WRITING` counts as live for `credits`: it is the transient state of a -//! reply between its linearization point (`OPEN -> WRITING`) and the -//! publication of the value (`WRITING -> REPLIED`) or the deadline check -//! (`WRITING -> TIMED_OUT`), so the token's credit returns at that second CAS. +//! A reply is one CAS `OPEN -> REPLIED` with the value written under the +//! slot lock; no reply reads the clock. The deadline is enforced only by an +//! expiry `OPEN -> TIMED_OUT` (the caller's timer, the owner's deadline +//! heap, or a `[deadline]` check), as OTP's caller-side `call` timeout: a +//! reply and an expiry race on the word and exactly one wins. //! //! This file is shared with the Loom model (`tests/reply_slot_loom.rs`), which //! supplies the synchronization types through `super::sync`. Time is passed in @@ -30,11 +31,12 @@ //! (An `AtomicWaker` beside a value mutex measured more instructions per //! request than one lock for both: two more read-modify-writes.) -use super::sync::{yield_now, Arc, AtomicU64, AtomicUsize, Mutex, Ordering}; +use super::sync::{Arc, AtomicU64, AtomicUsize, Mutex, Ordering}; use std::task::{Poll, Waker}; pub(crate) const OPEN: u64 = 0; -pub(crate) const WRITING: u64 = 1; +// State 1 was `WRITING` (a reply's deadline check between its CAS and its +// publication); replies no longer read the clock, so it no longer exists. pub(crate) const REPLIED: u64 = 2; pub(crate) const DROPPED: u64 = 3; pub(crate) const OVERLOADED: u64 = 4; @@ -76,7 +78,7 @@ pub(crate) fn jobs(word: u64) -> u64 { } #[inline] fn live(state: u64) -> bool { - state == OPEN || state == WRITING + state == OPEN } /// Credits a slot holds in its owner's pool. #[inline] @@ -101,10 +103,10 @@ pub(crate) enum Terminal { } /// The model's name (`DeferredReplies.tla`) of the terminal state a slot -/// leaves `OPEN` for; `WRITING` is the reply's linearization point. +/// leaves `OPEN` for; `REPLIED` is the reply's linearization point. fn trace_name(state: u64) -> &'static str { match state { - WRITING | REPLIED => "ok", + REPLIED => "ok", DROPPED => "dropped", OVERLOADED => "overloaded", TIMED_OUT => "timeout", @@ -147,7 +149,11 @@ pub enum Delivery { /// The caller left (abandoned the receiver) or the slot was already /// answered (dropped, overloaded). CallerGone, - /// The request's deadline had passed; the value was dropped. + /// The caller's deadline expired the request first (its timer, the + /// owner's deadline heap, or a deadline check); the value was dropped + /// and the caller sees `RequestTimeout`. A reply reads no clock (OTP's + /// `gen_server:call`): one that lands after the deadline but before + /// that expiry is `Delivered`. Expired, /// The owner died and its registry closed the slot (`ActorStopped`). OwnerExited, @@ -449,8 +455,7 @@ impl Slot { } /// Registry close at the owner's death: every slot still `OPEN` becomes - /// `STOPPED`. A reply in `WRITING` is a value write with no user code in - /// it, so the close waits for it to resolve. + /// `STOPPED`; one already answered is left as it is. pub(crate) fn close(&self) -> bool { match self.close_collect() { Some(wake) => { @@ -466,22 +471,17 @@ impl Slot { /// [`run_wake`](Self::run_wake) once a whole batch is closed, so no /// waker (executor or user code) runs between two closes. pub(crate) fn close_collect(&self) -> Option { - loop { - if let Ok((old, new)) = self.cas::<()>(None, |w| { + let (old, new) = self + .cas::<()>(None, |w| { if state(w) == OPEN { Ok(with_state(w, STOPPED)) } else { Err(()) } - }) { - // The credit returns with the transition; only wakes wait. - return Some(self.after(old, new)); - } - if state(self.word()) != WRITING { - return None; - } - yield_now(); - } + }) + .ok()?; + // The credit returns with the transition; only wakes wait. + Some(self.after(old, new)) } /// The owner's close records its final reason before the `STOPPED` @@ -542,11 +542,10 @@ impl Slot { /// `with_deadline(at)`: shorten only, then the `[deadline]` check. /// Returns whether the deadline was lowered. /// - /// The reply's `WRITING` CAS is the linearization point: the reply reads - /// the deadline under the slot lock, and a shortening changes it only - /// under that lock and only while the word is `OPEN`. A shortening that - /// finds `WRITING` (or later) is a no-op; one that lowers the deadline is - /// seen by any reply that has not read it yet. + /// A shortening changes the deadline only while the word is `OPEN` + /// (under the slot lock) and then runs the `[deadline]` check: an + /// earlier deadline already passed expires the slot at once, racing any + /// reply on the word (exactly one wins). Replies do not read it. pub(crate) fn shorten(&self, at: u64, now: &impl Fn() -> u64) -> bool { let (lowered, watchers, waker) = { let mut inner = lock(&self.inner); @@ -649,50 +648,55 @@ impl Slot { } } - /// Change only the state field of a `WRITING` word, retrying on concurrent - /// `JOBS` changes (a whole-word store would lose a job-end decrement). - fn resolve_writing(&self, to: u64) -> (u64, u64) { - match self.cas::<()>(None, |w| { - debug_assert_eq!(state(w), WRITING); - Ok(with_state(w, to)) - }) { - Ok(pair) => pair, - Err(_) => unreachable!("resolving WRITING never aborts or reserves"), - } + /// Reply: write the value under the slot lock and publish it with one + /// CAS `OPEN -> REPLIED`, the reply's linearization point (a receiver + /// reads the value under that lock). No clock is read: as in OTP's + /// `gen_server:call`, the caller's deadline is enforced only by the + /// caller's expiry (`OPEN -> TIMED_OUT`, its timer or the owner's deadline + /// heap); a reply that loses that race finds the slot `TIMED_OUT` and + /// gets its value back as `Expired`. A value that was not published is + /// returned so the caller drops it outside every lock. + #[allow(dead_code)] // unit tests and the Loom model + pub(crate) fn reply(&self, value: R) -> (Delivery, Option) { + self.reply_admitted(value, || Some(())) + } + + /// A reply from the owner's own turn (`PendingReply::reply`): the + /// owner's death point cannot run during its own turn, so no admission + /// is needed. + #[inline] + pub(crate) fn reply_in_turn(&self, value: R) -> (Delivery, Option) { + self.publish(value, || Some(())) } - /// Reply: `OPEN -> WRITING`, then read the deadline: at or past it, - /// `WRITING -> TIMED_OUT` and the value comes back as `Expired`; - /// otherwise write the value and `WRITING -> REPLIED`. A value that was - /// not published is returned so the caller drops it outside every lock. - pub(crate) fn reply(&self, value: R, now: &impl Fn() -> u64) -> (Delivery, Option) { - self.reply_admitted(value, now, || Some(())) + /// [`reply`](Self::reply) admitted by the owner: `admit` returns a + /// permit held across the publishing CAS (no user code: a lock and a + /// value move), or `None` once the owner is past its death point. + /// Refused, the slot is left to the owner's close (`OwnerExited`). + pub(crate) fn reply_admitted

( + &self, + value: R, + admit: impl FnOnce() -> Option

, + ) -> (Delivery, Option) { + self.publish(value, admit) } - /// A reply from the owner's own turn (`PendingReply::reply`): nothing - /// shortens the deadline concurrently (only a deferred token can) and the - /// owner's death point cannot run during its own turn, so the only - /// concurrent transitions are the caller's (abandonment, its timer's - /// expiry), each one CAS out of `OPEN`. With the deadline still ahead, - /// the value is written under the slot lock and published by one CAS - /// `OPEN -> REPLIED` (the same linearization point as `OPEN -> WRITING`: - /// a receiver reads the value under that lock); a lost CAS hands the - /// value back. A reply at or past the deadline takes [`reply`](Self::reply). #[inline] - pub(crate) fn reply_in_turn(&self, value: R, now: &impl Fn() -> u64) -> (Delivery, Option) { - let deadline = self.deadline(); - if deadline != NEVER && now() >= deadline { - return self.reply(value, now); - } + fn publish

(&self, value: R, admit: impl FnOnce() -> Option

) -> (Delivery, Option) { + let Some(permit) = admit() else { + return (Delivery::OwnerExited, Some(value)); + }; let mut inner = lock(&self.inner); inner.value = Some(value); - match self.cas(None, |w| { + let published = self.cas(None, |w| { if state(w) == OPEN { Ok(with_state(w, REPLIED)) } else { Err(state(w)) } - }) { + }); + drop(permit); + match published { Ok((old, new)) => { let wake = Self::finish( &mut inner, @@ -717,59 +721,6 @@ impl Slot { } } - /// [`reply`](Self::reply) admitted by the owner: `admit` returns a - /// permit held across the linearizing `OPEN -> WRITING` CAS (and - /// released before the value is written), or `None` once the owner is - /// past its death point. Refused, the slot is left to the owner's close - /// (`OwnerExited`). - pub(crate) fn reply_admitted

( - &self, - value: R, - now: &impl Fn() -> u64, - admit: impl FnOnce() -> Option

, - ) -> (Delivery, Option) { - let Some(permit) = admit() else { - return (Delivery::OwnerExited, Some(value)); - }; - let linearized = self.cas(None, |w| { - if state(w) == OPEN { - Ok(with_state(w, WRITING)) - } else { - Err(state(w)) - } - }); - drop(permit); - if let Err(Some(s)) = linearized { - let delivery = match s { - TIMED_OUT => Delivery::Expired, - STOPPED => Delivery::OwnerExited, - _ => Delivery::CallerGone, - }; - return (delivery, Some(value)); - } - // The deadline is read under the lock that shortening takes, so a - // shortening either precedes this read or finds `WRITING` and does - // nothing. - let mut inner = lock(&self.inner); - let deadline = self.deadline(); - let (delivery, unsent, to) = if deadline != NEVER && now() >= deadline { - (Delivery::Expired, Some(value), TIMED_OUT) - } else { - inner.value = Some(value); - (Delivery::Delivered, None, REPLIED) - }; - let (old, new) = self.resolve_writing(to); - let wake = Self::finish( - &mut inner, - credits(old).saturating_sub(credits(new)), - true, - new, - ); - drop(inner); - wake.run(); - (delivery, unsent) - } - /// Receiver side: take the outcome of a terminal word. `Replied` with no /// value means it was already taken. #[inline] diff --git a/concurrency/tests/job_core_loom.rs b/concurrency/tests/job_core_loom.rs index 7400759f..2fbb471d 100644 --- a/concurrency/tests/job_core_loom.rs +++ b/concurrency/tests/job_core_loom.rs @@ -15,7 +15,6 @@ mod sync { pub(crate) use loom::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; pub(crate) use loom::sync::{Arc, Condvar, Mutex, MutexGuard}; - pub(crate) use loom::thread::yield_now; pub(crate) type SlotOwner = Option<()>; pub(crate) type Reason = (); pub(crate) fn section(f: impl FnOnce() -> T) -> T { diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs index 8e7a12cf..75faef1f 100644 --- a/concurrency/tests/reply_slot_loom.rs +++ b/concurrency/tests/reply_slot_loom.rs @@ -10,7 +10,6 @@ mod sync { pub(crate) use loom::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; pub(crate) use loom::sync::{Arc, Mutex, MutexGuard}; - pub(crate) use loom::thread::yield_now; pub(crate) type SlotOwner = Option<()>; pub(crate) type Reason = (); pub(crate) fn section(f: impl FnOnce() -> T) -> T { @@ -105,42 +104,35 @@ fn deferred(pool: &Arc, deadline: u64) -> Arc> { slot } -/// Reply versus expiry by a polling receiver as the clock reaches the -/// deadline: one terminal outcome, and no success at or after the deadline. +/// Reply versus expiry as the clock reaches the deadline (OTP: the +/// caller's expiry is the only timeout authority; the reply reads no +/// clock). Exactly one wins the word: a reply that wins is delivered even +/// at the deadline (before the expiry ran); one that loses gets its value +/// back as `Expired`, and the caller sees `TimedOut`, never the value. #[test] fn reply_vs_expiry() { model(|| { let clock = Clock::at(0); let pool = Arc::new(CreditPool::new(1)); let slot = deferred(&pool, 10); - let replied_at = Arc::new(AtomicU64::new(NEVER)); let replier = { - let (slot, clock, replied_at) = (slot.clone(), clock.clone(), replied_at.clone()); - loom::thread::spawn(move || { - let now = || { - let t = clock.now()(); - replied_at.store(t, Ordering::SeqCst); - t - }; - slot.reply(7, &now) - }) + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply(7)) }; clock.set(10); let expired = slot.check_deadline(&clock.now()); let (delivery, back) = replier.join().unwrap(); - assert!(expired); + assert!(expired, "the deadline has passed"); match delivery { Delivery::Delivered => { assert_eq!(state(slot.word()), slot::REPLIED); - assert!( - replied_at.load(Ordering::SeqCst) < 10, - "success at the deadline" - ); assert!(back.is_none()); + assert_eq!(slot.try_take(&clock.now()), Some(Ok(7))); } Delivery::Expired => { assert_eq!(state(slot.word()), slot::TIMED_OUT); assert_eq!(back, Some(7)); + assert_eq!(slot.try_take(&clock.now()), Some(Err(Terminal::TimedOut))); } other => panic!("unexpected {other:?}"), } @@ -148,8 +140,10 @@ fn reply_vs_expiry() { }); } -/// Reply versus `with_deadline` shortening into the past: the reply's -/// `WRITING` CAS linearizes; success implies the reply read a live deadline. +/// Reply versus `shorten_deadline` into the past: the shortening's +/// `[deadline]` check expires the slot, racing the reply on the word. The +/// reply wins (delivered, whatever the deadline became) or loses +/// (`Expired`, and then the shortening was applied). #[test] fn reply_vs_shortening() { model(|| { @@ -157,22 +151,13 @@ fn reply_vs_shortening() { let pool = Arc::new(CreditPool::new(1)); let slot = deferred(&pool, 100); let replier = { - let (slot, clock) = (slot.clone(), clock.clone()); - loom::thread::spawn(move || slot.reply(1, &clock.now())) + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply(1)) }; let lowered = slot.shorten(3, &clock.now()); let (delivery, _) = replier.join().unwrap(); - // Linearization at the reply's WRITING CAS: a shortening after it is - // a no-op (the deadline stays), one before it is seen by the reply. match delivery { - Delivery::Delivered => { - assert_eq!(state(slot.word()), slot::REPLIED); - assert!( - !lowered, - "a shortening was applied but not seen by the reply" - ); - assert_eq!(slot.deadline(), 100); - } + Delivery::Delivered => assert_eq!(state(slot.word()), slot::REPLIED), Delivery::Expired => { assert_eq!(state(slot.word()), slot::TIMED_OUT); assert!(lowered); @@ -192,7 +177,7 @@ fn reply_vs_abandon() { let slot = deferred(&pool, NEVER); let replier = { let slot = slot.clone(); - loom::thread::spawn(move || slot.reply(1, &|| 0)) + loom::thread::spawn(move || slot.reply(1)) }; let abandoned = slot.abandon(); let (delivery, back) = replier.join().unwrap(); @@ -216,7 +201,7 @@ fn reply_in_turn_vs_abandon() { let slot: Arc> = Arc::new(Slot::new(NEVER)); let replier = { let slot = slot.clone(); - loom::thread::spawn(move || slot.reply_in_turn(1, &|| 0)) + loom::thread::spawn(move || slot.reply_in_turn(1)) }; let abandoned = slot.abandon(); let (delivery, back) = replier.join().unwrap(); @@ -239,7 +224,7 @@ fn reply_in_turn_vs_expiry() { let slot: Arc> = Arc::new(Slot::new(10)); let replier = { let slot = slot.clone(); - loom::thread::spawn(move || slot.reply_in_turn(7, &|| 0)) + loom::thread::spawn(move || slot.reply_in_turn(7)) }; let expired = slot.terminate(Terminal::TimedOut); let (delivery, back) = replier.join().unwrap(); @@ -261,7 +246,7 @@ fn polled_receiver_vs_reply_in_turn() { let slot: Arc> = Arc::new(Slot::new(NEVER)); let replier = { let slot = slot.clone(); - loom::thread::spawn(move || slot.reply_in_turn(4, &|| 0).0) + loom::thread::spawn(move || slot.reply_in_turn(4).0) }; let got = loom::future::block_on(std::future::poll_fn(|cx| slot.poll(cx.waker()))); assert_eq!(got, Ok(4)); @@ -278,7 +263,7 @@ fn reply_vs_owner_close() { let slot = deferred(&pool, NEVER); let replier = { let slot = slot.clone(); - loom::thread::spawn(move || slot.reply(1, &|| 0).0) + loom::thread::spawn(move || slot.reply(1).0) }; let closed = slot.close(); assert!( @@ -415,7 +400,7 @@ fn job_end_during_reply() { let slot = slot.clone(); loom::thread::spawn(move || slot.job_end()) }; - assert_eq!(slot.reply(9, &|| 0).0, Delivery::Delivered); + assert_eq!(slot.reply(9).0, Delivery::Delivered); ender.join().unwrap(); assert_eq!(jobs(slot.word()), 1); assert_eq!(pool.used(), 1, "the job still running keeps its credit"); @@ -461,7 +446,7 @@ fn parked_receiver_vs_reply() { let slot: Arc> = Arc::new(Slot::new(NEVER)); let replier = { let slot = slot.clone(); - loom::thread::spawn(move || slot.reply(3, &|| 0).0) + loom::thread::spawn(move || slot.reply(3).0) }; assert_eq!(slot.recv_blocking(&|| 0), Ok(3)); assert_eq!(replier.join().unwrap(), Delivery::Delivered); @@ -489,7 +474,7 @@ fn polled_receiver_vs_reply() { let slot: Arc> = Arc::new(Slot::new(NEVER)); let replier = { let slot = slot.clone(); - loom::thread::spawn(move || slot.reply(4, &|| 0).0) + loom::thread::spawn(move || slot.reply(4).0) }; let got = loom::future::block_on(std::future::poll_fn(|cx| slot.poll(cx.waker()))); assert_eq!(got, Ok(4)); @@ -572,7 +557,7 @@ fn parked_receiver_vs_reply_in_turn() { let slot: Arc> = Arc::new(Slot::new(NEVER)); let replier = { let slot = slot.clone(); - loom::thread::spawn(move || slot.reply_in_turn(5, &|| 0).0) + loom::thread::spawn(move || slot.reply_in_turn(5).0) }; assert_eq!(slot.recv_blocking(&|| 0), Ok(5)); assert_eq!(replier.join().unwrap(), Delivery::Delivered); @@ -612,7 +597,7 @@ fn watcher_and_receiver_vs_deferred_reply() { let slot = deferred(&pool, NEVER); let replier = { let slot = slot.clone(); - loom::thread::spawn(move || slot.reply(8, &|| 0).0) + loom::thread::spawn(move || slot.reply(8).0) }; loom::future::block_on(std::future::poll_fn(|cx| { slot.poll_closed(cx.waker(), &|| 0) @@ -637,7 +622,7 @@ fn polled_receiver_vs_shorten_and_reply() { let (slot, clock) = (slot.clone(), clock.clone()); loom::thread::spawn(move || { slot.shorten(5, &clock.now()); - slot.reply(2, &clock.now()).0 + slot.reply(2).0 }) }; let got = loom::future::block_on(std::future::poll_fn(|cx| slot.poll(cx.waker()))); @@ -679,7 +664,7 @@ fn lent_reply_vs_receiver_last_reference() { }) }; let borrowed: &Arc> = &lent; - let (delivery, back) = borrowed.reply_in_turn(Counted(drops.clone()), &|| 0); + let (delivery, back) = borrowed.reply_in_turn(Counted(drops.clone())); drop(back); drop(lent); let took = receiver.join().unwrap(); @@ -704,7 +689,7 @@ fn unshared_take_vs_lent_reply_and_release() { let lent: Arc> = Arc::new(Slot::new(NEVER)); let mut caller = Some(lent.clone()); let owner = loom::thread::spawn(move || { - let (delivery, back) = lent.reply_in_turn(6, &|| 0); + let (delivery, back) = lent.reply_in_turn(6); drop(lent); (delivery, back) }); diff --git a/docs/GUIDE.md b/docs/GUIDE.md index d8b2c506..0d9708aa 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -636,7 +636,11 @@ Rules (the [design](design/deferred-replies.md) has the full contract): (default 1024). At the bound the caller gets `Err(Overloaded)` at once. - A `ReplyTo` is `Send + 'static` and answers exactly once: `token.reply(v)` returns a `Delivery` (`Delivered`, `CallerGone`, `Expired`, - `OwnerExited`); `token.overloaded()` answers `Err(Overloaded)`. + `OwnerExited`); `token.overloaded()` answers `Err(Overloaded)`. As in OTP, + the caller's timeout is the only deadline authority: a reply reads no + clock, so `Expired` means the caller's expiry already ran (it sees + `RequestTimeout`); a reply just after the deadline, before that expiry, + is still `Delivered` and the caller gets the value. - Dropping an unanswered token answers `ReplyDropped`. If the turn panicked or the actor is exiting, it is held instead and answered at the actor's death with its final reason. `stopped()` may still answer stored tokens; at the diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 55d68695..653fa765 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -210,6 +210,15 @@ table is the full list). What to edit: This is a breaking release. The notes below grow with the implementation: +- Request timeouts follow OTP's `gen_server:call`: only the caller's expiry + (its timer, the owner's deadline heap for a raw `send_request` deadline, + or a deadline check such as `is_closed()`) times a request out; a reply + reads no clock. Code that relied on a reply at or past the deadline being + refused (`Delivery::Expired`) must not: a reply that lands after the + deadline but before the caller's expiry runs is `Delivered` and the caller + gets the value. `Expired` still means the caller already saw + `RequestTimeout`. + - `ActorError` is `#[non_exhaustive]` and gains `ReplyDropped` and `Overloaded`; add a wildcard arm to exhaustive matches. diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index a574c97d..965b8c75 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -82,7 +82,7 @@ compare-and-swap on that word, so "install a credit only if `OPEN`" and ```text word: AtomicU64 = state:4 | TOKEN:1 | REG:1 | JOBS:16 | (spare) - state: OPEN | WRITING | REPLIED | DROPPED | OVERLOADED | TIMED_OUT | STOPPED | ABANDONED + state: OPEN | REPLIED | DROPPED | OVERLOADED | TIMED_OUT | STOPPED | ABANDONED TOKEN: a deferred token holds this request's credit (set by defer()) REG: listed in the owner registry (set together with TOKEN) JOBS: running linked jobs (pipe_reply); MAX_OUTSTANDING is limited to @@ -142,29 +142,30 @@ operation first reads `now`, and if `now >= deadline` it performs | `defer()` `[deadline]` | `OPEN`, `TOKEN=0` | `OPEN`, `TOKEN=1`, `REG=1` | +1 if `JOBS=0` | | `pipe_reply` attach `[deadline]` | `OPEN`, `TOKEN=1` | `OPEN`, `JOBS+1` | +1 if `JOBS>=1` (the first job shares the token's credit) | | linked job ends | any, `JOBS>=1` | same state, `JOBS-1` | -1 if this lowers `credits` | -| reply | `OPEN` | `WRITING`, then `REPLIED` or `TIMED_OUT` | -1 if `TOKEN` and `JOBS=0` | +| reply | `OPEN` | `REPLIED` (no clock read) | -1 if `TOKEN` and `JOBS=0` | | drop, overloaded, expiry, abandonment, registry close | `OPEN` | `DROPPED` / `OVERLOADED` / `TIMED_OUT` / `ABANDONED` / `STOPPED` | -1 if `TOKEN` and `JOBS=0` | | `is_closed()` `[deadline]` | `OPEN`, past deadline | `TIMED_OUT` | as expiry | Rules: -1. **Reply.** The reply is `CAS OPEN -> WRITING`, which keeps the - `TOKEN`/`REG`/`JOBS` bits. It then reads `deadline` and `now`: - - at or past the deadline: CAS the state field `WRITING -> TIMED_OUT`, and - return the value as `Expired`; - - otherwise: write the value, then CAS the state field - `WRITING -> REPLIED`. - - These CASes change only the state field and retry on concurrent `JOBS` - changes; a whole-word store would lose a job-end decrement. - - Only `REPLIED` publishes the value; a receiver that sees `WRITING` waits. - A registry close that finds `WRITING` waits for it to resolve. The window - is a value write; no user code runs in it. -2. **Shortening.** `shorten_deadline(at)` is `fetch_min(deadline, at)`, then the - `[deadline]` check. The linearization point is the reply's `WRITING` CAS: - - a shortening before it is seen by the reply; - - a shortening after it finds the word not `OPEN` and does nothing. +1. **Reply.** The reply writes the value under the slot lock and publishes + it with one `CAS OPEN -> REPLIED` (keeping the `TOKEN`/`REG`/`JOBS` + bits; it retries on concurrent `JOBS` changes). A deferred reply holds + the owner's reply permit across that CAS (the death-point fence). The + reply reads no clock: the caller's expiry is the only timeout authority, as in OTP's +`gen_server:call`: a reply reads no clock. A reply that loses the race with +the expiry (the caller's timer, the owner's deadline heap, or a +`[deadline]` check) is discarded (`Delivery::Expired`) and the caller sees +`RequestTimeout`; a reply that lands after the deadline instant but before +the expiry runs is delivered (`Delivery::Delivered`, and the caller gets +the value), as an OTP reply already in the caller's mailbox when its +`after` clause would fire is received. A reply that finds the + word not `OPEN` takes its value back (`Expired`, `CallerGone`, + `OwnerExited`). +2. **Shortening.** `shorten_deadline(at)` lowers the deadline under the slot + lock while the word is `OPEN`, then runs the `[deadline]` check: a new + deadline already passed expires the slot at once, racing any reply on + the word (exactly one wins; a shortening after the reply does nothing). 3. **Registry.** The registry is a per-instance list of weak slot references, appended by `defer()` after its CAS and pruned lazily; the `REG` bit in the word is authoritative. Closing the registry CASes every listed slot that is @@ -245,7 +246,7 @@ absolute instant, and OTP's `call` starts waiting when it sends. - **The owner's deadline heap,** for deferred tokens and for job deadlines. It returns credits and delivers job deadline notifications even when no caller is polling. -- **A reply itself:** at or past the deadline it expires the slot (rule 2). +- **Not the reply:** a reply reads no clock (rule 1, as OTP). A queued or in-turn request that nobody polls stays open until it is polled, replied (and expired), deferred (`defer()` on an expired slot returns @@ -911,8 +912,10 @@ It establishes, within its bounds: - **Discharge.** At most one discharge per request (`AtMostOneDischarge`), and no open slot without a live holder, including job-held tokens at death (`NoOrphan`). -- **Deadlines.** No success at or after the deadline - (`NoSuccessAfterDeadline`); deadlines are serviced while a turn is stuck +- **Deadlines.** Only the expiry decides a timeout; no success after the + slot expired, so none after the deadline instant + (`NoSuccessAfterDeadline`: `okAt <= dl`; negative control `LateSuccess`, a + reply overriding an expired slot); deadlines are serviced while a turn is stuck (`TimelyExpiry`); shortening never extends past the caller's deadline (`DeadlineWithinCaller`). - **Credits.** Combined token and linked-job accounting with attach-on-open diff --git a/docs/design/semantics.md b/docs/design/semantics.md index 4567a594..e9615b98 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -142,8 +142,16 @@ The deadline is fixed when the request is sent (send time plus the timeout) and travels in the reply slot: there is one deadline, not a caller timer plus a server one. tasks: `request`-family futures enqueue on first poll and protocol methods enqueue when called; either way the deadline starts at the -enqueue. threads: the call blocks. Expired requests are still dispatched (OTP); -a late reply is dropped (`Delivery::Expired`). +enqueue. threads: the call blocks. Expired requests are still dispatched (OTP). +The caller's expiry is the only timeout authority, as in OTP's +`gen_server:call`: a reply reads no clock. A reply that loses the race with +the expiry (the caller's timer, the owner's deadline heap, or a +`[deadline]` check) is discarded (`Delivery::Expired`) and the caller sees +`RequestTimeout`; a reply that lands after the deadline instant but before +the expiry runs is delivered (`Delivery::Delivered`, and the caller gets +the value), as an OTP reply already in the caller's mailbox when its +`after` clause would fire is received. Timer resolution (the runtime timer's 1 ms +wheel, the deadline service's wake-up) bounds that window. Which clock expires a waiting caller is part of the contract: `request`, `request_with_timeout` and `Response::from(rx)` arm the runtime's timer, so they follow paused Tokio time; a request sent with `send_request` and a diff --git a/verification/tla/DeferredReplies.tla b/verification/tla/DeferredReplies.tla index 47441a27..2faa5324 100644 --- a/verification/tla/DeferredReplies.tla +++ b/verification/tla/DeferredReplies.tla @@ -5,10 +5,13 @@ (* Part 2 (job fencing, keyed replacement, emits) is DeferredJobs.tla. *) (* *) (* Each request owns a slot with one atomic open -> terminal transition. *) -(* The deadline is fixed at send and is the only timeout; a reply at or *) -(* after it expires the slot instead of delivering. Expiry is serviced *) -(* independently of A's turns ("waiting" callers are polling; deferred *) -(* tokens are covered by the deadline service). *) +(* The deadline is fixed at send and is the only timeout, and only the *) +(* expiry (Expire: the caller's timer, the owner's deadline heap, or a *) +(* [deadline] check) decides it, as OTP's gen_server:call: a reply reads *) +(* no clock and delivers while the slot is open; one that loses the race *) +(* with the expiry finds the slot closed and its value is discarded. *) +(* Expiry is serviced independently of A's turns ("waiting" callers are *) +(* polling; deferred tokens are covered by the deadline service). *) (* *) (* Owner A. Queued requests are dispatched even if their slot is already *) (* terminal (OTP: the server still handles the call). In a turn the *) @@ -50,7 +53,8 @@ (* violate the named invariant: *) (* NoAutoReply (NoOrphan) drops leave the slot open *) (* NoAtomicExpiry (AtMostOneDischarge) reply after expiry discharges *) -(* LateSuccess (NoSuccessAfterDeadline) *) +(* LateSuccess (NoSuccessAfterDeadline) a reply overrides an expired *) +(* slot (delivered after expiry) *) (* LoopDeadline (TimelyExpiry) expiry waits for A's turn *) (* Lengthen (DeadlineWithinCaller) *) (* Unbounded (BoundedA) defer ignores the bound *) @@ -125,10 +129,15 @@ Discharge(r, k) == DischargeSet({r}, k) Keep == UNCHANGED <> OpenOf(S) == {r \in S : Open(r)} +\* A reply reads no clock (OTP): it delivers while the slot is open; after +\* the expiry it finds the slot closed and its value is discarded. TryReply(r) == IF Open(r) - THEN IF now < dl[r] \/ Fault = "LateSuccess" THEN Discharge(r, "ok") - ELSE Discharge(r, "timeout") + THEN Discharge(r, "ok") + ELSE IF Fault = "LateSuccess" /\ slot[r] = "timeout" + THEN /\ slot' = [slot EXCEPT ![r] = "ok"] + /\ okAt' = [okAt EXCEPT ![r] = now] + /\ UNCHANGED <> ELSE IF Fault = "NoAtomicExpiry" /\ slot[r] = "timeout" THEN /\ discharges' = [discharges EXCEPT ![r] = @ + 1] /\ okAt' = [okAt EXCEPT ![r] = now] @@ -256,7 +265,7 @@ ReplyGateOpen == phase \in {"run", "exiting", "stuck"} LaterReply(r) == /\ ReplyGateOpen /\ turnA = 0 /\ holder[r] = "stateA" /\ holder' = [holder EXCEPT ![r] = "gone"] /\ TryReply(r) - /\ lateOk' = IF phase = "sealed" /\ Open(r) /\ now < dl[r] THEN lateOk \cup {r} ELSE lateOk + /\ lateOk' = IF phase = "sealed" /\ Open(r) THEN lateOk \cup {r} ELSE lateOk /\ U(<>) LaterDrop(r) == @@ -430,7 +439,11 @@ NoOrphan == \A r \in Req : Open(r) => /\ phase # "dead" /\ holder[r] \in {"qA", "turnA", "localA", "dropA", "stateA", "doneA", "pendA"} -NoSuccessAfterDeadline == \A r \in Req : slot[r] = "ok" => okAt[r] < dl[r] +\* A reply succeeds only while its slot is open: never after the expiry, so +\* no later than the deadline (the clock does not pass an open slot's +\* deadline: TimelyExpiry). It may land at the deadline instant, before the +\* expiry runs, as an OTP reply already in the caller's mailbox does. +NoSuccessAfterDeadline == \A r \in Req : slot[r] = "ok" => okAt[r] <= dl[r] TimelyExpiry == \A r \in Req : Open(r) => now <= dl[r] DeadlineWithinCaller == \A r \in Req : slot[r] # "none" => dl[r] <= sent[r] BoundedA == Credits <= BoundA @@ -453,7 +466,7 @@ WokenAfterDischarge == NoEarlyClose == earlyClosed = {} DownstreamOwned == \A r \in Req : down[r] \in {"qB", "stateB"} => jobs[r] > 0 CallerConsistent == - \A r \in Req : /\ caller[r] = "ok" => (slot[r] = "ok" /\ okAt[r] < sent[r]) + \A r \in Req : /\ caller[r] = "ok" => (slot[r] = "ok" /\ okAt[r] <= sent[r]) /\ caller[r] = "timeout" => slot[r] = "timeout" Safety == diff --git a/verification/tla/TraceReplies.tla b/verification/tla/TraceReplies.tla index 8a21b1fe..39f1b8b0 100644 --- a/verification/tla/TraceReplies.tla +++ b/verification/tla/TraceReplies.tla @@ -9,7 +9,9 @@ (* Send slot created (Send) *) (* Dispatch its request turn starts (DispatchA) *) (* Discharge the slot's one OPEN -> terminal transition, in the trace *) -(* section with its CAS; a reply counts at WRITING (x: kind) *) +(* section with its CAS; a reply's is OPEN -> REPLIED (x: ok) *) +(* and reads no clock, so "timeout" is always an expiry *) +(* (Expire), never a reply (OTP: the caller's timeout) *) (* Shorten shorten_deadline lowered the deadline, under the slot lock *) (* (Shorten; TLC picks the new deadline) *) (* PipeReply pipe_reply* launched a job ("linked") or the slot refused *) From 8f07305ff36b253ae4f366c0877ff7eeb42a4e80 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Fri, 13 Nov 2026 00:04:38 -0300 Subject: [PATCH 206/208] perf(reply): a raw deadline goes to the owner's heap only when its receiver waits, found by a key in the word f66429c0 put every raw `send_request` deadline in the owner's heap at send. Each entry held a `Weak` to its slot, so a slot answered long before its deadline kept its allocation alive until the entry was popped or the heap compacted, and every send paid the registry lock and a heap push: micro burst_dl fell to -35% against main's raw request. Now the slot records its owner's registry key (`Registry::key`, nonzero) in the word's top 40 bits at construction (no transition touches them; the slot still holds no reference to its owner and stays in the small size class). An awaited raw receiver that finds its slot still open looks the owner up by key (`Registry::by_key`, a global key -> Weak map written once per actor instance and cleared at its drop) and puts the deadline in the heap once, as before f66429c0; if the owner is gone it falls back to the runtime timer. A request answered before its receiver waits never touches the heap. The `TRACKED` word bit is gone. This also restores the earlier contract (registration at the receiver's first pending poll): semantics.md R3, deferred-replies.md and CHANGELOG are updated, and `shutdown_publication` goes back to its original deadline. Measured (micro, cores 4-7, load ~14, 6 interleaved runs each): burst_dl 3.43M -> 5.50M req/s (main raw, no deadline: 6.43M, -14.5%); burst_raw -2.9%. What remains of burst_dl is the deadline itself: `Deadline::after` (a clock read) and its conversion to ticks, ~145 instructions per request (callgrind: Deadline::after 65, to_ticks 80); the heap work is now under 0.1% of instructions. Callgrind (micro): burst_raw 103.4M -> 104.6M, seq_raw 105.8M -> 118.1M, seq_default 137.0M -> 141.2M (seq paths are scheduler-noise dominated; this change touches only raw deadline requests); burst_dl 113.6M. fmt, clippy (3 configurations), workspace tests, failpoints + death_boundary, sim 320/320, Loom reply_slot 25 / job_core 8 / link_fence 10 / registry_turn 3, traces 93/93 accepted with 130 negative-control counterexamples, model-sync events/models/paths: pass. No linearization point moved; no model change. --- CHANGELOG.md | 5 --- concurrency/src/reply/mod.rs | 43 ++++++++++++++++------- concurrency/src/reply/registry.rs | 32 +++++++++++++++-- concurrency/src/reply/slot.rs | 33 ++++++++++------- concurrency/tests/shutdown_publication.rs | 7 ++-- docs/design/deferred-replies.md | 17 +++++---- docs/design/semantics.md | 12 +++---- 7 files changed, 99 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad672eb3..a9410fdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,11 +23,6 @@ holds its owner at all (two flags became word bits), which keeps it in glibc's fast bins for typical reply sizes. Guarantees, Loom models and the TLA+ models are unchanged (no linearization point moved). -- **Behaviour (0.8)**: `send_request(msg, deadline)` puts a deadline in the - owner's deadline heap when the request is sent (it was when its receiver - was first awaited and pending): a raw receiver that is never awaited now - expires on time too. `request`/`request_with_timeout`/`Response` still - arm their own runtime timer. - `InsertError::kind()` (a `#[non_exhaustive]` `InsertErrorKind`) and `InsertError::into_parts()`: a refused insert's reason and token come together, so a caller answers the token for any reason, including one diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index cb0216d9..b72a9366 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -216,9 +216,11 @@ fn terminal_error(terminal: Terminal, slot: &Slot) -> ActorError { /// close records the final reason in the slot, and an undispatched /// envelope is disposed of through the owner's mailbox /// ([`SlotHandle::dispose`]). A `raw` request with a deadline (the public -/// `send_request`) puts its deadline in the owner's heap now, so its -/// receiver, awaited directly, needs no runtime timer and no owner; the -/// `request` paths arm their own timer and pass `raw = false`. +/// `send_request`) records the owner's registry key in the slot's word, so +/// its receiver, awaited directly and still pending, puts the deadline in +/// that owner's heap without a runtime timer (a request answered before +/// that never touches the heap); the `request` paths arm their own timer +/// and pass `raw = false`. pub(crate) fn reply_slot( deadline: Deadline, owner: Option<&Arc>, @@ -228,11 +230,11 @@ pub(crate) fn reply_slot( let slot_owner = owner.cloned(); #[cfg(not(feature = "lifecycle-trace"))] let slot_owner = (); - let slot = Arc::new(Slot::with_owner(deadline.ticks(), slot_owner)); - if let (true, Deadline::At(_), Some(owner)) = (raw, deadline, owner) { - slot.mark_tracked(); - owner.track_deadline(&slot); - } + let key = match (raw, deadline, owner) { + (true, Deadline::At(_), Some(owner)) => owner.key(), + _ => 0, + }; + let slot = Arc::new(Slot::with_owner_key(deadline.ticks(), slot_owner, key)); #[cfg(feature = "lifecycle-trace")] if let Some(owner) = owner { crate::lifecycle_trace::reply(owner.owner().raw(), "Send", Arc::as_ptr(&slot) as u64, ""); @@ -659,6 +661,7 @@ impl ReplyReceiver { first_ms: 0, sleep: None, timer: true, + tracked: false, } } @@ -724,6 +727,7 @@ impl std::future::IntoFuture for ReplyReceiver { first_ms: 0, sleep: None, timer: false, + tracked: false, } } } @@ -741,6 +745,8 @@ pin_project_lite::pin_project! { // Arm a runtime timer (`request`/`Response`), rather than putting the // deadline in the owner's heap (raw receivers). timer: bool, + // The deadline is in the owner's heap. + tracked: bool, #[pin] sleep: Option, } @@ -791,11 +797,22 @@ impl std::future::Future for ReplyFuture { if deadline == NEVER { return Poll::Pending; } - // A raw request put its deadline in the owner's heap at send; - // the service expires the slot (waking this future) without a - // per-request runtime timer. - if !*this.timer && slot.word() & slot::TRACKED != 0 { - return Poll::Pending; + // A raw request with a deadline puts it once in its owner's + // heap (found by the key in the word); the service expires the + // slot (waking this future) without a per-request runtime timer. + if !*this.timer { + let key = slot::owner_key(slot.word()); + if key != 0 { + if *this.tracked { + return Poll::Pending; + } + if let Some(owner) = Registry::by_key(key) { + owner.track_deadline(slot); + *this.tracked = true; + return Poll::Pending; + } + // The owner is gone: fall back to the runtime timer. + } } if *this.armed != deadline || this.sleep.is_none() { let wait = match std::mem::take(this.first_ms) { diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs index c5c36b50..a2d150c7 100644 --- a/concurrency/src/reply/registry.rs +++ b/concurrency/src/reply/registry.rs @@ -131,7 +131,7 @@ impl Registry { pub(crate) fn new(owner: ActorId, max_outstanding: usize) -> Arc { let max = max_outstanding.min(super::slot::MAX_JOBS as usize); static NEXT_KEY: AtomicU64 = AtomicU64::new(1); - Arc::new(Self { + let registry = Arc::new(Self { owner, key: NEXT_KEY.fetch_add(1, Ordering::Relaxed), pool: Arc::new(CreditPool::new(max)), @@ -147,7 +147,23 @@ impl Registry { discharge: Mutex::new(None), #[cfg(all(spawned_sim, feature = "async"))] sim: deadline::SimService::default(), - }) + }); + keys() + .lock() + .unwrap_or_else(|p| p.into_inner()) + .insert(registry.key, Arc::downgrade(®istry)); + registry + } + + /// The live registry with this `key` (a raw request's slot records it). + #[cfg_attr(not(feature = "async"), allow(dead_code))] // awaited receivers only + pub(crate) fn by_key(key: u64) -> Option> { + let weak = keys() + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&key) + .cloned()?; + weak.upgrade() } /// The registry of replies not tied to an actor (`reply_pair`): always @@ -516,8 +532,20 @@ pub(crate) fn contain_entry(step: impl FnOnce()) { } } +/// Registries by key, for a raw receiver that finds its owner by the key in +/// its slot's word (one insert per actor instance, one removal at its drop). +fn keys() -> &'static Mutex>> { + static KEYS: OnceLock>>> = OnceLock::new(); + KEYS.get_or_init(Default::default) +} + impl Drop for Registry { fn drop(&mut self) { + let forgotten = keys() + .lock() + .unwrap_or_else(|p| p.into_inner()) + .remove(&self.key); + drop(forgotten); // Every handle to the instance is gone; nothing can still answer. if self.turn.load() != DEAD { self.close(); diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index a3684155..ca463edb 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -2,7 +2,7 @@ //! (docs/design/deferred-replies.md §1, "Slot synchronization protocol"). //! //! ```text -//! word: state:4 | TOKEN:1 | REG:1 | JOBS:16 | PARKED:1 +//! word: state:4 | TOKEN:1 | REG:1 | JOBS:16 | PARKED:1 | -:1 | OWNER:40 //! credits(word) = max(JOBS, TOKEN and state is live ? 1 : 0) //! ``` //! @@ -60,10 +60,19 @@ const JOBS_MASK: u64 = MAX_JOBS << JOBS_SHIFT; /// and `credits` leave it alone). A word bit, not a field: it keeps the /// slot within the allocator's small size classes. pub(crate) const PARKED: u64 = 1 << (JOBS_SHIFT + 16); -/// The deadline was put in the owner's heap at send (a raw request with a -/// deadline): an awaited receiver waits for the heap's expiry instead of -/// arming its own timer. Set before the slot is shared; never changes. -pub(crate) const TRACKED: u64 = PARKED << 1; +/// The owner's registry key (`Registry::key`, nonzero) of a raw request +/// with a deadline, fixed at construction in the word's top bits (no +/// transition touches them): an awaited receiver finds the owner by it and +/// puts the deadline in the owner's heap on its first pending poll, without +/// the slot holding a reference to the owner. +const KEY_SHIFT: u32 = JOBS_SHIFT + 18; +/// Largest owner key the word holds; a larger one is not recorded. +pub(crate) const MAX_KEY: u64 = u64::MAX >> KEY_SHIFT; +#[inline] +#[cfg_attr(not(feature = "async"), allow(dead_code))] // awaited receivers only +pub(crate) fn owner_key(word: u64) -> u64 { + word >> KEY_SHIFT +} /// Deadline value meaning "never" (`Deadline::Never`). pub(crate) const NEVER: u64 = u64::MAX; @@ -296,9 +305,15 @@ impl Slot { } pub(crate) fn with_owner(deadline: u64, owner: super::sync::SlotOwner) -> Self { + Self::with_owner_key(deadline, owner, 0) + } + + /// A slot whose word records the owner's registry `key` (0: none). + pub(crate) fn with_owner_key(deadline: u64, owner: super::sync::SlotOwner, key: u64) -> Self { + let key = if key <= MAX_KEY { key } else { 0 }; Self { owner, - word: AtomicU64::new(OPEN), + word: AtomicU64::new(OPEN | (key << KEY_SHIFT)), deadline: AtomicU64::new(deadline), inner: Mutex::new(Inner { value: None, @@ -496,12 +511,6 @@ impl Slot { .reason = Some(reason); } - /// Mark the deadline as tracked by the owner's heap (before sharing). - #[allow(dead_code)] // the Loom model has no heap - pub(crate) fn mark_tracked(&self) { - self.word.fetch_or(TRACKED, Ordering::Release); - } - /// The final reason the owner's close recorded, if it closed this slot. #[allow(dead_code)] // the Loom model has no reasons to read pub(crate) fn stopped_reason(&self) -> Option { diff --git a/concurrency/tests/shutdown_publication.rs b/concurrency/tests/shutdown_publication.rs index 80550c99..1aaf4c40 100644 --- a/concurrency/tests/shutdown_publication.rs +++ b/concurrency/tests/shutdown_publication.rs @@ -123,11 +123,8 @@ fn admitted_request_racing_shutdown_drops_its_reply_before_join_completes() { let producer = std::thread::spawn(move || { PRODUCER.with(|p| p.set(true)); for _ in 0..1024 { - // No deadline: a raw request with one puts it in the owner's - // deadline heap at send, whose growth would trip the gate - // before the mailbox send this test holds. - let reply = - sender.send_request(Command::Work, spawned_concurrency::Deadline::Never); + let reply = sender + .send_request(Command::Work, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT); if PAUSED.load(Ordering::SeqCst) { return reply; } diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index 965b8c75..213d9a5b 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -102,13 +102,16 @@ inner: one mutex (the slot lock) over allocates it owner: none outside lifecycle-trace builds (whose events name it): a request takes no shared reference count on its owner. A raw - request with a deadline (`send_request`) puts the deadline in the - owner's heap at send and sets TRACKED in the word (an awaited - receiver then needs no timer and no owner); an undispatched - envelope is answered through the owner's mailbox disposal at the - death point (`discharge(replies)`). PARKED (bit 22) and TRACKED - (bit 23) are word bits no transition reads or clears; as bits, - they keep the slot inside the allocator's small size classes + request with a deadline (`send_request`) records the owner's + registry key in the word's top 40 bits; its receiver, awaited and + still pending, finds the owner by that key and puts the deadline + in the owner's heap once (a request answered first never touches + the heap, and no heap entry outlives a slot answered before it). + An undispatched envelope is answered through the owner's mailbox + disposal at the death point (`discharge(replies)`). PARKED (bit + 22) and the key are word bits no transition reads or changes; as + bits, they keep the slot inside the allocator's small size + classes the slot lock also orders a shortening against a reply's deadline read ``` diff --git a/docs/design/semantics.md b/docs/design/semantics.md index e9615b98..e253b6f6 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -154,12 +154,12 @@ the value), as an OTP reply already in the caller's mailbox when its wheel, the deadline service's wake-up) bounds that window. Which clock expires a waiting caller is part of the contract: `request`, `request_with_timeout` and `Response::from(rx)` arm the runtime's timer, so -they follow paused Tokio time; a request sent with `send_request` and a -deadline puts that deadline in the owner's deadline heap when it is sent, -and its `ReplyReceiver` awaited directly (`send_request(..)?.await`) waits -for that heap's expiry, serviced on the real clock (the simulated one under -`spawned_sim`), not paused Tokio time. (A receiver never awaited expires -then too.) The reason is cost: a raw receiver is the building +they follow paused Tokio time; a `ReplyReceiver` awaited directly +(`send_request(..)?.await`) puts its deadline in the owner's deadline heap +on its first pending poll instead, serviced on the real clock (the +simulated one under `spawned_sim`), not paused Tokio time. (The slot finds +its owner by a registry key in its word, so it holds no reference to it, +and a request answered before that poll never touches the heap.) The reason is cost: a raw receiver is the building block for fan-out and needs no runtime timer per request. Tests that pause Tokio time wait through `request` or `Response::from`. Either way a shortened deadline (`ReplyTo::shorten_deadline`) wakes the waiting caller. From 155682f49d8944311d2d0b5e579f96ed907a2582 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Fri, 13 Nov 2026 00:55:05 -0300 Subject: [PATCH 207/208] perf(tasks): a sent message carries no reply slot; a consumed PendingReply skips its drop The CI hot-path gate (benchmarks/hotpath.py, callgrind instructions per operation against main) measured async_ct_send_p1/p2 at +10.9%/+11.1%. Per operation, the plain-send path paid for request machinery: a `MessageEnvelope` with a detached `SlotHandle`, a `lent` field and their drops; `PendingReply::reply` on the detached obligation (the macro's send handler answers `()`), out of line, followed by the obligation's `Drop`. - `send` builds a `SendEnvelope` (`enqueue_send`): just the message; its dispatch builds a detached `PendingReply` and lends no caller. Requests keep `MessageEnvelope`. - `PendingReply::reply` is `#[inline]` and consumes the obligation through `ManuallyDrop`: once its slot is taken there is nothing left for `Drop` (fields are a reference and a marker). `Drop` itself is `#[inline]`. Semantics unchanged: a sent message still runs inside a turn (the turn phase classifies ReplyTo drops in it), and its `PendingReply` is detached as before. Gate (base 0791ace5 = main, Ir/op): async_ct_send_p1 855 -> 948 (+10.9%) now 841 (-1.6%) async_ct_send_p2 849 -> 944 (+11.1%) now 837 (-1.5%) async_ct_burst 3756 -> 4161 now 4119 (+9.7%) async_ct_request_seq 3479 -> 3709 now 3667 (+5.4%) async_ct_request_conc16 2589 -> 2923 now 2881 (+11.3%) threads_request_seq 3255 -> 2326 now 2269 (-30.3%) fmt, clippy (3 configurations), workspace tests, failpoints + death_boundary, sim 320/320, Loom reply_slot 25 / job_core 8 / link_fence 10 / registry_turn 3, traces 93/93 accepted with 130 negative-control counterexamples, model-sync: pass. No model change. --- concurrency/src/reply/mod.rs | 9 +++- concurrency/src/tasks/actor.rs | 75 ++++++++++++++++++++++++++++++++-- 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index b72a9366..7b8a9a08 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -389,8 +389,12 @@ impl<'t, R: Send + 'static> PendingReply<'t, R> { } /// Answer the caller now. A value that is not delivered is dropped. - pub fn reply(mut self, value: R) -> Delivery { - match self.slot.take() { + #[inline] + pub fn reply(self, value: R) -> Delivery { + // Consumed here: the obligation's `Drop` (an unanswered reply) has + // nothing to do once the slot is taken, so it is skipped. + let mut this = std::mem::ManuallyDrop::new(self); + match this.slot.take() { None => { drop(value); Delivery::CallerGone @@ -460,6 +464,7 @@ impl<'t, R: Send + 'static> PendingReply<'t, R> { } impl Drop for PendingReply<'_, R> { + #[inline] fn drop(&mut self) { if let Some(slot) = self.slot.take() { if slot.is_terminal() { diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index a67b12cb..26b4e03c 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -293,6 +293,53 @@ where } } +/// A sent message (no caller): no reply slot to hold, lend or dispose of, +/// so its dispatch builds only a detached `PendingReply`. +struct SendEnvelope { + msg: Option, + #[cfg(feature = "metrics")] + observation: Option, + #[cfg(feature = "lifecycle-trace")] + from: u64, +} + +impl Envelope for SendEnvelope +where + A: Actor + Handler, + M: Message, +{ + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + if self.from != 0 { + "msg" + } else { + crate::lifecycle_trace::life_kind::() + } + } + #[cfg(any(feature = "diagnostics", spawned_sim))] + fn message_type(&self) -> &'static str { + std::any::type_name::() + } + #[cfg(feature = "metrics")] + fn observation(&mut self) -> Option { + self.observation.take() + } + #[cfg(feature = "metrics")] + fn reject(mut self: Box) { + if let Some(observation) = self.observation.take() { + observation.reject(); + } + } + #[inline] + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let msg = dispatched(&mut self.msg); + ( + Box::pin(actor.handle(msg, PendingReply::detached(), ctx)), + None, + ) + } +} + impl> Envelope for DownEnvelope { #[cfg(feature = "lifecycle-trace")] fn trace_kind(&self) -> &'static str { @@ -353,6 +400,28 @@ where admit::(sender, replies, envelope, from) } +/// Admit a sent message (no caller): no reply slot at all. +fn enqueue_send( + sender: &mpsc::Sender + Send>>, + _mailbox: &crate::instrumentation::Mailbox, + replies: &Registry, + msg: M, +) -> Result<(), ActorError> +where + A: Actor + Handler, + M: Message, +{ + let from = crate::lifecycle_trace::life_sender::(replies.owner().raw(), true); + let envelope: Box + Send> = Box::new(SendEnvelope { + msg: Some(msg), + #[cfg(feature = "metrics")] + observation: Some(_mailbox.enqueued()), + #[cfg(feature = "lifecycle-trace")] + from, + }); + admit::(sender, replies, envelope, from) +} + /// A monitor's `Down`, enqueued by the framework (the waiter at the target's /// publication, or `monitor` past the target's death point). Its dispatch /// runs `Handler` only while the monitor's table entry is still @@ -732,12 +801,11 @@ impl Context { A: Handler, M: Message, { - enqueue( + enqueue_send( &self.shared.sender, &self.shared.mailbox, &self.shared.replies, msg, - SlotHandle::detached(), ) } @@ -1309,12 +1377,11 @@ impl ActorRef { A: Handler, M: Message, { - enqueue( + enqueue_send( &self.shared.sender, &self.shared.mailbox, &self.shared.replies, msg, - SlotHandle::detached(), ) } From e8847a3989bda5bd7d56776a20937a728a960689 Mon Sep 17 00:00:00 2001 From: Federico Carrone Date: Fri, 13 Nov 2026 01:09:01 -0300 Subject: [PATCH 208/208] perf(request): a clock seam for deadlines, tick arithmetic at send, and a leaner in-turn reply Request-path trims measured with the CI hot-path gate (benchmarks/hotpath.py gate, callgrind Ir/op against main 0791ace5): - `reply::clock` is the one place deadlines read time (`now`, `after`): today the precise `spawned_rt` clock, so a cheaper source can replace it without touching the reply protocol (that change awaits a decision and is not part of this commit). `request`/`request_with_timeout` fix their deadline as `clock::after(timeout)` in ticks (no `Instant::checked_add`, no `Deadline` round trip); `send_request_with` and `reply_slot` take ticks. The public `Deadline` API is unchanged. - `PendingReply`'s in-turn reply (`Slot::reply_in_turn`) takes a fast path: a slot answered in its own turn was never deferred (`defer()` consumes the obligation), so it holds no credit, job or side state: one CAS under the slot lock, then the receiver's waker. Any other word (closed first, or unexpectedly deferred) takes the general `publish`. - The first timer's millisecond count is computed in 64-bit arithmetic (no 128-bit division); `ReplyReceiver` and `SlotHandle` drops are inlined. Gate (base 0791ace5, Ir/op; before = 155682f4): async_ct_burst 4119 -> 4102 (+9.2%) async_ct_request_seq 3667 -> 3612 (+3.8%) async_ct_request_conc16 2881 -> 2826 (+9.2%) async_ct_send_p1/p2 unchanged (841/837, -1.6%/-1.5%) No semantics change: the deadline is the same instant (send time plus the timeout, in ticks), and the in-turn fast path is the same linearization (`OPEN -> REPLIED` under the slot lock) as `publish`. fmt, clippy (3 configurations), workspace tests, failpoints + death_boundary, sim 320/320, Loom reply_slot 25 / job_core 8 / link_fence 10 / registry_turn 3, traces 93/93 accepted with 130 negative-control counterexamples, model-sync: pass. --- concurrency/src/reply/mod.rs | 51 +++++++++++++++++++++++++++----- concurrency/src/reply/slot.rs | 45 ++++++++++++++++++++++++++-- concurrency/src/tasks/actor.rs | 20 ++++++------- concurrency/src/threads/actor.rs | 22 +++++++------- 4 files changed, 109 insertions(+), 29 deletions(-) diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs index 7b8a9a08..0fbca77d 100644 --- a/concurrency/src/reply/mod.rs +++ b/concurrency/src/reply/mod.rs @@ -23,6 +23,8 @@ use crate::ActorId; use registry::Registry; pub(crate) use slot::Slot; use slot::{Terminal, NEVER}; +/// `Deadline::Never` in ticks. +pub(crate) const NEVER_TICKS: u64 = NEVER; use spawned_rt::time::Instant; use std::borrow::Cow; use std::marker::PhantomData; @@ -121,7 +123,33 @@ mod sync { /// Clock ticks now, on the `spawned_rt` clock (virtual under simulation). pub(crate) fn now_ticks() -> u64 { - spawned_rt::time::now_ticks() + clock::now() +} + +/// The clock deadlines are fixed against: one seam, so the source (today +/// the precise `spawned_rt` clock) can change without touching the reply +/// protocol. Ticks are nanoseconds on the `spawned_rt` epoch; `NEVER` is +/// reserved. +pub(crate) mod clock { + use super::NEVER; + use std::time::Duration; + + /// Ticks now. + #[inline] + pub(crate) fn now() -> u64 { + spawned_rt::time::now_ticks() + } + + /// The deadline `timeout` from now, in ticks (`NEVER` if it overflows), + /// computed directly: no `Instant` arithmetic or conversion. + #[inline] + pub(crate) fn after(timeout: Duration) -> u64 { + u64::try_from(timeout.as_nanos()) + .ok() + .and_then(|nanos| now().checked_add(nanos)) + .filter(|ticks| *ticks < spawned_rt::time::MAX_TICKS) + .unwrap_or(NEVER) + } } // --------------------------------------------------------------------------- @@ -222,7 +250,7 @@ fn terminal_error(terminal: Terminal, slot: &Slot) -> ActorError { /// that never touches the heap); the `request` paths arm their own timer /// and pass `raw = false`. pub(crate) fn reply_slot( - deadline: Deadline, + deadline: u64, owner: Option<&Arc>, raw: bool, ) -> (SlotHandle, ReplyReceiver) { @@ -230,11 +258,11 @@ pub(crate) fn reply_slot( let slot_owner = owner.cloned(); #[cfg(not(feature = "lifecycle-trace"))] let slot_owner = (); - let key = match (raw, deadline, owner) { - (true, Deadline::At(_), Some(owner)) => owner.key(), + let key = match owner { + Some(owner) if raw && deadline != NEVER => owner.key(), _ => 0, }; - let slot = Arc::new(Slot::with_owner_key(deadline.ticks(), slot_owner, key)); + let slot = Arc::new(Slot::with_owner_key(deadline, slot_owner, key)); #[cfg(feature = "lifecycle-trace")] if let Some(owner) = owner { crate::lifecycle_trace::reply(owner.owner().raw(), "Send", Arc::as_ptr(&slot) as u64, ""); @@ -292,6 +320,7 @@ impl Caller for Arc> { /// `NoProc` by the send itself): closed `STOPPED` now (trace builds, whose /// slots know their owner, dispose of it there). impl Drop for SlotHandle { + #[inline] fn drop(&mut self) { if let Some(slot) = self.0.take() { #[cfg(feature = "lifecycle-trace")] @@ -635,6 +664,7 @@ pub struct ReplyReceiver { } impl Drop for ReplyReceiver { + #[inline] fn drop(&mut self) { if let Some(slot) = self.slot.take() { slot.abandon(); @@ -680,7 +710,14 @@ impl ReplyReceiver { #[cfg(feature = "async")] pub(crate) fn into_timed_future_after(self, timeout: Duration) -> ReplyFuture { let deadline = self.slot().deadline(); - let first_ms = u32::try_from(timeout.as_nanos().div_ceil(1_000_000)).unwrap_or(0); + // Whole milliseconds, rounded up, in 64-bit arithmetic (no 128-bit + // division); one that does not fit falls back to the clock. + let first_ms = timeout + .as_secs() + .checked_mul(1000) + .and_then(|ms| ms.checked_add(u64::from(timeout.subsec_nanos().div_ceil(1_000_000)))) + .and_then(|ms| u32::try_from(ms).ok()) + .unwrap_or(0); ReplyFuture { armed: deadline, first_ms, @@ -853,7 +890,7 @@ impl std::future::Future for ReplyFuture { pub(crate) fn reply_pair( deadline: Deadline, ) -> (PendingReply<'static, R>, ReplyReceiver) { - let (mut handle, receiver) = reply_slot(deadline, None, false); + let (mut handle, receiver) = reply_slot(deadline.ticks(), None, false); let reply = PendingReply::new(handle.take(), Registry::detached()); (reply, receiver) } diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs index ca463edb..d51cc8f0 100644 --- a/concurrency/src/reply/slot.rs +++ b/concurrency/src/reply/slot.rs @@ -672,10 +672,51 @@ impl Slot { /// A reply from the owner's own turn (`PendingReply::reply`): the /// owner's death point cannot run during its own turn, so no admission - /// is needed. + /// is needed. Such a slot was never deferred (`defer()` consumes the + /// `PendingReply`), so it holds no credit, no job and no side state: + /// one CAS with the value written under the lock, then the receiver's + /// waker. Anything else (a lost race, an unexpected word) takes + /// [`publish`](Self::publish). #[inline] pub(crate) fn reply_in_turn(&self, value: R) -> (Delivery, Option) { - self.publish(value, || Some(())) + let mut inner = lock(&self.inner); + let current = self.word(); + if state(current) != OPEN || credits(current) != 0 || inner.side.is_some() { + drop(inner); + return self.publish(value, || Some(())); + } + inner.value = Some(value); + // `PARKED` changes only under this lock, and nothing else moves a + // word without credits but a terminal transition: a failed CAS + // means the slot closed first. + let published = super::sync::section(|| { + let exchanged = self.word.compare_exchange( + current, + with_state(current, REPLIED), + Ordering::AcqRel, + Ordering::Acquire, + ); + if exchanged.is_ok() { + super::sync::left_open( + &self.owner, + self as *const Self as usize, + trace_name(REPLIED), + false, + ); + } + exchanged + }); + if published.is_err() { + let value = inner.value.take().expect("written above"); + drop(inner); + return self.publish(value, || Some(())); + } + let waker = inner.waker.take(); + drop(inner); + if let Some(waker) = waker { + waker.wake(); + } + (Delivery::Delivered, None) } /// [`reply`](Self::reply) admitted by the owner: `admit` returns a diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index 26b4e03c..4c21ebc9 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -824,7 +824,7 @@ impl Context { M: Message, { let deadline = deadline.into(); - self.send_request_with(msg, deadline, true) + self.send_request_with(msg, deadline.ticks(), true) } /// `send_request`; `raw` puts a deadline in the owner's heap now (the @@ -833,7 +833,7 @@ impl Context { pub(crate) fn send_request_with( &self, msg: M, - deadline: Deadline, + deadline: u64, raw: bool, ) -> Result, ActorError> where @@ -875,7 +875,7 @@ impl Context { // Bind the receiver before awaiting it: awaiting inside a `match` on // the `Result` keeps the scrutinee's storage alive beside the awaited // receiver and doubles this future's size. - let rx = match self.send_request_with(msg, Deadline::after(duration), false) { + let rx = match self.send_request_with(msg, crate::reply::clock::after(duration), false) { Ok(rx) => rx, Err(error) => { let result = Err(error); @@ -1258,7 +1258,7 @@ where msg: M, deadline: Deadline, ) -> Result, ActorError> { - Context::send_request_with(self, msg, deadline, false) + Context::send_request_with(self, msg, deadline.ticks(), false) } } @@ -1400,7 +1400,7 @@ impl ActorRef { M: Message, { let deadline = deadline.into(); - self.send_request_with(msg, deadline, true) + self.send_request_with(msg, deadline.ticks(), true) } /// `send_request`; `raw` puts a deadline in the owner's heap now (the @@ -1409,7 +1409,7 @@ impl ActorRef { pub(crate) fn send_request_with( &self, msg: M, - deadline: Deadline, + deadline: u64, raw: bool, ) -> Result, ActorError> where @@ -1445,7 +1445,7 @@ impl ActorRef { R: Send + 'static, F: FnOnce(&mut A, &Context) -> R + Send + 'static, { - let (slot, rx) = reply_slot(Deadline::Never, Some(&self.shared.replies), false); + let (slot, rx) = reply_slot(crate::reply::NEVER_TICKS, Some(&self.shared.replies), false); let envelope: Box + Send> = Box::new(InTurnEnvelope { f: Some(Box::new(f)), slot, @@ -1476,7 +1476,7 @@ impl ActorRef { // Bind the receiver before awaiting it: awaiting inside a `match` on // the `Result` keeps the scrutinee's storage alive beside the awaited // receiver and doubles this future's size. - let rx = match self.send_request_with(msg, Deadline::after(duration), false) { + let rx = match self.send_request_with(msg, crate::reply::clock::after(duration), false) { Ok(rx) => rx, Err(error) => { let result = Err(error); @@ -1602,7 +1602,7 @@ where msg: M, deadline: Deadline, ) -> Result, ActorError> { - ActorRef::send_request_with(self, msg, deadline, false) + ActorRef::send_request_with(self, msg, deadline.ticks(), false) } } @@ -4700,7 +4700,7 @@ mod idle_stop_regression { // Start and park the actual actor loop, without allowing another // executor task to run it between admission and stop below. assert!(futures::poll!(actor.as_mut()).is_pending()); - let (slot, response) = reply_slot(crate::reply::Deadline::Never, None, false); + let (slot, response) = reply_slot(crate::reply::NEVER_TICKS, None, false); tx.send(Box::new(MessageEnvelope { msg: Some(Increment), slot, diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 18edd00f..39476702 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -608,7 +608,7 @@ impl Context { M: Message, { let deadline = deadline.into(); - self.send_request_with(msg, deadline, true) + self.send_request_with(msg, deadline.ticks(), true) } /// `send_request`; `raw` puts a deadline in the owner's heap now (the @@ -617,7 +617,7 @@ impl Context { pub(crate) fn send_request_with( &self, msg: M, - deadline: Deadline, + deadline: u64, raw: bool, ) -> Result, ActorError> where @@ -649,7 +649,8 @@ impl Context { M: Message, { let observation = crate::instrumentation::RequestObservation::new(); - let result = match self.send_request_with(msg, Deadline::after(duration), false) { + let result = match self.send_request_with(msg, crate::reply::clock::after(duration), false) + { Err(error) => Err(error), Ok(rx) => rx.recv(), }; @@ -962,7 +963,7 @@ where msg: M, deadline: Deadline, ) -> Result, ActorError> { - Context::send_request_with(self, msg, deadline, false) + Context::send_request_with(self, msg, deadline.ticks(), false) } } @@ -1104,7 +1105,7 @@ impl ActorRef { M: Message, { let deadline = deadline.into(); - self.send_request_with(msg, deadline, true) + self.send_request_with(msg, deadline.ticks(), true) } /// `send_request`; `raw` puts a deadline in the owner's heap now (the @@ -1113,7 +1114,7 @@ impl ActorRef { pub(crate) fn send_request_with( &self, msg: M, - deadline: Deadline, + deadline: u64, raw: bool, ) -> Result, ActorError> where @@ -1142,7 +1143,7 @@ impl ActorRef { R: Send + 'static, F: FnOnce(&mut A, &Context) -> R + Send + 'static, { - let (slot, rx) = reply_slot(Deadline::Never, Some(&self.replies), false); + let (slot, rx) = reply_slot(crate::reply::NEVER_TICKS, Some(&self.replies), false); let envelope: Box + Send> = Box::new(InTurnEnvelope { f: Box::new(f), slot, @@ -1168,7 +1169,8 @@ impl ActorRef { M: Message, { let observation = crate::instrumentation::RequestObservation::new(); - let result = match self.send_request_with(msg, Deadline::after(duration), false) { + let result = match self.send_request_with(msg, crate::reply::clock::after(duration), false) + { Err(error) => Err(error), Ok(rx) => rx.recv(), }; @@ -1267,7 +1269,7 @@ where msg: M, deadline: Deadline, ) -> Result, ActorError> { - ActorRef::send_request_with(self, msg, deadline, false) + ActorRef::send_request_with(self, msg, deadline.ticks(), false) } } @@ -2706,7 +2708,7 @@ mod idle_stop_regression { drop(counter); }); entered.wait(); - let (slot, response) = reply_slot(crate::reply::Deadline::Never, None, false); + let (slot, response) = reply_slot(crate::reply::NEVER_TICKS, None, false); tx.send(MailboxItem::Message(Box::new(MessageEnvelope { msg: Increment(events.clone(), monitors), slot,